@secrecy/lib 1.50.0 → 1.51.0-feat-improvements.2

package/dist/lib/cache.js

@@ -8,6 +8,6 @@ export const dataContentCache = new LRUCache({
     max: 500,
     maxSize: gigaToBytes(0.5),
     sizeCalculation: (value) => {
-        return value.byteLength;
+        return value.data.byteLength;
     },
 });

package/dist/lib/client/SecrecyCloudClient.js

@@ -49,9 +49,9 @@ export class SecrecyCloudClient {
         }
         return internalNodeFullToNodeFull(node);
     }
-    async uploadLiteData({ data, encrypted = true, encryptProgress, uploadProgress, signal, }) {
+    async uploadData({ storageType, data, encrypted = true, encryptProgress, uploadProgress, signal, }) {
         const dataBuffer = data instanceof File ? new Uint8Array(await data.arrayBuffer()) : data;
-        if (dataBuffer.byteLength > kiloToBytes(4500)) {
+        if (storageType === 'lite' && dataBuffer.byteLength > kiloToBytes(4500)) {
             throw new Error('The data is too big for lite upload!');
         }
         const compressed = compress(dataBuffer);
@@ -70,137 +70,132 @@ export class SecrecyCloudClient {
             current: 0,
             percent: 0,
         });
-        const uploadData = await this.#apiClient.cloud.uploadLiteData.mutate(encryptedDataKey && md5Encrypted
-            ? {
-                type: 'encrypted',
-                content: Buffer.from(encryptedData),
-                sizeEncrypted: BigInt(encryptedData.byteLength),
-                size: BigInt(dataBuffer.byteLength),
-                key: sodium.to_hex(encryptedDataKey),
-                md5Encrypted,
-                md5: md5Data,
-            }
-            : {
-                type: 'unencrypted',
-                content: Buffer.from(encryptedData),
-                md5: md5Data,
-                size: BigInt(dataBuffer.byteLength),
-            }, { signal });
-        await uploadProgress?.({
-            total: encryptedData.byteLength,
-            current: encryptedData.byteLength,
-            percent: 1,
-        });
-        return uploadData.id;
-    }
-    async uploadData({ data, encrypted = true, encryptProgress, uploadProgress, signal, }) {
-        const dataKey = encrypted ? secretStreamKeygen() : null;
-        const dataBuffer = data instanceof File ? new Uint8Array(await data.arrayBuffer()) : data;
-        const compressed = compress(dataBuffer);
-        const { data: encryptedData, md5: md5Data, md5Encrypted, } = dataKey
-            ? await encrypt(dataKey, compressed, encryptProgress, signal)
-            : {
-                data: compressed,
-                md5: await md5(compressed),
-            };
-        const encryptedDataKey = dataKey
-            ? encryptCryptoBox(dataKey, this.#keys.publicKey, this.#keys.privateKey)
-            : null;
-        const uploadData = await this.#apiClient.cloud.uploadData.mutate(encryptedDataKey && md5Encrypted
-            ? {
-                type: 'encrypted',
-                sizeEncrypted: BigInt(encryptedData.byteLength),
-                size: BigInt(dataBuffer.byteLength),
-                key: sodium.to_hex(encryptedDataKey),
-                md5Encrypted,
-                md5: md5Data,
-            }
-            : {
-                type: 'unencrypted',
-                md5: md5Data,
-                size: BigInt(dataBuffer.byteLength),
-            }, { signal });
-        await uploadProgress?.({
-            total: encryptedData.byteLength,
-            current: 0,
-            percent: 0,
-        });
-        if (uploadData.parts.length === 0) {
-            if (uploadData.keyPair.pub !== this.#keys.publicKey) {
-                throw new Error('The public key does not match with cached key!');
-            }
+        if (storageType === 'lite') {
+            const uploadData = await this.#apiClient.cloud.uploadLiteData.mutate(encryptedDataKey && md5Encrypted
+                ? {
+                    type: 'encrypted',
+                    content: Buffer.from(encryptedData),
+                    sizeEncrypted: BigInt(encryptedData.byteLength),
+                    size: BigInt(dataBuffer.byteLength),
+                    key: sodium.to_hex(encryptedDataKey),
+                    md5Encrypted,
+                    md5: md5Data,
+                }
+                : {
+                    type: 'unencrypted',
+                    content: Buffer.from(encryptedData),
+                    md5: md5Data,
+                    size: BigInt(dataBuffer.byteLength),
+                }, { signal });
             await uploadProgress?.({
                 total: encryptedData.byteLength,
                 current: encryptedData.byteLength,
                 percent: 1,
             });
-            return uploadData.id;
-        }
-        const uploadDataPartEnd = async (md5, order) => {
-            return this.#apiClient.cloud.uploadDataPartEnd.mutate({
-                dataId: uploadData.id,
-                md5,
-                order,
-            }, { signal });
-        };
-        const chunkParts = new Array();
-        for (const [index, chunk] of enumerate(chunks(encryptedData, Number(uploadData.partSize)))) {
-            chunkParts.push({
-                order: index + 1,
-                data: chunk,
-                md5: await md5(chunk),
-            });
+            return {
+                object: 'lite',
+                id: uploadData.id,
+            };
         }
-        const progressParts = {};
-        const onProgress = (part, progressEvent) => {
-            progressParts[part] = progressEvent;
-            const current = Object.values(progressParts).reduce((prv, cur) => prv + cur.loaded, 0);
-            void uploadProgress?.({
-                percent: current / encryptedData.byteLength,
-                total: encryptedData.byteLength,
-                current,
-            });
-        };
-        const byPart = async (part) => {
-            const formData = new FormData();
-            const chunk = chunkParts.find((p) => p.order === part.order);
-            if (chunk === undefined) {
-                return;
+        if (storageType === 's3') {
+            const uploadDataArgs = encryptedDataKey && md5Encrypted
+                ? {
+                    type: 'encrypted',
+                    sizeEncrypted: BigInt(encryptedData.byteLength),
+                    size: BigInt(dataBuffer.byteLength),
+                    key: sodium.to_hex(encryptedDataKey),
+                    md5Encrypted,
+                    md5: md5Data,
+                }
+                : {
+                    type: 'unencrypted',
+                    md5: md5Data,
+                    size: BigInt(dataBuffer.byteLength),
+                };
+            const uploadData = await this.#apiClient.cloud.uploadData.mutate(uploadDataArgs, { signal });
+            if (uploadData.parts.length === 0) {
+                if (uploadData.keyPair.pub !== this.#keys.publicKey) {
+                    throw new Error('The public key does not match with cached key!');
+                }
+                await uploadProgress?.({
+                    total: encryptedData.byteLength,
+                    current: encryptedData.byteLength,
+                    percent: 1,
+                });
+                return {
+                    object: 's3',
+                    id: uploadData.id,
+                };
             }
-            for (const [key, value] of Object.entries(part.fields)) {
-                formData.append(key, value);
+            const uploadDataPartEnd = async (md5, order) => {
+                return this.#apiClient.cloud.uploadDataPartEnd.mutate({
+                    dataId: uploadData.id,
+                    md5,
+                    order,
+                }, { signal });
+            };
+            const chunkParts = new Array();
+            for (const [index, chunk] of enumerate(chunks(encryptedData, Number(uploadData.partSize)))) {
+                chunkParts.push({
+                    order: index + 1,
+                    data: chunk,
+                    md5: await md5(chunk),
+                });
             }
-            formData.append('file', new Blob([chunk.data]), `${uploadData.id}-${chunk.order}`);
-            await axios.post(part.url, formData, {
-                onUploadProgress: (progressEvent) => {
-                    onProgress(part.order, progressEvent);
-                },
-                signal,
+            const progressParts = {};
+            const onProgress = (part, progressEvent) => {
+                progressParts[part] = progressEvent;
+                const current = Object.values(progressParts).reduce((prv, cur) => prv + cur.loaded, 0);
+                void uploadProgress?.({
+                    percent: current / encryptedData.byteLength,
+                    total: encryptedData.byteLength,
+                    current,
+                });
+            };
+            const byPart = async (part) => {
+                const formData = new FormData();
+                const chunk = chunkParts.find((p) => p.order === part.order);
+                if (chunk === undefined) {
+                    return;
+                }
+                for (const [key, value] of Object.entries(part.fields)) {
+                    formData.append(key, value);
+                }
+                formData.append('file', new Blob([chunk.data]), `${uploadData.id}-${chunk.order}`);
+                await axios.post(part.url, formData, {
+                    onUploadProgress: (progressEvent) => {
+                        onProgress(part.order, progressEvent);
+                    },
+                    signal,
+                });
+                return uploadDataPartEnd(chunk.md5, chunk.order);
+            };
+            await promiseAllLimit(3, uploadData.parts.map((p) => async () => {
+                await byPart(p);
+            }));
+            dataContentCache.set(uploadData.id, {
+                id: uploadData.id,
+                storageType: 's3',
+                size: uploadDataArgs.size,
+                data: dataBuffer,
             });
-            return uploadDataPartEnd(chunk.md5, chunk.order);
-        };
-        await promiseAllLimit(3, uploadData.parts.map((p) => async () => {
-            await byPart(p);
-        }));
-        dataContentCache.set(uploadData.id, dataBuffer);
-        return uploadData.id;
+            return {
+                id: uploadData.id,
+                object: 's3',
+            };
+        }
+        throw new Error(`The "${storageType}" is not implemented yet!`);
     }
-    async uploadDataInCloud({ data, name, nodeId, encryptProgress, uploadProgress, signal, isLite, }) {
-        const dataId = isLite
-            ? await this.uploadLiteData({
-                data,
-                encryptProgress,
-                uploadProgress,
-                signal,
-            })
-            : await this.uploadData({
-                data,
-                encryptProgress,
-                uploadProgress,
-                signal,
-            });
+    async uploadDataInCloud({ data, name, nodeId, encryptProgress, uploadProgress, storageType = 's3', signal, }) {
+        const uploadedData = await this.uploadData({
+            storageType,
+            data,
+            encryptProgress,
+            uploadProgress,
+            signal,
+        });
         return await this.saveInCloud({
-            dataId,
+            dataId: uploadedData.id,
             name,
             nodeId,
         });
@@ -415,7 +410,141 @@ export class SecrecyCloudClient {
             throw `Can't find content for data ${dataId}`;
         }
         const data = await finalize(encryptedContent);
-        dataContentCache.set(dataId, data);
+        dataContentCache.set(dataId, {
+            id: dataId,
+            storageType: dataContent.storageType,
+            size: dataContent.totalSize,
+            data,
+        });
+        return {
+            id: dataId,
+            storageType: dataContent.storageType,
+            data,
+        };
+    }
+    async dataContents({ dataIds, onDownloadProgress, progressDecrypt, signal, }) {
+        const cachedData = dataIds
+            .map((dataId) => dataContentCache.get(dataId))
+            .filter((data) => typeof data !== 'undefined');
+        if (cachedData.length === dataIds.length) {
+            return cachedData;
+        }
+        const missingContents = await this.#apiClient.cloud.dataContentByIds.query({
+            ids: dataIds.filter((dataId) => !cachedData.some((datum) => datum.id === dataId)),
+        });
+        const allDataContents = [
+            ...missingContents.map((data) => ({
+                id: data.id,
+                size: data.totalSize,
+                storageType: data.storageType,
+            })),
+            ...cachedData.map((data) => ({
+                id: data.id,
+                size: data.size,
+                storageType: data.storageType,
+            })),
+        ];
+        const allDataContentsBytes = Number(allDataContents.reduce((curr, next) => curr + next.size, 0n));
+        const progressParts = {};
+        const onProgress = (dataId, part, progressEvent) => {
+            progressParts[`${dataId}-${part}`] = progressEvent;
+            const transferredBytes = Object.values(progressParts).reduce((prv, cur) => prv + cur.transferredBytes, 0);
+            onDownloadProgress?.({
+                percent: transferredBytes / allDataContentsBytes,
+                totalBytes: allDataContentsBytes,
+                transferredBytes,
+            });
+        };
+        const encryptedContentFromParts = async ({ dataId, dataParts, }) => {
+            const parts = {};
+            const byPart = async (part) => {
+                const buf = new Uint8Array(await ky
+                    .get(part.contentUrl, {
+                    timeout: false,
+                    onDownloadProgress: (pr) => {
+                        onProgress(dataId, part.order, pr);
+                    },
+                    signal,
+                })
+                    .arrayBuffer());
+                const md5Part = await md5(buf);
+                if (md5Part !== part.md5) {
+                    throw new Error(`Invalid md5 for part ${part.order} of data ${dataId}`);
+                }
+                if (typeof parts[dataId] === 'undefined') {
+                    parts[dataId] = [
+                        {
+                            data: buf,
+                            order: part.order,
+                        },
+                    ];
+                }
+                else {
+                    parts[dataId].push({
+                        data: buf,
+                        order: part.order,
+                    });
+                }
+            };
+            await promiseAllLimit(3, dataParts.map((p) => async () => byPart(p)));
+            return concatenate(...parts[dataId].sort((a, b) => a.order - b.order).map((p) => p.data));
+        };
+        const finalize = async (dataContent, encryptedContent) => {
+            // const md5Encrypted = await firstValueFrom(md5(of(encryptedContent)));
+            const md5Encrypted = await md5(encryptedContent);
+            if (md5Encrypted !== dataContent.md5Encrypted) {
+                throw new Error(`Encrypted content does not match`);
+            }
+            const key = dataContent.key
+                ? decryptCryptoBox(sodium.from_hex(dataContent.key), dataContent.type === 'received_mail'
+                    ? dataContent.senderPublicKey
+                    : dataContent.type === 'cloud'
+                        ? dataContent.publicKey
+                        : this.#keys.publicKey, this.#keys.privateKey)
+                : null;
+            const src = key
+                ? await decrypt(key, encryptedContent, progressDecrypt, signal)
+                : encryptedContent;
+            // const md5Content = await firstValueFrom(md5(of(src)));
+            const md5Content = await md5(src);
+            if (md5Content !== dataContent.md5) {
+                throw new Error(`Content does not match`);
+            }
+            return decompress(src);
+        };
+        const encrypt = async (dataContent) => {
+            const encryptedContent = dataContent.type === 'lite'
+                ? new Uint8Array(dataContent.content)
+                : dataContent.type === 'cloud'
+                    ? await encryptedContentFromParts({
+                        dataId: dataContent.id,
+                        dataParts: dataContent.parts,
+                    })
+                    : dataContent.maybeContent !== null
+                        ? new Uint8Array(dataContent.maybeContent)
+                        : dataContent.maybeParts !== null
+                            ? await encryptedContentFromParts({
+                                dataId: dataContent.id,
+                                dataParts: dataContent.maybeParts,
+                            })
+                            : null;
+            if (encryptedContent === null) {
+                throw `Can't find content for data ${dataContent.id}`;
+            }
+            return encryptedContent;
+        };
+        const data = await Promise.all(missingContents.map(async (missingContent) => {
+            const encrypted = await encrypt(missingContent);
+            const finalized = await finalize(missingContent, encrypted);
+            const datum = {
+                id: missingContent.id,
+                size: missingContent.totalSize,
+                storageType: missingContent.storageType,
+                data: finalized,
+            };
+            dataContentCache.set(missingContent.id, datum);
+            return datum;
+        }));
         return data;
     }
     async deleteData({ dataId, nodeId, }) {
@@ -431,11 +560,6 @@ export class SecrecyCloudClient {
         });
         return isDeleted;
     }
-    async deleteNodes({ nodeIds, }) {
-        return this.#apiClient.cloud.deleteNodes.mutate({
-            ids: nodeIds,
-        });
-    }
     async emptyTrash() {
         const { isCleaned } = await this.#apiClient.cloud.emptyNodeCloudTrash.mutate({});
         return isCleaned;

package/dist/types/cache.d.ts

@@ -1,4 +1,4 @@
-import type { InternalNode, InternalData, InternalNodeFull } from './client/types/index.js';
+import type { InternalNode, InternalData, InternalNodeFull, DataStorageType } from './client/types/index.js';
 import { LRUCache } from 'lru-cache';
 export declare const dataCache: Map<string, InternalData>;
 export declare const nodesCache: Map<string, InternalNode | InternalNodeFull>;
@@ -10,4 +10,9 @@ export declare const usersCache: Map<string, {
     isSearchable: boolean;
 }>;
 export declare const publicKeysCache: Map<string, string>;
-export declare const dataContentCache: LRUCache<string, Uint8Array, unknown>;
+export declare const dataContentCache: LRUCache<string, {
+    id: string;
+    storageType: DataStorageType;
+    size: bigint;
+    data: Uint8Array;
+}, unknown>;

package/dist/types/client/SecrecyCloudClient.d.ts

@@ -1,5 +1,5 @@
 import type { ProgressCallback, SecrecyClient } from '../index.js';
-import type { DataMetadata, KeyPair, Node, NodeFull, NodeType, Rights } from './types/index.js';
+import type { DataMetadata, DataStorageType, KeyPair, Node, NodeFull, NodeType, Rights } from './types/index.js';
 import { type RouterInputs, type ApiClient, type RouterOutputs } from '../client.js';
 import { type DownloadProgress } from '../types.js';
 export declare class SecrecyCloudClient {
@@ -9,28 +9,25 @@ export declare class SecrecyCloudClient {
         dataId: string;
         nodeId: string;
     }): Promise<NodeFull>;
-    uploadLiteData({ data, encrypted, encryptProgress, uploadProgress, signal, }: {
+    uploadData({ storageType, data, encrypted, encryptProgress, uploadProgress, signal, }: {
+        storageType: DataStorageType;
         data: globalThis.File | Uint8Array;
         encrypted?: boolean;
         encryptProgress?: ProgressCallback;
         uploadProgress?: ProgressCallback;
         signal?: AbortSignal;
-    }): Promise<string>;
-    uploadData({ data, encrypted, encryptProgress, uploadProgress, signal, }: {
-        data: globalThis.File | Uint8Array;
-        encrypted?: boolean;
-        encryptProgress?: ProgressCallback;
-        uploadProgress?: ProgressCallback;
-        signal?: AbortSignal;
-    }): Promise<string>;
-    uploadDataInCloud({ data, name, nodeId, encryptProgress, uploadProgress, signal, isLite, }: {
+    }): Promise<{
+        id: string;
+        object: DataStorageType;
+    }>;
+    uploadDataInCloud({ data, name, nodeId, encryptProgress, uploadProgress, storageType, signal, }: {
         data: globalThis.File | Uint8Array;
         name: string;
         nodeId?: string;
         encryptProgress?: ProgressCallback;
         uploadProgress?: ProgressCallback;
         signal?: AbortSignal;
-        isLite?: boolean;
+        storageType?: DataStorageType;
     }): Promise<NodeFull>;
     deletedNodes(): Promise<Node[]>;
     sharedNodes(): Promise<Node[]>;
@@ -74,7 +71,21 @@ export declare class SecrecyCloudClient {
         onDownloadProgress?: (progress: DownloadProgress) => void;
         progressDecrypt?: ProgressCallback;
         signal?: AbortSignal;
-    }): Promise<Uint8Array>;
+    }): Promise<{
+        id: string;
+        storageType: DataStorageType;
+        data: Uint8Array;
+    }>;
+    dataContents({ dataIds, onDownloadProgress, progressDecrypt, signal, }: {
+        dataIds: string[];
+        onDownloadProgress?: (progress: DownloadProgress) => void;
+        progressDecrypt?: ProgressCallback;
+        signal?: AbortSignal;
+    }): Promise<{
+        id: string;
+        storageType: DataStorageType;
+        data: Uint8Array;
+    }[]>;
     deleteData({ dataId, nodeId, }: {
         dataId: string;
         nodeId: string;
@@ -82,11 +93,6 @@ export declare class SecrecyCloudClient {
     deleteNode({ nodeId }: {
         nodeId: string;
     }): Promise<boolean>;
-    deleteNodes({ nodeIds, }: {
-        nodeIds: string[];
-    }): Promise<{
-        count: number;
-    }>;
     emptyTrash(): Promise<boolean>;
     recoverNode(id: string): Promise<boolean>;
     moveNodes({ nodeIds, parentNodeId, }: {

package/dist/types/client/types/data.d.ts

@@ -1,6 +1,8 @@
 import { type RouterOutputs } from '../../client.js';
+export type ApiData = NonNullable<RouterOutputs['cloud']['dataById']>;
+export type ApiExtendedData = NonNullable<RouterOutputs['cloud']['dataContentByIds'][number]>;
+export type DataStorageType = ApiData['storageType'];
 export type DataMetadata = Pick<ApiData, 'id' | 'size' | 'sizeEncrypted' | 'md5' | 'md5Encrypted' | 'createdAt'>;
 export type InternalData = DataMetadata & {
     key: string | null;
 };
-export type ApiData = NonNullable<RouterOutputs['cloud']['dataById']>;