@protontech/drive-sdk 0.4.1 → 0.5.1

package/src/internal/nodes/apiService.ts

@@ -1,11 +1,12 @@
 import { c } from 'ttag';
 
-import { ProtonDriveError, ValidationError } from '../../errors';
+import { NodeWithSameNameExistsValidationError, ProtonDriveError, ValidationError } from '../../errors';
 import { Logger, NodeResult } from '../../interface';
 import { MemberRole, RevisionState } from '../../interface/nodes';
 import {
     DriveAPIService,
     drivePaths,
+    ErrorCode,
     InvalidRequirementsAPIError,
     isCodeOk,
     nodeTypeNumberToNodeType,
@@ -128,7 +129,7 @@ export class NodeAPIService {
 
     async *iterateNodes(
         nodeUids: string[],
-        ownVolumeId: string,
+        ownVolumeId: string | undefined,
         filterOptions?: FilterOptions,
         signal?: AbortSignal,
     ): AsyncGenerator<EncryptedNode> {
@@ -389,55 +390,49 @@ export class NodeAPIService {
         return makeNodeUid(volumeId, response.LinkID);
     }
 
-    // Improvement requested: split into multiple calls for many nodes.
     async *trashNodes(nodeUids: string[], signal?: AbortSignal): AsyncGenerator<NodeResult> {
-        const nodeIds = nodeUids.map(splitNodeUid);
-        const volumeId = assertAndGetSingleVolumeId(c('Operation').t`Trashing items`, nodeIds);
-
-        const response = await this.apiService.post<PostTrashNodesRequest, PostTrashNodesResponse>(
-            `drive/v2/volumes/${volumeId}/trash_multiple`,
-            {
-                LinkIDs: nodeIds.map(({ nodeId }) => nodeId),
-            },
-            signal,
-        );
+        for (const { volumeId, batchNodeIds, batchNodeUids } of groupNodeUidsByVolumeAndIteratePerBatch(nodeUids)) {
+            const response = await this.apiService.post<PostTrashNodesRequest, PostTrashNodesResponse>(
+                `drive/v2/volumes/${volumeId}/trash_multiple`,
+                {
+                    LinkIDs: batchNodeIds,
+                },
+                signal,
+            );
 
-        // TODO: remove `as` when backend fixes OpenAPI schema.
-        yield* handleResponseErrors(nodeUids, volumeId, response.Responses as LinkResponse[]);
+            // TODO: remove `as` when backend fixes OpenAPI schema.
+            yield* handleResponseErrors(batchNodeUids, volumeId, response.Responses as LinkResponse[]);
+        }
     }
 
-    // Improvement requested: split into multiple calls for many nodes.
     async *restoreNodes(nodeUids: string[], signal?: AbortSignal): AsyncGenerator<NodeResult> {
-        const nodeIds = nodeUids.map(splitNodeUid);
-        const volumeId = assertAndGetSingleVolumeId(c('Operation').t`Restoring items`, nodeIds);
-
-        const response = await this.apiService.put<PutRestoreNodesRequest, PutRestoreNodesResponse>(
-            `drive/v2/volumes/${volumeId}/trash/restore_multiple`,
-            {
-                LinkIDs: nodeIds.map(({ nodeId }) => nodeId),
-            },
-            signal,
-        );
+        for (const { volumeId, batchNodeIds, batchNodeUids } of groupNodeUidsByVolumeAndIteratePerBatch(nodeUids)) {
+            const response = await this.apiService.put<PutRestoreNodesRequest, PutRestoreNodesResponse>(
+                `drive/v2/volumes/${volumeId}/trash/restore_multiple`,
+                {
+                    LinkIDs: batchNodeIds,
+                },
+                signal,
+            );
 
-        // TODO: remove `as` when backend fixes OpenAPI schema.
-        yield* handleResponseErrors(nodeUids, volumeId, response.Responses as LinkResponse[]);
+            // TODO: remove `as` when backend fixes OpenAPI schema.
+            yield* handleResponseErrors(batchNodeUids, volumeId, response.Responses as LinkResponse[]);
+        }
     }
 
-    // Improvement requested: split into multiple calls for many nodes.
     async *deleteNodes(nodeUids: string[], signal?: AbortSignal): AsyncGenerator<NodeResult> {
-        const nodeIds = nodeUids.map(splitNodeUid);
-        const volumeId = assertAndGetSingleVolumeId(c('Operation').t`Deleting items`, nodeIds);
-
-        const response = await this.apiService.post<PostDeleteNodesRequest, PostDeleteNodesResponse>(
-            `drive/v2/volumes/${volumeId}/trash/delete_multiple`,
-            {
-                LinkIDs: nodeIds.map(({ nodeId }) => nodeId),
-            },
-            signal,
-        );
+        for (const { volumeId, batchNodeIds, batchNodeUids } of groupNodeUidsByVolumeAndIteratePerBatch(nodeUids)) {
+            const response = await this.apiService.post<PostDeleteNodesRequest, PostDeleteNodesResponse>(
+                `drive/v2/volumes/${volumeId}/trash/delete_multiple`,
+                {
+                    LinkIDs: batchNodeIds,
+                },
+                signal,
+            );
 
-        // TODO: remove `as` when backend fixes OpenAPI schema.
-        yield* handleResponseErrors(nodeUids, volumeId, response.Responses as LinkResponse[]);
+            // TODO: remove `as` when backend fixes OpenAPI schema.
+            yield* handleResponseErrors(batchNodeUids, volumeId, response.Responses as LinkResponse[]);
+        }
     }
 
     async createFolder(
@@ -455,21 +450,41 @@ export class NodeAPIService {
     ): Promise<string> {
         const { volumeId, nodeId: parentId } = splitNodeUid(parentUid);
 
-        const response = await this.apiService.post<PostCreateFolderRequest, PostCreateFolderResponse>(
-            `drive/v2/volumes/${volumeId}/folders`,
-            {
-                ParentLinkID: parentId,
-                NodeKey: newNode.armoredKey,
-                NodeHashKey: newNode.armoredHashKey,
-                NodePassphrase: newNode.armoredNodePassphrase,
-                NodePassphraseSignature: newNode.armoredNodePassphraseSignature,
-                SignatureEmail: newNode.signatureEmail,
-                Name: newNode.encryptedName,
-                Hash: newNode.hash,
-                // @ts-expect-error: XAttr is optional as undefined.
-                XAttr: newNode.armoredExtendedAttributes,
-            },
-        );
+        let response: PostCreateFolderResponse;
+        try {
+            response = await this.apiService.post<PostCreateFolderRequest, PostCreateFolderResponse>(
+                `drive/v2/volumes/${volumeId}/folders`,
+                {
+                    ParentLinkID: parentId,
+                    NodeKey: newNode.armoredKey,
+                    NodeHashKey: newNode.armoredHashKey,
+                    NodePassphrase: newNode.armoredNodePassphrase,
+                    NodePassphraseSignature: newNode.armoredNodePassphraseSignature,
+                    SignatureEmail: newNode.signatureEmail,
+                    Name: newNode.encryptedName,
+                    Hash: newNode.hash,
+                    // @ts-expect-error: XAttr is optional as undefined.
+                    XAttr: newNode.armoredExtendedAttributes,
+                },
+            );
+        } catch (error: unknown) {
+            if (error instanceof ValidationError) {
+                if (error.code === ErrorCode.ALREADY_EXISTS) {
+                    const typedDetails = error.details as
+                        | {
+                              ConflictLinkID: string;
+                          }
+                        | undefined;
+
+                    const existingNodeUid = typedDetails?.ConflictLinkID
+                        ? makeNodeUid(volumeId, typedDetails.ConflictLinkID)
+                        : undefined;
+
+                    throw new NodeWithSameNameExistsValidationError(error.message, error.code, existingNodeUid);
+                }
+            }
+            throw error;
+        }
 
         return makeNodeUid(volumeId, response.Folder.ID);
     }
@@ -513,15 +528,6 @@ export class NodeAPIService {
     }
 }
 
-function assertAndGetSingleVolumeId(operationForErrorMessage: string, nodeIds: { volumeId: string }[]): string {
-    const uniqueVolumeIds = new Set(nodeIds.map(({ volumeId }) => volumeId));
-    if (uniqueVolumeIds.size !== 1) {
-        throw new ValidationError(c('Error').t`${operationForErrorMessage} from multiple sections is not allowed`);
-    }
-    const volumeId = nodeIds[0].volumeId;
-    return volumeId;
-}
-
 type LinkResponse = {
     LinkID: string;
     Response: {
@@ -657,6 +663,34 @@ function linkToEncryptedNode(
     throw new Error(`Unknown node type: ${link.Link.Type}`);
 }
 
+export function* groupNodeUidsByVolumeAndIteratePerBatch(
+    nodeUids: string[],
+): Generator<{ volumeId: string; batchNodeIds: string[]; batchNodeUids: string[] }> {
+    const allNodeIds = nodeUids.map((nodeUid: string) => {
+        const { volumeId, nodeId } = splitNodeUid(nodeUid);
+        return { volumeId, nodeIds: { nodeId, nodeUid } };
+    });
+
+    const nodeIdsByVolumeId = new Map<string, { nodeId: string; nodeUid: string }[]>();
+    for (const { volumeId, nodeIds } of allNodeIds) {
+        if (!nodeIdsByVolumeId.has(volumeId)) {
+            nodeIdsByVolumeId.set(volumeId, []);
+        }
+        nodeIdsByVolumeId.get(volumeId)?.push(nodeIds);
+    }
+
+    for (const [volumeId, nodeIds] of nodeIdsByVolumeId.entries()) {
+        for (const nodeIdsBatch of batch(nodeIds, API_NODES_BATCH_SIZE)) {
+            yield {
+                volumeId,
+                batchNodeIds: nodeIdsBatch.map(({ nodeId }) => nodeId),
+                batchNodeUids: nodeIdsBatch.map(({ nodeUid }) => nodeUid),
+            };
+        }
+    }
+}
+
+
 function transformRevisionResponse(
     volumeId: string,
     nodeId: string,

package/src/internal/nodes/debouncer.test.ts

@@ -0,0 +1,141 @@
+import { ProtonDriveTelemetry } from '../../interface';
+import { getMockTelemetry } from '../../tests/telemetry';
+import { NodesDebouncer } from './debouncer';
+
+describe('NodesDebouncer', () => {
+    let debouncer: NodesDebouncer;
+    let mockTelemetry: ReturnType<typeof getMockTelemetry>;
+
+    beforeEach(() => {
+        mockTelemetry = getMockTelemetry();
+        debouncer = new NodesDebouncer(mockTelemetry);
+
+        jest.useFakeTimers();
+    });
+
+    afterEach(() => {
+        jest.useRealTimers();
+        debouncer.clear();
+    });
+
+    it('should register a node for loading and wait for it to finish', async () => {
+        const nodeUid = 'test-node-1';
+        debouncer.loadingNode(nodeUid);
+
+        // Verify that the node is registered by checking if waitForLoadingNode works
+        const waitPromise = debouncer.waitForLoadingNode(nodeUid);
+        expect(waitPromise).toBeInstanceOf(Promise);
+
+        // Finish loading to clean up
+        debouncer.finishedLoadingNode(nodeUid);
+        await waitPromise;
+    });
+
+    it('should allow multiple nodes to be registered', async () => {
+        const nodeUid1 = 'test-node-1';
+        const nodeUid2 = 'test-node-2';
+
+        debouncer.loadingNode(nodeUid1);
+        debouncer.loadingNode(nodeUid2);
+
+        const wait1 = debouncer.waitForLoadingNode(nodeUid1);
+        const wait2 = debouncer.waitForLoadingNode(nodeUid2);
+
+        expect(wait1).toBeInstanceOf(Promise);
+        expect(wait2).toBeInstanceOf(Promise);
+
+        debouncer.finishedLoadingNode(nodeUid1);
+        debouncer.finishedLoadingNode(nodeUid2);
+        await Promise.all([wait1, wait2]);
+    });
+
+    it('should register multiple nodes at once', async () => {
+        const nodeUid1 = 'test-node-1';
+        const nodeUid2 = 'test-node-2';
+
+        debouncer.loadingNodes([nodeUid1, nodeUid2]);
+
+        const wait1 = debouncer.waitForLoadingNode(nodeUid1);
+        const wait2 = debouncer.waitForLoadingNode(nodeUid2);
+
+        expect(wait1).toBeInstanceOf(Promise);
+        expect(wait2).toBeInstanceOf(Promise);
+
+        debouncer.finishedLoadingNode(nodeUid1);
+        debouncer.finishedLoadingNode(nodeUid2);
+        await Promise.all([wait1, wait2]);
+    });
+
+    it('should warn about registering the same node twice', async () => {
+        const nodeUid = 'test-node-1';
+
+        // Register the same node twice
+        debouncer.loadingNode(nodeUid);
+        debouncer.loadingNode(nodeUid);
+
+        expect(mockTelemetry.mockLogger.warn).toHaveBeenCalledWith(`Loading twice for: ${nodeUid}`);
+    });
+
+    it('should send metric when waiting for a long time', async () => {
+        const nodeUid = 'test-node-1';
+        debouncer.loadingNode(nodeUid);
+
+        const waitPromise = debouncer.waitForLoadingNode(nodeUid);
+        expect(mockTelemetry.recordMetric).not.toHaveBeenCalled();
+        jest.advanceTimersByTime(1500);
+        expect(mockTelemetry.recordMetric).toHaveBeenCalledWith({
+            eventName: 'debounceLongWait',
+        });
+
+        debouncer.finishedLoadingNode(nodeUid);
+        await waitPromise;
+    });
+
+    it('should timeout', async () => {
+        const nodeUid = 'test-node-1';
+        debouncer.loadingNode(nodeUid);
+
+        jest.advanceTimersByTime(6000);
+        expect(mockTelemetry.mockLogger.warn).toHaveBeenCalledWith(`Timeout for: ${nodeUid}`);
+        await expect(debouncer.waitForLoadingNode(nodeUid)).resolves.toBeUndefined();
+    });
+
+    describe('finishedLoadingNode', () => {
+        it('should handle non-existent node gracefully', async () => {
+            const nodeUid = 'non-existent-node';
+
+            expect(() => debouncer.finishedLoadingNode(nodeUid)).not.toThrow();
+        });
+
+        it('should remove node from internal map after finishing', async () => {
+            const nodeUid = 'test-node-1';
+            debouncer.loadingNode(nodeUid);
+            debouncer.finishedLoadingNode(nodeUid);
+
+            const waitPromise = debouncer.waitForLoadingNode(nodeUid);
+            await expect(waitPromise).resolves.toBe(undefined);
+        });
+    });
+
+    describe('waitForLoadingNode', () => {
+        it('should return immediately for non-registered node', async () => {
+            const nodeUid = 'non-existent-node';
+
+            const result = await debouncer.waitForLoadingNode(nodeUid);
+            expect(result).toBeUndefined();
+            expect(mockTelemetry.mockLogger.debug).not.toHaveBeenCalled();
+        });
+
+        it('should wait for registered node and log debug message', async () => {
+            const nodeUid = 'test-node-1';
+            debouncer.loadingNode(nodeUid);
+
+            const waitPromise = debouncer.waitForLoadingNode(nodeUid);
+
+            expect(mockTelemetry.mockLogger.debug).toHaveBeenCalledWith(`Wait for: ${nodeUid}`);
+
+            debouncer.finishedLoadingNode(nodeUid);
+            await waitPromise;
+        });
+    });
+});

package/src/internal/nodes/debouncer.ts

@@ -0,0 +1,109 @@
+import { Logger, ProtonDriveTelemetry } from '../../interface';
+
+/**
+ * The timeout for which the node is considered to be loading.
+ * If the node is not loaded after this timeout, it is considered to be
+ * loaded or failed to be loaded, and allowed other places to proceed.
+ *
+ * Decrypting many nodes in parallel can take a lot of time, so we allow
+ * more time for this.
+ */
+const DEBOUNCE_TIMEOUT = 5000;
+
+/**
+ * The timeout for which the node is considered to be waiting for a long time.
+ * After this timeout the metric is sent.
+ */
+const DEBOUNCE_LONG_WAIT_TIMEOUT = 1000;
+
+/**
+ * Helper to avoid loading the same node twice.
+ *
+ * Each place that loads a node should report it is being loaded,
+ * and when it is finished, it should report it is finished.
+ * The finish must be called even if the node fails to be loaded
+ * to clear the promise.
+ *
+ * Each place that loads a node from cache should first wait for
+ * the node to be loaded if that is the case.
+ */
+export class NodesDebouncer {
+    private logger: Logger;
+
+    private promises: Map<
+        string,
+        {
+            promise: Promise<void>;
+            resolve: () => void;
+            timeout: NodeJS.Timeout;
+        }
+    > = new Map();
+
+    constructor(private telemetry: ProtonDriveTelemetry) {
+        this.logger = telemetry.getLogger('nodes-debouncer');
+        this.telemetry = telemetry;
+    }
+
+    loadingNodes(nodeUids: string[]) {
+        for (const nodeUid of nodeUids) {
+            this.loadingNode(nodeUid);
+        }
+    }
+
+    loadingNode(nodeUid: string) {
+        const { promise, resolve } = Promise.withResolvers<void>();
+        if (this.promises.has(nodeUid)) {
+            this.logger.warn(`Loading twice for: ${nodeUid}`);
+            return;
+        }
+
+        const timeout = setTimeout(() => {
+            this.logger.warn(`Timeout for: ${nodeUid}`);
+            this.finishedLoadingNode(nodeUid);
+        }, DEBOUNCE_TIMEOUT);
+        this.promises.set(nodeUid, { promise, resolve, timeout });
+    }
+
+    finishedLoadingNodes(nodeUids: string[]) {
+        for (const nodeUid of nodeUids) {
+            this.finishedLoadingNode(nodeUid);
+        }
+    }
+
+    finishedLoadingNode(nodeUid: string) {
+        const result = this.promises.get(nodeUid);
+        if (!result) {
+            return;
+        }
+
+        clearTimeout(result.timeout);
+        result.resolve();
+        this.promises.delete(nodeUid);
+    }
+
+    async waitForLoadingNode(nodeUid: string) {
+        const result = this.promises.get(nodeUid);
+        if (!result) {
+            return;
+        }
+
+        const metricTimeout = setTimeout(() => {
+            this.telemetry.recordMetric({
+                eventName: 'debounceLongWait',
+            });
+        }, DEBOUNCE_LONG_WAIT_TIMEOUT);
+
+        this.logger.debug(`Wait for: ${nodeUid}`);
+        await result.promise;
+
+        clearTimeout(metricTimeout);
+    }
+
+    clear() {
+        for (const result of this.promises.values()) {
+            clearTimeout(result.timeout);
+            result.resolve();
+        }
+        this.promises.clear();
+    }
+}

package/src/internal/nodes/extendedAttributes.ts

@@ -208,11 +208,11 @@ function parseBlockSizes(
         return undefined;
     }
     if (!Array.isArray(blockSizes)) {
-        logger.warn(`XAttr block sizes "${blockSizes}" is not valid`);
+        logger.warn(`XAttr block sizes "${JSON.stringify(blockSizes)}" is not valid`);
         return undefined;
     }
     if (blockSizes.some((size) => typeof size !== 'number' || size <= 0)) {
-        logger.warn(`XAttr block sizes "${blockSizes}" is not valid`);
+        logger.warn(`XAttr block sizes "${JSON.stringify(blockSizes)}" is not valid`);
         return undefined;
     }
     if (blockSizes.length === 0) {

package/src/internal/nodes/index.ts

@@ -43,14 +43,7 @@ export function initNodesModule(
     const cryptoCache = new NodesCryptoCache(telemetry.getLogger('nodes-cache'), driveCryptoCache);
     const cryptoReporter = new NodesCryptoReporter(telemetry, sharesService);
     const cryptoService = new NodesCryptoService(telemetry, driveCrypto, account, cryptoReporter);
-    const nodesAccess = new NodesAccess(
-        telemetry.getLogger('nodes'),
-        api,
-        cache,
-        cryptoCache,
-        cryptoService,
-        sharesService,
-    );
+    const nodesAccess = new NodesAccess(telemetry, api, cache, cryptoCache, cryptoService, sharesService);
     const nodesEventHandler = new NodesEventsHandler(telemetry.getLogger('nodes-events'), cache);
     const nodesManagement = new NodesManagement(api, cryptoCache, cryptoService, nodesAccess);
     const nodesRevisions = new NodesRevisons(telemetry.getLogger('nodes'), api, cryptoService, nodesAccess);

package/src/internal/nodes/nodesAccess.test.ts

@@ -1,4 +1,4 @@
-import { getMockLogger } from '../../tests/logger';
+import { getMockTelemetry } from '../../tests/telemetry';
 import { PrivateKey } from '../../crypto';
 import { DecryptionError } from '../../errors';
 import { NodeAPIService } from './apiService';
@@ -50,7 +50,7 @@ describe('nodesAccess', () => {
             getSharePrivateKey: jest.fn(),
         };
 
-        access = new NodesAccess(getMockLogger(), apiService, cache, cryptoCache, cryptoService, shareService);
+        access = new NodesAccess(getMockTelemetry(), apiService, cache, cryptoCache, cryptoService, shareService);
     });
 
     describe('getNode', () => {
@@ -352,7 +352,7 @@ describe('nodesAccess', () => {
                 expect(cache.setFolderChildrenLoaded).not.toHaveBeenCalled();
             });
 
-            it.only('should return only filtered nodes from API', async () => {
+            it('should return only filtered nodes from API', async () => {
                 cache.isFolderChildrenLoaded = jest.fn().mockResolvedValue(false);
                 cache.getNode = jest.fn().mockImplementation((uid: string) => {
                     if (uid === parentNode.uid) {
@@ -444,7 +444,7 @@ describe('nodesAccess', () => {
             const node1 = { uid: 'volumeId~node1', isStale: false, treeEventScopeId: 'volumeId' } as DecryptedNode;
             const node2 = { uid: 'volumeId~node2', isStale: false, treeEventScopeId: 'volumeId' } as DecryptedNode;
             const node3 = { uid: 'volumeId~node3', isStale: false, treeEventScopeId: 'volumeId' } as DecryptedNode;
-            const node4 = { uid: 'volume~node4', isStale: false, treeEventScopeId: 'volumeId' } as DecryptedNode;
+            const node4 = { uid: 'volumeId~node4', isStale: false, treeEventScopeId: 'volumeId' } as DecryptedNode;
 
             it('should serve fully from cache', async () => {
                 cache.iterateNodes = jest.fn().mockImplementation(async function* () {
@@ -569,7 +569,11 @@ describe('nodesAccess', () => {
         it('should get share parent keys', async () => {
             shareService.getSharePrivateKey = jest.fn(() => Promise.resolve('shareKey' as any as PrivateKey));
 
-            const result = await access.getParentKeys({ shareId: 'shareId', parentUid: undefined });
+            const result = await access.getParentKeys({
+                uid: 'volumeId~nodeId',
+                shareId: 'shareId',
+                parentUid: undefined,
+            });
             expect(result).toEqual({ key: 'shareKey' });
             expect(cryptoCache.getNodeKeys).not.toHaveBeenCalled();
         });
@@ -577,7 +581,11 @@ describe('nodesAccess', () => {
         it('should get node parent keys', async () => {
             cryptoCache.getNodeKeys = jest.fn(() => Promise.resolve({ key: 'parentKey' } as any as DecryptedNodeKeys));
 
-            const result = await access.getParentKeys({ shareId: undefined, parentUid: 'volumeId~parentNodeid' });
+            const result = await access.getParentKeys({
+                uid: 'volumeId~nodeId',
+                shareId: undefined,
+                parentUid: 'volumeId~parentNodeid',
+            });
             expect(result).toEqual({ key: 'parentKey' });
             expect(shareService.getSharePrivateKey).not.toHaveBeenCalled();
         });
@@ -585,7 +593,11 @@ describe('nodesAccess', () => {
         it('should get node parent keys even if share is set', async () => {
             cryptoCache.getNodeKeys = jest.fn(() => Promise.resolve({ key: 'parentKey' } as any as DecryptedNodeKeys));
 
-            const result = await access.getParentKeys({ shareId: 'shareId', parentUid: 'volume1~parentNodeid' });
+            const result = await access.getParentKeys({
+                uid: 'volume1~nodeId',
+                shareId: 'shareId',
+                parentUid: 'volume1~parentNodeid',
+            });
             expect(result).toEqual({ key: 'parentKey' });
             expect(shareService.getSharePrivateKey).not.toHaveBeenCalled();
         });