@protontech/drive-sdk 0.0.10

package/src/internal/upload/manager.ts

@@ -0,0 +1,320 @@
+import { c } from "ttag";
+
+import { Logger, MemberRole, NodeType, ProtonDriveTelemetry, resultOk, Revision, RevisionState, UploadMetadata } from "../../interface";
+import { ValidationError, NodeAlreadyExistsValidationError } from "../../errors";
+import { ErrorCode } from "../apiService";
+import { DecryptedNode, generateFileExtendedAttributes } from "../nodes";
+import { UploadAPIService } from "./apiService";
+import { UploadCryptoService } from "./cryptoService";
+import { NodeRevisionDraft, NodesService, NodesEvents, NodeCrypto } from "./interface";
+
+/**
+ * UploadManager is responsible for creating and deleting draft nodes
+ * on the server. It handles the creation of draft nodes, including
+ * generating the necessary cryptographic keys and metadata.
+ */
+export class UploadManager {
+    private logger: Logger;
+
+    constructor(
+        telemetry: ProtonDriveTelemetry,
+        private apiService: UploadAPIService,
+        private cryptoService: UploadCryptoService,
+        private nodesService: NodesService,
+        private nodesEvents: NodesEvents,
+    ) {
+        this.logger = telemetry.getLogger('upload');
+        this.apiService = apiService;
+        this.cryptoService = cryptoService;
+        this.nodesService = nodesService;
+    }
+
+    async createDraftNode(parentFolderUid: string, name: string, metadata: UploadMetadata): Promise<NodeRevisionDraft> {
+        const parentKeys = await this.nodesService.getNodeKeys(parentFolderUid);
+        if (!parentKeys.hashKey) {
+            throw new ValidationError(c('Error').t`Creating files in non-folders is not allowed`);
+        }
+
+        const generatedNodeCrypto = await this.cryptoService.generateFileCrypto(
+            parentFolderUid,
+            { key: parentKeys.key, hashKey: parentKeys.hashKey },
+            name,
+        );
+
+        const { nodeUid, nodeRevisionUid } = await this.createDraftOnAPI(
+            parentFolderUid,
+            parentKeys.hashKey,
+            name,
+            metadata,
+            generatedNodeCrypto,
+        );
+
+        return {
+            nodeUid,
+            nodeRevisionUid,
+            nodeKeys: {
+                key: generatedNodeCrypto.nodeKeys.decrypted.key,
+                contentKeyPacketSessionKey: generatedNodeCrypto.contentKey.decrypted.contentKeyPacketSessionKey,
+                signatureAddress: generatedNodeCrypto.signatureAddress,
+            },
+            newNodeInfo: {
+                parentUid: parentFolderUid,
+                name,
+                encryptedName: generatedNodeCrypto.encryptedNode.encryptedName,
+                hash: generatedNodeCrypto.encryptedNode.hash,
+            },
+        };
+    }
+
+    private async createDraftOnAPI(
+        parentFolderUid: string,
+        parentHashKey: Uint8Array,
+        name: string,
+        metadata: UploadMetadata,
+        generatedNodeCrypto: NodeCrypto,
+    ): Promise<{
+        nodeUid: string,
+        nodeRevisionUid: string,
+    }> {
+        try {
+            const result = await this.apiService.createDraft(parentFolderUid, {
+                armoredEncryptedName: generatedNodeCrypto.encryptedNode.encryptedName,
+                hash: generatedNodeCrypto.encryptedNode.hash,
+                mediaType: metadata.mediaType,
+                intendedUploadSize: metadata.expectedSize,
+                armoredNodeKey: generatedNodeCrypto.nodeKeys.encrypted.armoredKey,
+                armoredNodePassphrase: generatedNodeCrypto.nodeKeys.encrypted.armoredPassphrase,
+                armoredNodePassphraseSignature: generatedNodeCrypto.nodeKeys.encrypted.armoredPassphraseSignature,
+                base64ContentKeyPacket: generatedNodeCrypto.contentKey.encrypted.base64ContentKeyPacket,
+                armoredContentKeyPacketSignature: generatedNodeCrypto.contentKey.encrypted.armoredContentKeyPacketSignature,
+                signatureEmail: generatedNodeCrypto.signatureAddress.email,
+                // FIXME: client UID
+            });
+            return result;
+        } catch (error: unknown) {
+            if (error instanceof ValidationError) {
+                if (error.code === ErrorCode.ALREADY_EXISTS) {
+                    this.logger.info(`Node with given name already exists`);
+                    const availableName = await this.findAvailableName(
+                        parentFolderUid,
+                        parentHashKey,
+                        name,
+                        generatedNodeCrypto.encryptedNode.hash,
+                    );
+
+                    // If there is existing draft created by this client,
+                    // automatically delete it and try to create a new one
+                    // with the same name again.
+                    if (availableName.existingDraftNodeUid) {
+                        let deleteFailed = false;
+                        try {
+                            this.logger.warn(`Deleting existing draft node ${availableName.existingDraftNodeUid}`);
+                            await this.apiService.deleteDraft(availableName.existingDraftNodeUid);
+                        } catch (deleteDraftError: unknown) {
+                            // Do not throw, let return the next available name to the client.
+                            deleteFailed = true;
+                            this.logger.error('Failed to delete existing draft node', deleteDraftError);
+                        }
+                        if (!deleteFailed) {
+                            return this.createDraftOnAPI(parentFolderUid, parentHashKey, name, metadata, generatedNodeCrypto);
+                        }
+                    }
+
+                    // If there is existing node, return special error
+                    // that includes the available name the client can use.
+                    throw new NodeAlreadyExistsValidationError(
+                        error.message,
+                        error.code,
+                        availableName.availableName,
+                    );
+                }
+            }
+            throw error;
+        }
+    }
+
+    private async findAvailableName(parentFolderUid: string, parentHashKey: Uint8Array, name: string, nameHash: string): Promise<{
+        availableName: string,
+        existingDraftNodeUid?: string,
+    }> {
+        const [namePart, extension] = splitExtension(name);
+
+        const batchSize = 10;
+        let startIndex = 1;
+        while (true) {
+            const namesToCheck = [];
+            for (let i = startIndex; i < startIndex + batchSize; i++) {
+                namesToCheck.push(joinNameAndExtension(namePart, i, extension));
+            }
+
+            const hashesToCheck = await this.cryptoService.generateNameHashes(parentHashKey, namesToCheck);
+
+            const { pendingHashes, availalbleHashes } = await this.apiService.checkAvailableHashes(
+                parentFolderUid,
+                [
+                    ...hashesToCheck.map(({ hash }) => hash),
+                    // Adding the current name hash to get the existing draft
+                    // node UID if it exists.
+                    ...startIndex ? [nameHash] : [],
+                ],
+            );
+
+            if (!availalbleHashes.length) {
+                startIndex += batchSize;
+                continue;
+            }
+
+            const availableHash = hashesToCheck.find(({ hash }) => hash === availalbleHashes[0]);
+            if (!availableHash) {
+                throw Error('Backend returned unexpected hash');
+            }
+
+            // FIXME: use client UID to ensure its own pending draft
+            const ownPendingHash = pendingHashes.find(({ hash }) => hash === nameHash);
+            return {
+                availableName: availableHash.name,
+                existingDraftNodeUid: ownPendingHash?.nodeUid,
+            }
+        }
+    }
+
+    async deleteDraftNode(nodeUid: string): Promise<void> {
+        try {
+            await this.apiService.deleteDraft(nodeUid);
+        } catch (error: unknown) {
+            // Only log the error but do not fail the operation as we are
+            // deleting draft only when somethign fails and original error
+            // will bubble up.
+            this.logger.error('Failed to delete draft node', error);
+        }
+    }
+
+    async createDraftRevision(nodeUid: string, metadata: UploadMetadata): Promise<NodeRevisionDraft> {
+        const node = await this.nodesService.getNode(nodeUid);
+        const nodeKeys = await this.nodesService.getNodeKeys(nodeUid);
+
+        if (!node.activeRevision?.ok || !nodeKeys.contentKeyPacketSessionKey) {
+            throw new ValidationError(c('Error').t`Creating revisions in non-files is not allowed`);
+        }
+
+        const signatureAddress = await this.nodesService.getRootNodeEmailKey(nodeUid);
+
+        const { nodeRevisionUid } = await this.apiService.createDraftRevision(nodeUid, {
+            currentRevisionUid: node.activeRevision.value.uid,
+            intendedUploadSize: metadata.expectedSize,
+        });
+
+        return {
+            nodeUid,
+            nodeRevisionUid,
+            nodeKeys: {
+                key: nodeKeys.key,
+                contentKeyPacketSessionKey: nodeKeys.contentKeyPacketSessionKey,
+                signatureAddress: signatureAddress,
+            },
+        }
+    }
+
+    async deleteDraftRevision(nodeRevisionUid: string): Promise<void> {
+        try {
+            await this.apiService.deleteDraftRevision(nodeRevisionUid);
+        } catch (error: unknown) {
+            // Only log the error but do not fail the operation as we are
+            // deleting draft only when somethign fails and original error
+            // will bubble up.
+            this.logger.error('Failed to delete draft node revision', error);
+        }
+    }
+
+    async commitDraft(
+        nodeRevisionDraft: NodeRevisionDraft,
+        manifest: Uint8Array,
+        metadata: UploadMetadata,
+        extendedAttributes: {
+            modificationTime?: Date,
+            size?: number,
+            blockSizes?: number[],
+            digests?: {
+                sha1?: string,
+            },
+        },
+        encryptedSize: number,
+    ): Promise<void> {
+        const generatedExtendedAttributes = generateFileExtendedAttributes(extendedAttributes);
+        const nodeCommitCrypto = await this.cryptoService.commitFile(nodeRevisionDraft.nodeKeys, manifest, generatedExtendedAttributes);
+        await this.apiService.commitDraftRevision(nodeRevisionDraft.nodeRevisionUid, nodeCommitCrypto);
+
+        const activeRevision = resultOk<Revision, Error>({
+            uid: nodeRevisionDraft.nodeRevisionUid,
+            state: RevisionState.Active,
+            creationTime: new Date(),
+            storageSize: encryptedSize,
+            contentAuthor: resultOk(nodeCommitCrypto.signatureEmail),
+            claimedSize: metadata.expectedSize,
+            claimedModificationTime: extendedAttributes.modificationTime,
+            claimedDigests: {
+                sha1: extendedAttributes.digests?.sha1,
+            },
+        });
+        if (nodeRevisionDraft.newNodeInfo) {
+            const node: DecryptedNode = {
+                // Internal metadata
+                hash: nodeRevisionDraft.newNodeInfo.hash,
+                encryptedName: nodeRevisionDraft.newNodeInfo.encryptedName,
+    
+                // Basic node metadata
+                uid: nodeRevisionDraft.nodeUid,
+                parentUid: nodeRevisionDraft.newNodeInfo.parentUid,
+                type: NodeType.File,
+                mediaType: metadata.mediaType,
+                creationTime: new Date(),
+                totalStorageSize: encryptedSize,
+    
+                // Share node metadata
+                isShared: false,
+                directMemberRole: MemberRole.Inherited,
+    
+                // Decrypted metadata
+                isStale: false,
+                keyAuthor: resultOk(nodeRevisionDraft.nodeKeys.signatureAddress.email),
+                nameAuthor: resultOk(nodeRevisionDraft.nodeKeys.signatureAddress.email),
+                name: resultOk(nodeRevisionDraft.newNodeInfo.name),
+
+                activeRevision,
+            }
+            await this.nodesEvents.nodeCreated(node);
+        } else {
+            await this.nodesEvents.nodeUpdated({
+                uid: nodeRevisionDraft.nodeUid,
+                activeRevision,
+            });
+        }
+    }
+}
+
+/**
+ * Split a filename into `[name, extension]`
+ */
+function splitExtension(filename = ''): [string, string] {
+    const endIdx = filename.lastIndexOf('.');
+    if (endIdx === -1 || endIdx === filename.length-1) {
+        return [filename, ''];
+    }
+    return [filename.slice(0, endIdx), filename.slice(endIdx + 1)];
+};
+
+/**
+ * Join a filename into `name (index).extension`
+ */
+function joinNameAndExtension(name: string, index: number, extension: string): string {
+    if (!name && !extension) {
+        return `(${index})`;
+    }
+    if (!name) {
+        return `(${index}).${extension}`;
+    }
+    if (!extension) {
+        return `${name} (${index})`;
+    }
+    return `${name} (${index}).${extension}`;
+}

package/src/internal/upload/queue.ts

@@ -0,0 +1,31 @@
+import { waitForCondition } from '../wait';
+
+/**
+ * A queue that limits the number of concurrent uploads.
+ * 
+ * This is used to limit the number of concurrent uploads to avoid
+ * overloading the server, or get rate limited.
+ * 
+ * Each file upload consumes memory and is limited by the number of
+ * concurrent block uploads for each file.
+ * 
+ * This queue is straitforward and does not have any priority mechanism
+ * or other features, such as limiting total number of blocks being
+ * uploaded. That is something we want to add in the future to be
+ * more performant for many small file uploads.
+ */
+const MAX_CONCURRENT_UPLOADS = 5;
+
+export class UploadQueue {
+    private capacity = 0;
+
+    // TODO: use expected size to control the size of the queue
+    async waitForCapacity(signal?: AbortSignal) {
+        await waitForCondition(() => this.capacity < MAX_CONCURRENT_UPLOADS, signal);
+        this.capacity++;
+    }
+
+    releaseCapacity() {
+        this.capacity--;
+    }
+}

package/src/internal/upload/telemetry.test.ts

@@ -0,0 +1,129 @@
+import { RateLimitedError, ValidationError, IntegrityError } from '../../errors';
+import { ProtonDriveTelemetry } from '../../interface';
+import { APIHTTPError } from '../apiService';
+import { SharesService } from './interface';
+import { UploadTelemetry } from './telemetry';
+
+describe('UploadTelemetry', () => {
+    let mockTelemetry: jest.Mocked<ProtonDriveTelemetry>;
+    let sharesService: jest.Mocked<SharesService>;
+    let uploadTelemetry: UploadTelemetry;
+
+    const parentNodeUid = 'volumeId~parentNodeId';
+    const revisionUid = 'volumeId~nodeId~revisionId';
+
+    beforeEach(() => {
+        mockTelemetry = {
+            logEvent: jest.fn(),
+            getLogger: jest.fn().mockReturnValue({
+                info: jest.fn(),
+                warn: jest.fn(),
+                error: jest.fn(),
+                debug: jest.fn(),
+            }),
+        } as unknown as jest.Mocked<ProtonDriveTelemetry>;
+
+        sharesService = {
+            getVolumeMetricContext: jest.fn().mockResolvedValue('own_volume'),
+        }
+
+        uploadTelemetry = new UploadTelemetry(mockTelemetry, sharesService);
+    });
+
+    it('should log failure during init (excludes uploaded size)', async () => {
+        await uploadTelemetry.uploadInitFailed(parentNodeUid, new Error('Failed'), 1000);
+
+        expect(mockTelemetry.logEvent).toHaveBeenCalledWith({
+            eventName: "upload",
+            context: "own_volume",
+            uploadedSize: 0,
+            expectedSize: 1000,
+            error: "unknown",
+        });
+    });
+
+    it('should log failure upload', async () => {
+        await uploadTelemetry.uploadFailed(revisionUid, new Error('Failed'), 500, 1000);
+
+        expect(mockTelemetry.logEvent).toHaveBeenCalledWith({
+            eventName: "upload",
+            context: "own_volume",
+            uploadedSize: 500,
+            expectedSize: 1000,
+            error: "unknown",
+        });
+    });
+
+    it('should log successful upload (excludes error)', async () => {
+        await uploadTelemetry.uploadFinished(revisionUid, 1000);
+
+        expect(mockTelemetry.logEvent).toHaveBeenCalledWith({
+            eventName: "upload",
+            context: "own_volume",
+            uploadedSize: 1000,
+            expectedSize: 1000,
+        });
+    });
+
+    describe('detect error category', () => {
+        const verifyErrorCategory = (error: string) => {
+            expect(mockTelemetry.logEvent).toHaveBeenCalledWith(
+                expect.objectContaining({
+                    error,
+                })
+            );
+        };
+
+        it('should ignore ValidationError', async () => {
+            const error = new ValidationError('Validation error');
+            await uploadTelemetry.uploadFailed(revisionUid, error, 500, 1000);
+            expect(mockTelemetry.logEvent).not.toHaveBeenCalled();
+        });
+
+        it('should ignore AbortError', async () => {
+            const error = new Error('Aborted');
+            error.name = 'AbortError';
+            await uploadTelemetry.uploadFailed(revisionUid, error, 500, 1000);
+
+            expect(mockTelemetry.logEvent).not.toHaveBeenCalled();
+        });
+
+        it('should detect "rate_limited" error for RateLimitedError', async () => {
+            const error = new RateLimitedError('Rate limited');
+            await uploadTelemetry.uploadFailed(revisionUid, error, 500, 1000);
+            verifyErrorCategory('rate_limited');
+        });
+
+        it('should detect "integrity_error" for IntegrityError', async () => {
+            const error = new IntegrityError('Integrity check failed');
+            await uploadTelemetry.uploadFailed(revisionUid, error, 500, 1000);
+            verifyErrorCategory('integrity_error');
+        });
+
+        it('should detect "4xx" error for APIHTTPError with 4xx status code', async () => {
+            const error = new APIHTTPError('Client error', 404);
+            await uploadTelemetry.uploadFailed(revisionUid, error, 500, 1000);
+            verifyErrorCategory('4xx');
+        });
+
+        it('should detect "5xx" error for APIHTTPError with 5xx status code', async () => {
+            const error = new APIHTTPError('Server error', 500);
+            await uploadTelemetry.uploadFailed(revisionUid, error, 500, 1000);
+            verifyErrorCategory('5xx');
+        });
+
+        it('should detect "server_error" for TimeoutError', async () => {
+            const error = new Error('Timeout');
+            error.name = 'TimeoutError';
+            await uploadTelemetry.uploadFailed(revisionUid, error, 500, 1000);
+            verifyErrorCategory('server_error');
+        });
+
+        it('should detect "network_error" for NetworkError', async () => {
+            const error = new Error('Network error');
+            error.name = 'NetworkError';
+            await uploadTelemetry.uploadFailed(revisionUid, error, 500, 1000);
+            verifyErrorCategory('network_error');
+        });
+    });
+});

package/src/internal/upload/telemetry.ts

@@ -0,0 +1,121 @@
+import { RateLimitedError, ValidationError, IntegrityError } from "../../errors";
+import { ProtonDriveTelemetry, MetricsUploadErrorType, Logger } from "../../interface";
+import { LoggerWithPrefix } from "../../telemetry";
+import { APIHTTPError } from '../apiService';
+import { splitNodeUid, splitNodeRevisionUid } from "../uids";
+import { SharesService } from "./interface";
+
+export class UploadTelemetry {
+    private logger: Logger;
+
+    constructor(private telemetry: ProtonDriveTelemetry, private sharesService: SharesService) {
+        this.telemetry = telemetry;
+        this.logger = this.telemetry.getLogger("download");
+        this.sharesService = sharesService;
+    }
+
+    getLoggerForRevision(revisionUid: string) {
+        const logger = this.telemetry.getLogger("upload");
+        return new LoggerWithPrefix(logger, `revision ${revisionUid}`);
+    }
+
+    logBlockVerificationError(retryHelped: boolean) {
+        this.telemetry.logEvent({
+            eventName: 'blockVerificationError',
+            retryHelped,
+        });
+    }
+
+    async uploadInitFailed(parentFolderUid: string, error: unknown, expectedSize: number) {
+        const { volumeId } = splitNodeUid(parentFolderUid);
+        const errorCategory = getErrorCategory(error);
+
+        // No error category means ignored error from telemetry.
+        // For example, aborted request.
+        if (!errorCategory) {
+            return;
+        }
+
+        await this.sendTelemetry(volumeId, {
+            uploadedSize: 0,
+            expectedSize,
+            error: errorCategory,
+        });
+    }
+
+    async uploadFailed(revisionUid: string, error: unknown, uploadedSize: number, expectedSize: number) {
+        const { volumeId } = splitNodeRevisionUid(revisionUid);
+        const errorCategory = getErrorCategory(error);
+
+        // No error category means ignored error from telemetry.
+        // For example, aborted request.
+        if (!errorCategory) {
+            return;
+        }
+
+        await this.sendTelemetry(volumeId, {
+            uploadedSize,
+            expectedSize,
+            error: errorCategory,
+        });
+    }
+
+    async uploadFinished(revisionUid: string, uploadedSize: number) {
+        const { volumeId } = splitNodeRevisionUid(revisionUid);
+        await this.sendTelemetry(volumeId, {
+            uploadedSize,
+            expectedSize: uploadedSize,
+        });
+    }
+
+    private async sendTelemetry(volumeId: string, options: {
+        uploadedSize: number,
+        expectedSize: number,
+        error?: MetricsUploadErrorType,
+    }) {
+        let context;
+        try {
+            context = await this.sharesService.getVolumeMetricContext(volumeId);
+        } catch (error: unknown) {
+            this.logger.error('Failed to get metric context', error);
+        }
+
+        this.telemetry.logEvent({
+            eventName: 'upload',
+            context,
+            ...options,
+        });
+    }
+}
+
+function getErrorCategory(error: unknown): MetricsUploadErrorType | undefined {
+    if (error instanceof ValidationError) {
+        return undefined;
+    }
+    if (error instanceof RateLimitedError) {
+        return 'rate_limited';
+    }
+    if (error instanceof IntegrityError) {
+        return 'integrity_error';
+    }
+    if (error instanceof APIHTTPError) {
+        if (error.statusCode >= 400 && error.statusCode < 500) {
+            return '4xx';
+        }
+        if (error.statusCode >= 500) {
+            return '5xx';
+        }
+    }
+    if (error instanceof Error) {
+        if (error.name === 'TimeoutError') {
+            return 'server_error';
+        }
+        if (error.name === 'OfflineError' || error.name === 'NetworkError' || error.message?.toLowerCase() === 'network error') {
+            return 'network_error';
+        }
+        if (error.name === 'AbortError') {
+            return undefined;
+        }
+    }
+    return 'unknown';
+}

package/src/internal/utils.ts

@@ -0,0 +1,9 @@
+export function mergeUint8Arrays(arrays: Uint8Array[]) {
+    const length = arrays.reduce((sum, arr) => sum + arr.length, 0);
+    const chunksAll = new Uint8Array(length);
+    arrays.reduce((position, arr) => {
+        chunksAll.set(arr, position);
+        return position + arr.length;
+    }, 0);
+    return chunksAll;
+}

package/src/internal/wait.test.ts

@@ -0,0 +1,21 @@
+import { waitForCondition } from "./wait";
+
+describe('waitForCondition', () => {
+    it('should resolve immediately if condition is met', async () => {
+        const callback = jest.fn().mockReturnValue(true);
+        await waitForCondition(callback);
+        expect(callback).toHaveBeenCalled();
+    });
+
+    it('should resolve after condition is met', async () => {
+        const callback = jest.fn().mockReturnValueOnce(false).mockReturnValueOnce(true);
+        await waitForCondition(callback);
+        expect(callback).toHaveBeenCalledTimes(2);
+    });
+
+    it('should reject if signal is aborted', async () => {
+        const signal = { aborted: true } as any as AbortSignal;
+        const callback = jest.fn().mockReturnValue(false);
+        await expect(waitForCondition(callback, signal)).rejects.toThrow('aborted');
+    });
+});

package/src/internal/wait.ts

@@ -0,0 +1,26 @@
+import { AbortError } from "../errors";
+
+const WAIT_TIME = 50;
+
+export function waitForCondition(callback: () => boolean, signal?: AbortSignal) {
+    return new Promise<void>((resolve, reject) => {
+        const waitForCondition = () => {
+            if (signal?.aborted) {
+                return reject(new AbortError());
+            }
+            if (callback()) {
+                return resolve();
+            }
+            setTimeout(waitForCondition, WAIT_TIME);
+        };
+        waitForCondition();
+    });
+}
+
+export async function waitSeconds(seconds: number){
+    return wait(seconds * 1000);
+}
+
+export async function wait(miliseconds: number){
+    return new Promise<void>((resolve) => setTimeout(resolve, miliseconds));
+}