@protontech/drive-sdk 0.5.1 → 0.6.1

package/src/diagnostic/{sdkDiagnostic.ts → sdkDiagnosticBase.ts}

@@ -1,48 +1,129 @@
-import { Author, FileDownloader, MaybeNode, NodeType, Revision, ThumbnailType } from '../interface';
-import { ProtonDriveClient } from '../protonDriveClient';
-import { DiagnosticOptions, DiagnosticResult, NodeDetails, ExcpectedTreeNode } from './interface';
+import { Author, FileDownloader, MaybeNode, NodeOrUid, NodeType, ThumbnailType, ThumbnailResult } from '../interface';
+import {
+    DiagnosticOptions,
+    DiagnosticResult,
+    ExpectedTreeNode,
+    DiagnosticProgressCallback,
+} from './interface';
 import { IntegrityVerificationStream } from './integrityVerificationStream';
+import {
+    getNodeType,
+    getNodeDetails,
+    getActiveRevision,
+    getMediaType,
+    getExpectedTreeNodeDetails,
+    getNodeName,
+} from './nodeUtils';
+
+const PROGRESS_REPORT_INTERVAL = 500;
+
+interface SDKClient {
+    getFileDownloader(nodeOrUid: NodeOrUid): Promise<FileDownloader>;
+    iterateThumbnails(nodeUids: string[], thumbnailType: ThumbnailType): AsyncGenerator<ThumbnailResult>;
+}
 
 /**
- * Diagnostic tool that uses SDK to traverse the node tree and verify
- * the integrity of the node tree.
- *
- * It produces only events that can be read by direct SDK invocation.
- * To get the full diagnostic, use {@link FullSDKDiagnostic}.
+ * Base class for all SDK diagnostic tools that verifies the integrity of
+ * the individual nodes.
  */
-export class SDKDiagnostic {
-    constructor(private protonDriveClient: ProtonDriveClient) {
-        this.protonDriveClient = protonDriveClient;
+export class SDKDiagnosticBase {
+    private options: Pick<DiagnosticOptions, 'verifyContent' | 'verifyThumbnails'>;
+
+    private onProgress?: DiagnosticProgressCallback;
+    private progressReportInterval: NodeJS.Timeout | undefined;
+
+    protected nodesQueue: { node: MaybeNode; expected?: ExpectedTreeNode }[] = [];
+    protected allNodesLoaded: boolean = false;
+    protected loadedNodes: number = 0;
+    protected checkedNodes: number = 0;
+
+    constructor(
+        private sdkClient: SDKClient,
+        options?: Pick<DiagnosticOptions, 'verifyContent' | 'verifyThumbnails'>,
+        onProgress?: DiagnosticProgressCallback,
+    ) {
+        this.sdkClient = sdkClient;
+        this.options = options || { verifyContent: false, verifyThumbnails: false };
+        this.onProgress = onProgress;
     }
 
-    async *verifyMyFiles(options?: DiagnosticOptions): AsyncGenerator<DiagnosticResult> {
-        let myFilesRootFolder: MaybeNode;
+    protected startProgress(): void {
+        this.allNodesLoaded = false;
+        this.loadedNodes = 0;
+        this.checkedNodes = 0;
 
-        try {
-            myFilesRootFolder = await this.protonDriveClient.getMyFilesRootFolder();
-        } catch (error: unknown) {
-            yield {
-                type: 'fatal_error',
-                message: `Error getting my files root folder`,
-                error,
-            };
-            return;
+        this.reportProgress();
+        this.progressReportInterval = setInterval(() => {
+            this.reportProgress();
+        }, PROGRESS_REPORT_INTERVAL);
+    }
+
+    protected finishProgress(): void {
+        if (this.progressReportInterval) {
+            clearInterval(this.progressReportInterval);
+            this.progressReportInterval = undefined;
         }
 
-        yield* this.verifyNodeTree(myFilesRootFolder, options);
+        this.reportProgress();
+    }
+
+    private reportProgress(): void {
+        this.onProgress?.({
+            allNodesLoaded: this.allNodesLoaded,
+            loadedNodes: this.loadedNodes,
+            checkedNodes: this.checkedNodes,
+        });
     }
 
-    async *verifyNodeTree(node: MaybeNode, options?: DiagnosticOptions): AsyncGenerator<DiagnosticResult> {
-        const isFolder = getNodeType(node) === NodeType.Folder;
+    protected async *verifyExpectedNodeChildren(
+        parentNodeUid: string,
+        children: MaybeNode[],
+        expectedStructure?: ExpectedTreeNode,
+    ): AsyncGenerator<DiagnosticResult> {
+        if (!expectedStructure) {
+            return;
+        }
+
+        const expectedNodes = expectedStructure.children ?? [];
+        const actualNodeNames = children.map((child) => getNodeName(child));
+
+        for (const expectedNode of expectedNodes) {
+            if (!actualNodeNames.includes(expectedNode.name)) {
+                yield {
+                    type: 'expected_structure_missing_node',
+                    expectedNode: getExpectedTreeNodeDetails(expectedNode),
+                    parentNodeUid,
+                };
+            }
+        }
 
-        yield* this.verifyNode(node, options);
+        for (const child of children) {
+            const childName = getNodeName(child);
+            const isExpected = expectedNodes.some((expectedNode) => expectedNode.name === childName);
 
-        if (isFolder) {
-            yield* this.verifyNodeChildren(node, options);
+            if (!isExpected) {
+                yield {
+                    type: 'expected_structure_unexpected_node',
+                    ...getNodeDetails(child),
+                };
+            }
         }
     }
 
-    private async *verifyNode(node: MaybeNode, options?: DiagnosticOptions): AsyncGenerator<DiagnosticResult> {
+    protected async *verifyNodesQueue(): AsyncGenerator<DiagnosticResult> {
+        while (this.nodesQueue.length > 0 || !this.allNodesLoaded) {
+            const result = this.nodesQueue.shift();
+            if (result) {
+                yield* this.verifyNode(result.node, result.expected);
+                this.checkedNodes++;
+            } else {
+                // Wait for 100ms before checking again.
+                await new Promise((resolve) => setTimeout(resolve, 100));
+            }
+        }
+    }
+
+    private async *verifyNode(node: MaybeNode, expectedStructure?: ExpectedTreeNode): AsyncGenerator<DiagnosticResult> {
         if (!node.ok) {
             yield {
                 type: 'degraded_node',
@@ -50,25 +131,48 @@ export class SDKDiagnostic {
             };
         }
 
-        yield* this.verifyAuthor(node.ok ? node.value.keyAuthor : node.error.keyAuthor, 'key', node);
-        yield* this.verifyAuthor(node.ok ? node.value.nameAuthor : node.error.nameAuthor, 'name', node);
+        yield* this.verifyAuthor(node.ok ? node.value.keyAuthor : node.error.keyAuthor, 'key', node, expectedStructure);
+        yield* this.verifyAuthor(
+            node.ok ? node.value.nameAuthor : node.error.nameAuthor,
+            'name',
+            node,
+            expectedStructure,
+        );
 
         const activeRevision = getActiveRevision(node);
         if (activeRevision) {
-            yield* this.verifyAuthor(activeRevision.contentAuthor, 'content', node);
+            yield* this.verifyAuthor(activeRevision.contentAuthor, 'content', node, expectedStructure);
         }
 
-        yield* this.verifyFileExtendedAttributes(node, options);
+        yield* this.verifyFileExtendedAttributes(node, expectedStructure);
 
-        if (options?.verifyContent) {
+        if (this.options.verifyContent === 'peakOnly') {
+            yield* this.verifyContentPeak(node);
+        } else if (this.options.verifyContent) {
             yield* this.verifyContent(node);
         }
-        if (options?.verifyThumbnails) {
+        if (this.options.verifyThumbnails) {
             yield* this.verifyThumbnails(node);
         }
+
+        if (expectedStructure?.expectedMediaType) {
+            const mediaType = getMediaType(node);
+            if (mediaType !== expectedStructure.expectedMediaType) {
+                yield {
+                    type: 'expected_structure_integrity_error',
+                    expectedNode: getExpectedTreeNodeDetails(expectedStructure),
+                    ...getNodeDetails(node),
+                };
+            }
+        }
     }
 
-    private async *verifyAuthor(author: Author, authorType: string, node: MaybeNode): AsyncGenerator<DiagnosticResult> {
+    private async *verifyAuthor(
+        author: Author,
+        authorType: 'key' | 'name' | 'content',
+        node: MaybeNode,
+        expectedStructure?: ExpectedTreeNode,
+    ): AsyncGenerator<DiagnosticResult> {
         if (!author.ok) {
             yield {
                 type: 'unverified_author',
@@ -78,11 +182,31 @@ export class SDKDiagnostic {
                 ...getNodeDetails(node),
             };
         }
+
+        if (expectedStructure?.expectedAuthors) {
+            let expectedEmail: string | null | undefined =
+                typeof expectedStructure.expectedAuthors === 'string'
+                    ? expectedStructure.expectedAuthors
+                    : expectedStructure.expectedAuthors[authorType];
+
+            if (expectedEmail === 'anonymous') {
+                expectedEmail = null;
+            }
+
+            const email = author.ok ? author.value : author.error.claimedAuthor;
+            if (expectedEmail !== undefined && email !== expectedEmail) {
+                yield {
+                    type: 'expected_structure_integrity_error',
+                    expectedNode: getExpectedTreeNodeDetails(expectedStructure),
+                    ...getNodeDetails(node),
+                };
+            }
+        }
     }
 
     private async *verifyFileExtendedAttributes(
         node: MaybeNode,
-        options?: DiagnosticOptions,
+        expectedStructure?: ExpectedTreeNode,
     ): AsyncGenerator<DiagnosticResult> {
         const activeRevision = getActiveRevision(node);
 
@@ -108,9 +232,9 @@ export class SDKDiagnostic {
             };
         }
 
-        if (options?.expectedStructure) {
-            const expectedSha1 = options.expectedStructure.expectedSha1;
-            const expectedSizeInBytes = options.expectedStructure.expectedSizeInBytes;
+        if (expectedStructure) {
+            const expectedSha1 = expectedStructure.expectedSha1;
+            const expectedSizeInBytes = expectedStructure.expectedSizeInBytes;
 
             const wrongSha1 = expectedSha1 !== undefined && claimedSha1 !== expectedSha1;
             const wrongSizeInBytes = expectedSizeInBytes !== undefined && claimedSizeInBytes !== expectedSizeInBytes;
@@ -120,13 +244,49 @@ export class SDKDiagnostic {
                     type: 'expected_structure_integrity_error',
                     claimedSha1,
                     claimedSizeInBytes,
-                    expectedNode: getExpectedTreeNodeDetails(options.expectedStructure),
+                    expectedNode: getExpectedTreeNodeDetails(expectedStructure),
                     ...getNodeDetails(node),
                 };
             }
         }
     }
 
+    private async *verifyContentPeak(node: MaybeNode): AsyncGenerator<DiagnosticResult> {
+        if (getNodeType(node) !== NodeType.File) {
+            return;
+        }
+
+        let downloader: FileDownloader;
+        try {
+            downloader = await this.sdkClient.getFileDownloader(node);
+        } catch (error: unknown) {
+            yield {
+                type: 'sdk_error',
+                call: `getFileDownloader(${node.ok ? node.value.uid : node.error.uid})`,
+                error,
+            };
+            return;
+        }
+
+        try {
+            const stream = downloader.getSeekableStream();
+            const peak = await stream.read(1024);
+            if (peak.value.length === 0) {
+                yield {
+                    type: 'content_download_error',
+                    error: new Error('No data read'),
+                    ...getNodeDetails(node),
+                };
+            }
+        } catch (error: unknown) {
+            yield {
+                type: 'content_download_error',
+                error,
+                ...getNodeDetails(node),
+            };
+        }
+    }
+
     private async *verifyContent(node: MaybeNode): AsyncGenerator<DiagnosticResult> {
         if (getNodeType(node) !== NodeType.File) {
             return;
@@ -142,11 +302,11 @@ export class SDKDiagnostic {
 
         let downloader: FileDownloader;
         try {
-            downloader = await this.protonDriveClient.getFileRevisionDownloader(activeRevision.uid);
+            downloader = await this.sdkClient.getFileDownloader(node);
         } catch (error: unknown) {
             yield {
                 type: 'sdk_error',
-                call: `getFileRevisionDownloader(${activeRevision.uid})`,
+                call: `getFileDownloader(${node.ok ? node.value.uid : node.error.uid})`,
                 error,
             };
             return;
@@ -190,9 +350,7 @@ export class SDKDiagnostic {
         const nodeUid = node.ok ? node.value.uid : node.error.uid;
 
         try {
-            const result = await Array.fromAsync(
-                this.protonDriveClient.iterateThumbnails([nodeUid], ThumbnailType.Type1),
-            );
+            const result = await Array.fromAsync(this.sdkClient.iterateThumbnails([nodeUid], ThumbnailType.Type1));
 
             if (result.length === 0) {
                 yield {
@@ -217,162 +375,4 @@ export class SDKDiagnostic {
             };
         }
     }
-
-    private async *verifyNodeChildren(
-        parentNode: MaybeNode,
-        options?: DiagnosticOptions,
-    ): AsyncGenerator<DiagnosticResult> {
-        const parentNodeUid = parentNode.ok ? parentNode.value.uid : parentNode.error.uid;
-        const children: MaybeNode[] = [];
-
-        try {
-            for await (const child of this.protonDriveClient.iterateFolderChildren(parentNode)) {
-                if (options?.expectedStructure) {
-                    children.push(child);
-                }
-
-                yield *
-                    this.verifyNodeTree(child, {
-                        ...options,
-                        expectedStructure: options?.expectedStructure
-                            ? getTreeNodeChildByNodeName(options.expectedStructure, getNodeName(child))
-                            : undefined,
-                    });
-            }
-        } catch (error: unknown) {
-            yield {
-                type: 'sdk_error',
-                call: `iterateFolderChildren(${parentNodeUid})`,
-                error,
-            };
-        }
-
-        if (options?.expectedStructure) {
-            yield* this.verifyExpectedNodeChildren(parentNodeUid, children, options);
-        }
-    }
-
-    private async *verifyExpectedNodeChildren(
-        parentNodeUid: string,
-        children: MaybeNode[],
-        options: DiagnosticOptions,
-    ): AsyncGenerator<DiagnosticResult> {
-        if (!options.expectedStructure) {
-            return;
-        }
-
-        const expectedNodes = options.expectedStructure.children ?? [];
-        const actualNodeNames = children.map((child) => getNodeName(child));
-
-        for (const expectedNode of expectedNodes) {
-            if (!actualNodeNames.includes(expectedNode.name)) {
-                yield {
-                    type: 'expected_structure_missing_node',
-                    expectedNode: getExpectedTreeNodeDetails(expectedNode),
-                    parentNodeUid,
-                };
-            }
-        }
-
-        for (const child of children) {
-            const childName = getNodeName(child);
-            const isExpected = expectedNodes.some((expectedNode) => expectedNode.name === childName);
-
-            if (!isExpected) {
-                yield {
-                    type: 'expected_structure_unexpected_node',
-                    ...getNodeDetails(child),
-                };
-            }
-        }
-    }
-}
-
-function getNodeDetails(node: MaybeNode): NodeDetails {
-    const errors: {
-        field: string;
-        error: unknown;
-    }[] = [];
-
-    if (!node.ok) {
-        const degradedNode = node.error;
-        if (!degradedNode.name.ok) {
-            errors.push({
-                field: 'name',
-                error: degradedNode.name.error,
-            });
-        }
-        if (degradedNode.activeRevision?.ok === false) {
-            errors.push({
-                field: 'activeRevision',
-                error: degradedNode.activeRevision.error,
-            });
-        }
-        for (const error of degradedNode.errors ?? []) {
-            if (error instanceof Error) {
-                errors.push({
-                    field: 'error',
-                    error,
-                });
-            }
-        }
-    }
-
-    return {
-        safeNodeDetails: {
-            ...getNodeUids(node),
-            nodeType: getNodeType(node),
-            nodeCreationTime: node.ok ? node.value.creationTime : node.error.creationTime,
-            keyAuthor: node.ok ? node.value.keyAuthor : node.error.keyAuthor,
-            nameAuthor: node.ok ? node.value.nameAuthor : node.error.nameAuthor,
-            errors,
-        },
-        sensitiveNodeDetails: node,
-    };
-}
-
-function getNodeUids(node: MaybeNode): { nodeUid: string; revisionUid?: string } {
-    const activeRevision = getActiveRevision(node);
-    return {
-        nodeUid: node.ok ? node.value.uid : node.error.uid,
-        revisionUid: activeRevision?.uid,
-    };
-}
-
-function getNodeType(node: MaybeNode): NodeType {
-    return node.ok ? node.value.type : node.error.type;
-}
-
-function getActiveRevision(node: MaybeNode): Revision | undefined {
-    if (node.ok) {
-        return node.value.activeRevision;
-    }
-    if (node.error.activeRevision?.ok) {
-        return node.error.activeRevision.value;
-    }
-    return undefined;
-}
-
-function getNodeName(node: MaybeNode): string {
-    if (node.ok) {
-        return node.value.name;
-    }
-    if (node.error.name.ok) {
-        return node.error.name.value;
-    }
-    return 'N/A';
-}
-
-function getExpectedTreeNodeDetails(expectedNode: ExcpectedTreeNode): ExcpectedTreeNode {
-    return {
-        ...expectedNode,
-        children: undefined,
-    };
-}
-
-function getTreeNodeChildByNodeName(
-    expectedSubtree: ExcpectedTreeNode,
-    nodeName: string,
-): ExcpectedTreeNode | undefined {
-    return expectedSubtree.children?.find((expectedNode) => expectedNode.name === nodeName);
 }

package/src/diagnostic/sdkDiagnosticMain.ts

@@ -0,0 +1,95 @@
+import { MaybeNode, NodeType } from '../interface';
+import { ProtonDriveClient } from '../protonDriveClient';
+import { DiagnosticOptions, DiagnosticProgressCallback, DiagnosticResult, ExpectedTreeNode } from './interface';
+import { zipGenerators } from './zipGenerators';
+import { getNodeType, getNodeName, getTreeNodeChildByNodeName } from './nodeUtils';
+import { SDKDiagnosticBase } from './sdkDiagnosticBase';
+
+/**
+ * Diagnostic tool that uses the main Drive SDK to traverse and verify
+ * the integrity of the node tree.
+ */
+export class SDKDiagnosticMain extends SDKDiagnosticBase {
+    constructor(
+        private protonDriveClient: ProtonDriveClient,
+        options?: Pick<DiagnosticOptions, 'verifyContent' | 'verifyThumbnails'>,
+        onProgress?: DiagnosticProgressCallback,
+    ) {
+        super(protonDriveClient, options, onProgress);
+        this.protonDriveClient = protonDriveClient;
+    }
+
+    async *verifyMyFiles(expectedStructure?: ExpectedTreeNode): AsyncGenerator<DiagnosticResult> {
+        let myFilesRootFolder: MaybeNode;
+
+        try {
+            myFilesRootFolder = await this.protonDriveClient.getMyFilesRootFolder();
+        } catch (error: unknown) {
+            yield {
+                type: 'fatal_error',
+                message: `Error getting my files root folder`,
+                error,
+            };
+            return;
+        }
+
+        yield* this.verifyNodeTree(myFilesRootFolder, expectedStructure);
+    }
+
+    async *verifyNodeTree(node: MaybeNode, expectedStructure?: ExpectedTreeNode): AsyncGenerator<DiagnosticResult> {
+        this.startProgress();
+        this.nodesQueue.push({ node, expected: expectedStructure });
+        this.loadedNodes++;
+        yield* zipGenerators(this.loadNodeTree(node, expectedStructure), this.verifyNodesQueue());
+        this.finishProgress();
+    }
+
+    private async *loadNodeTree(
+        parentNode: MaybeNode,
+        expectedStructure?: ExpectedTreeNode,
+    ): AsyncGenerator<DiagnosticResult> {
+        const isFolder = getNodeType(parentNode) === NodeType.Folder;
+        if (isFolder) {
+            yield* this.loadNodeTreeRecursively(parentNode, expectedStructure);
+        }
+        this.allNodesLoaded = true;
+    }
+
+    private async *loadNodeTreeRecursively(
+        parentNode: MaybeNode,
+        expectedStructure?: ExpectedTreeNode,
+    ): AsyncGenerator<DiagnosticResult> {
+        const parentNodeUid = parentNode.ok ? parentNode.value.uid : parentNode.error.uid;
+        const children: MaybeNode[] = [];
+
+        try {
+            for await (const child of this.protonDriveClient.iterateFolderChildren(parentNode)) {
+                children.push(child);
+                this.nodesQueue.push({
+                    node: child,
+                    expected: getTreeNodeChildByNodeName(expectedStructure, getNodeName(child)),
+                });
+                this.loadedNodes++;
+            }
+        } catch (error: unknown) {
+            yield {
+                type: 'sdk_error',
+                call: `iterateFolderChildren(${parentNodeUid})`,
+                error,
+            };
+        }
+
+        if (expectedStructure) {
+            yield* this.verifyExpectedNodeChildren(parentNodeUid, children, expectedStructure);
+        }
+
+        for (const child of children) {
+            if (getNodeType(child) === NodeType.Folder) {
+                yield* this.loadNodeTreeRecursively(
+                    child,
+                    getTreeNodeChildByNodeName(expectedStructure, getNodeName(child)),
+                );
+            }
+        }
+    }
+}

package/src/diagnostic/sdkDiagnosticPhotos.ts

@@ -0,0 +1,70 @@
+import { MaybeNode } from '../interface';
+import { ProtonDrivePhotosClient } from '../protonDrivePhotosClient';
+import { DiagnosticOptions, DiagnosticProgressCallback, DiagnosticResult, ExpectedTreeNode } from './interface';
+import { zipGenerators } from './zipGenerators';
+import { getNodeName, getTreeNodeChildByNodeName } from './nodeUtils';
+import { SDKDiagnosticBase } from './sdkDiagnosticBase';
+
+/**
+ * Diagnostic tool that uses the Photos SDK to traverse and verify
+ * the integrity of the Photos in the timeline.
+ */
+export class SDKDiagnosticPhotos extends SDKDiagnosticBase {
+    constructor(
+        private protonDrivePhotosClient: ProtonDrivePhotosClient,
+        options?: Pick<DiagnosticOptions, 'verifyContent' | 'verifyThumbnails'>,
+        onProgress?: DiagnosticProgressCallback,
+    ) {
+        super(protonDrivePhotosClient, options, onProgress);
+        this.protonDrivePhotosClient = protonDrivePhotosClient;
+    }
+
+    async *verifyTimeline(expectedStructure?: ExpectedTreeNode): AsyncGenerator<DiagnosticResult> {
+        this.startProgress();
+        yield* zipGenerators(this.loadTimeline(expectedStructure), this.verifyNodesQueue());
+        this.finishProgress();
+    }
+
+    private async *loadTimeline(expectedStructure?: ExpectedTreeNode): AsyncGenerator<DiagnosticResult> {
+        let nodeUids: string[] = [];
+        try {
+            const results = await Array.fromAsync(this.protonDrivePhotosClient.iterateTimeline());
+            nodeUids = results.map((result) => result.nodeUid);
+            this.loadedNodes = nodeUids.length;
+        } catch (error: unknown) {
+            yield {
+                type: 'sdk_error',
+                call: `iterateTimeline()`,
+                error,
+            };
+        }
+
+        const photos: MaybeNode[] = [];
+        try {
+            for await (const maybeMissingNode of this.protonDrivePhotosClient.iterateNodes(nodeUids)) {
+                if (!maybeMissingNode.ok && 'missingUid' in maybeMissingNode.error) {
+                    continue;
+                }
+                const maybeNode = maybeMissingNode as MaybeNode;
+
+                photos.push(maybeNode);
+                this.nodesQueue.push({
+                    node: maybeNode,
+                    expected: getTreeNodeChildByNodeName(expectedStructure, getNodeName(maybeNode)),
+                });
+            }
+        } catch (error: unknown) {
+            yield {
+                type: 'sdk_error',
+                call: `iterateNodes(...)`,
+                error,
+            };
+        }
+
+        if (expectedStructure) {
+            yield* this.verifyExpectedNodeChildren('photo-timeline', photos, expectedStructure);
+        }
+
+        this.allNodesLoaded = true;
+    }
+}

package/src/interface/index.ts

@@ -82,7 +82,7 @@ export type {
     MetricEvent,
 } from './telemetry';
 export { MetricVolumeType } from './telemetry';
-export type { FileUploader, FileRevisionUploader, UploadController, UploadMetadata } from './upload';
+export type { FileUploader, UploadController, UploadMetadata } from './upload';
 export type { Thumbnail, ThumbnailResult } from './thumbnail';
 export { ThumbnailType } from './thumbnail';