@protontech/drive-sdk 0.0.13 → 0.1.1

package/src/diagnostic/httpClient.ts

@@ -0,0 +1,84 @@
+import {
+    ProtonDriveHTTPClient,
+    ProtonDriveHTTPClientBlobOptions,
+    ProtonDriveHTTPClientJsonOptions,
+} from '../interface';
+import { EventsGenerator } from './eventsGenerator';
+
+/**
+ * Special HTTP client that is compatible with the SDK.
+ *
+ * It is a probe into SDK to observe whats going on and report any suspicious
+ * behavior.
+ *
+ * It should be used only for diagnostic purposes.
+ */
+export class DiagnosticHTTPClient extends EventsGenerator implements ProtonDriveHTTPClient {
+    constructor(private httpClient: ProtonDriveHTTPClient) {
+        super();
+        this.httpClient = httpClient;
+    }
+
+    async fetchJson(options: ProtonDriveHTTPClientJsonOptions): Promise<Response> {
+        try {
+            const response = await this.httpClient.fetchJson(options);
+
+            if (response.status >= 400 && response.status !== 429) {
+                try {
+                    const json = await response.json();
+
+                    this.enqueueEvent({
+                        type: 'http_error',
+                        request: {
+                            url: options.url,
+                            method: options.method,
+                            json: options.json,
+                        },
+                        response: {
+                            status: response.status,
+                            statusText: response.statusText,
+                            json,
+                        },
+                    });
+
+                    return new Response(JSON.stringify(json), {
+                        status: response.status,
+                        statusText: response.statusText,
+                        headers: response.headers,
+                    });
+                } catch (jsonError: unknown) {
+                    this.enqueueEvent({
+                        type: 'http_error',
+                        request: {
+                            url: options.url,
+                            method: options.method,
+                            json: options.json,
+                        },
+                        response: {
+                            status: response.status,
+                            statusText: response.statusText,
+                            jsonError,
+                        },
+                    });
+                }
+            }
+
+            return response;
+        } catch (error: unknown) {
+            this.enqueueEvent({
+                type: 'http_error',
+                request: {
+                    url: options.url,
+                    method: options.method,
+                    json: options.json,
+                },
+                error,
+            });
+            throw error;
+        }
+    }
+
+    fetchBlob(options: ProtonDriveHTTPClientBlobOptions): Promise<Response> {
+        return this.httpClient.fetchBlob(options);
+    }
+}

package/src/diagnostic/index.ts

@@ -0,0 +1,40 @@
+import { MemoryCache, NullCache } from '../cache';
+import { ProtonDriveClientContructorParameters } from '../interface';
+import { ProtonDriveClient } from '../protonDriveClient';
+import { DiagnosticHTTPClient } from './httpClient';
+import { Diagnostic } from './interface';
+import { SDKDiagnostic } from './sdkDiagnostic';
+import { FullSDKDiagnostic } from './sdkDiagnosticFull';
+import { DiagnosticTelemetry } from './telemetry';
+
+export type { Diagnostic, DiagnosticResult } from './interface';
+
+/**
+ * Initializes the diagnostic tool. It creates the instance of
+ * ProtonDriveClient with the special probes to observe the logs,
+ * metrics and HTTP calls; and enforced null/empty cache to always
+ * start from scratch.
+ */
+export function initDiagnostic(
+    options: Omit<ProtonDriveClientContructorParameters, 'entitiesCache' | 'cryptoCache' | 'telemetry'>,
+): Diagnostic {
+    const httpClient = new DiagnosticHTTPClient(options.httpClient);
+    const telemetry = new DiagnosticTelemetry();
+
+    const protonDriveClient = new ProtonDriveClient({
+        ...options,
+        httpClient,
+        // Ensure we always start with a clean state.
+        // Do not use memory cache as diagnostic should visit each node
+        // only once and we don't want to grow memory usage.
+        entitiesCache: new NullCache(),
+        // However, we need to use memory cache for crypto cache to avoid
+        // re-fetching the same key for all the children.
+        cryptoCache: new MemoryCache(),
+        // Special telemetry that observes the logs and metrics.
+        telemetry,
+    });
+
+    const diagnostic = new SDKDiagnostic(protonDriveClient);
+    return new FullSDKDiagnostic(diagnostic, telemetry, httpClient);
+}

package/src/diagnostic/integrityVerificationStream.ts

@@ -0,0 +1,55 @@
+import { sha1 } from '@noble/hashes/legacy';
+import { bytesToHex } from '@noble/hashes/utils';
+
+/**
+ * A WritableStream that computes SHA1 hash on the fly.
+ * The computed SHA1 hash is available after the stream is closed.
+ */
+export class IntegrityVerificationStream extends WritableStream<Uint8Array> {
+    private sha1Hash = sha1.create();
+    private _computedSha1: string | undefined = undefined;
+    private _computedSizeInBytes: number = 0;
+    private _isClosed = false;
+
+    constructor() {
+        super({
+            start: () => {},
+            write: (chunk: Uint8Array) => {
+                if (this._isClosed) {
+                    throw new Error('Cannot write to a closed stream');
+                }
+                this.sha1Hash.update(chunk);
+                this._computedSizeInBytes += chunk.length;
+            },
+            close: () => {
+                if (!this._isClosed) {
+                    this._computedSha1 = bytesToHex(this.sha1Hash.digest());
+                    this._isClosed = true;
+                }
+            },
+            abort: () => {
+                this._isClosed = true;
+                this._computedSha1 = undefined;
+            },
+        });
+    }
+
+    /**
+     * Get the computed SHA1 hash. Only available after the stream is closed.
+     * @returns The SHA1 hash as a hex string, or null if not yet computed or stream was aborted
+     */
+    get computedSha1(): string | undefined {
+        return this._computedSha1;
+    }
+
+    /**
+     * Get the computed size in bytes. Only available after the stream is closed.
+     * @returns The size in bytes, or 0 if not yet computed or stream was aborted
+     */
+    get computedSizeInBytes(): number | undefined {
+        if (!this._isClosed) {
+            return undefined;
+        }
+        return this._computedSizeInBytes;
+    }
+}

package/src/diagnostic/interface.ts

@@ -0,0 +1,158 @@
+import { DegradedNode, MaybeNode, MetricEvent } from '../interface';
+import { LogRecord } from '../telemetry';
+
+export interface Diagnostic {
+    verifyMyFiles(options?: DiagnosticOptions): AsyncGenerator<DiagnosticResult>;
+    verifyNodeTree(node: MaybeNode, options?: DiagnosticOptions): AsyncGenerator<DiagnosticResult>;
+}
+
+export type DiagnosticOptions = {
+    verifyContent?: boolean;
+    verifyThumbnails?: boolean;
+};
+
+export type DiagnosticResult =
+    | FatalErrorResult
+    | SdkErrorResult
+    | HttpErrorResult
+    | DegradedNodeResult
+    | UnverifiedAuthorResult
+    | ExtendedAttributesErrorResult
+    | ExtendedAttributesMissingFieldResult
+    | ContentFileMissingRevisionResult
+    | ContentIntegrityErrorResult
+    | ContentDownloadErrorResult
+    | ThumbnailsErrorResult
+    | LogErrorResult
+    | LogWarningResult
+    | MetricResult;
+
+// Event representing that fatal error occurred during the diagnostic.
+// This error prevents the diagnostic to finish.
+export type FatalErrorResult = {
+    type: 'fatal_error';
+    message: string;
+    error?: unknown;
+};
+
+// Event representing that SDK call failed.
+// It can be any throwable error from any SDK call. Normally no error should be thrown.
+export type SdkErrorResult = {
+    type: 'sdk_error';
+    call: string;
+    error?: unknown;
+};
+
+// Event representing that HTTP call failed.
+// It can be any call from the SDK, including validation error. Normally no error should be present.
+export type HttpErrorResult = {
+    type: 'http_error';
+    request: {
+        url: string;
+        method: string;
+        json: unknown;
+    };
+    // Error if the whole call failed (`fetch` failed).
+    error?: unknown;
+    // Response if the response is not 2xx or 3xx.
+    response?: {
+        status: number;
+        statusText: string;
+        // Either json object or error if the response is not JSON.
+        json?: object;
+        jsonError?: unknown;
+    };
+};
+
+// Event representing that node has some decryption or other (e.g., invalid name) issues.
+export type DegradedNodeResult = {
+    type: 'degraded_node';
+    nodeUid: string;
+    node: DegradedNode;
+};
+
+// Event representing that signature verification failing.
+export type UnverifiedAuthorResult = {
+    type: 'unverified_author';
+    nodeUid: string;
+    revisionUid?: string;
+    authorType: string;
+    claimedAuthor?: string;
+    error: string;
+    node: MaybeNode;
+};
+
+// Event representing that field from the extended attributes is not valid format.
+// Currently only `sha1` verification is supported.
+export type ExtendedAttributesErrorResult = {
+    type: 'extended_attributes_error';
+    nodeUid: string;
+    revisionUid?: string;
+    field: 'sha1';
+    value: string;
+};
+
+// Event representing that field from the extended attributes is missing.
+// Currently only `sha1` verification is supported.
+export type ExtendedAttributesMissingFieldResult = {
+    type: 'extended_attributes_missing_field';
+    nodeUid: string;
+    revisionUid?: string;
+    missingField: 'sha1';
+};
+
+// Event representing that file is missing the active revision.
+export type ContentFileMissingRevisionResult = {
+    type: 'content_file_missing_revision';
+    nodeUid: string;
+    revisionUid?: string;
+};
+
+// Event representing that file content is not valid - either sha1 or size is not correct.
+export type ContentIntegrityErrorResult = {
+    type: 'content_integrity_error';
+    nodeUid: string;
+    revisionUid?: string;
+    claimedSha1?: string;
+    computedSha1?: string;
+    claimedSizeInBytes?: number;
+    computedSizeInBytes?: number;
+};
+
+// Event representing that downloading the file content failed.
+// This can be connection issue or server error. If its integrity issue,
+// it should be reported as `ContentIntegrityErrorResult`.
+export type ContentDownloadErrorResult = {
+    type: 'content_download_error';
+    nodeUid: string;
+    revisionUid?: string;
+    error: unknown;
+};
+
+// Event representing that getting the thumbnails failed.
+// This can be connection issue or server error.
+export type ThumbnailsErrorResult = {
+    type: 'thumbnails_error';
+    nodeUid: string;
+    revisionUid?: string;
+    message?: string;
+    error?: unknown;
+};
+
+// Event representing errors logged during the diagnostic.
+export type LogErrorResult = {
+    type: 'log_error';
+    log: LogRecord;
+};
+
+// Event representing warnings logged during the diagnostic.
+export type LogWarningResult = {
+    type: 'log_warning';
+    log: LogRecord;
+};
+
+// Event representing metrics logged during the diagnostic.
+export type MetricResult = {
+    type: 'metric';
+    event: MetricEvent;
+};

package/src/diagnostic/sdkDiagnostic.ts

@@ -0,0 +1,249 @@
+import { Author, FileDownloader, MaybeNode, NodeType, Revision, ThumbnailType } from '../interface';
+import { ProtonDriveClient } from '../protonDriveClient';
+import { Diagnostic, DiagnosticOptions, DiagnosticResult } from './interface';
+import { IntegrityVerificationStream } from './integrityVerificationStream';
+
+/**
+ * Diagnostic tool that uses SDK to traverse the node tree and verify
+ * the integrity of the node tree.
+ *
+ * It produces only events that can be read by direct SDK invocation.
+ * To get the full diagnostic, use {@link FullSDKDiagnostic}.
+ */
+export class SDKDiagnostic implements Diagnostic {
+    constructor(private protonDriveClient: ProtonDriveClient) {
+        this.protonDriveClient = protonDriveClient;
+    }
+
+    async *verifyMyFiles(options?: DiagnosticOptions): AsyncGenerator<DiagnosticResult> {
+        let myFilesRootFolder: MaybeNode;
+
+        try {
+            myFilesRootFolder = await this.protonDriveClient.getMyFilesRootFolder();
+        } catch (error: unknown) {
+            yield {
+                type: 'fatal_error',
+                message: `Error getting my files root folder`,
+                error,
+            };
+            return;
+        }
+
+        yield* this.verifyNodeTree(myFilesRootFolder, options);
+    }
+
+    async *verifyNodeTree(node: MaybeNode, options?: DiagnosticOptions): AsyncGenerator<DiagnosticResult> {
+        const isFolder = getNodeType(node) === NodeType.Folder;
+
+        yield* this.verifyNode(node, options);
+
+        if (isFolder) {
+            yield* this.verifyNodeChildren(node, options);
+        }
+    }
+
+    private async *verifyNode(node: MaybeNode, options?: DiagnosticOptions): AsyncGenerator<DiagnosticResult> {
+        const nodeUid = node.ok ? node.value.uid : node.error.uid;
+
+        if (!node.ok) {
+            yield {
+                type: 'degraded_node',
+                nodeUid,
+                node: node.error,
+            };
+        }
+
+        const activeRevision = getActiveRevision(node);
+        const nodeInfo = {
+            ...getNodeUids(node),
+            node,
+        };
+
+        yield* this.verifyAuthor(node.ok ? node.value.keyAuthor : node.error.keyAuthor, {
+            ...nodeInfo,
+            authorType: 'key',
+        });
+        yield* this.verifyAuthor(node.ok ? node.value.nameAuthor : node.error.nameAuthor, {
+            ...nodeInfo,
+            authorType: 'name',
+        });
+        if (activeRevision) {
+            yield* this.verifyAuthor(activeRevision.contentAuthor, { ...nodeInfo, authorType: 'content' });
+        }
+
+        yield* this.verifyFileExtendedAttributes(node);
+
+        if (options?.verifyContent) {
+            yield* this.verifyContent(node);
+        }
+        if (options?.verifyThumbnails) {
+            yield* this.verifyThumbnails(node);
+        }
+    }
+
+    private async *verifyAuthor(
+        author: Author,
+        info: { nodeUid: string; authorType: string; revisionUid?: string; node: MaybeNode },
+    ): AsyncGenerator<DiagnosticResult> {
+        if (!author.ok) {
+            yield {
+                type: 'unverified_author',
+                claimedAuthor: author.error.claimedAuthor,
+                error: author.error.error,
+                ...info,
+            };
+        }
+    }
+
+    private async *verifyFileExtendedAttributes(node: MaybeNode): AsyncGenerator<DiagnosticResult> {
+        const activeRevision = getActiveRevision(node);
+
+        const expectedAttributes = getNodeType(node) === NodeType.File;
+
+        const claimedSha1 = activeRevision?.claimedDigests?.sha1;
+        if (claimedSha1 && !/^[0-9a-f]{40}$/i.test(claimedSha1)) {
+            yield {
+                type: 'extended_attributes_error',
+                ...getNodeUids(node),
+                field: 'sha1',
+                value: claimedSha1,
+            };
+        }
+
+        if (expectedAttributes && !claimedSha1) {
+            yield {
+                type: 'extended_attributes_missing_field',
+                ...getNodeUids(node),
+                missingField: 'sha1',
+            };
+        }
+    }
+
+    private async *verifyContent(node: MaybeNode): AsyncGenerator<DiagnosticResult> {
+        if (getNodeType(node) !== NodeType.File) {
+            return;
+        }
+        const activeRevision = getActiveRevision(node);
+        if (!activeRevision) {
+            yield {
+                type: 'content_file_missing_revision',
+                nodeUid: node.ok ? node.value.uid : node.error.uid,
+            };
+            return;
+        }
+
+        let downloader: FileDownloader;
+        try {
+            downloader = await this.protonDriveClient.getFileRevisionDownloader(activeRevision.uid);
+        } catch (error: unknown) {
+            yield {
+                type: 'sdk_error',
+                call: `getFileRevisionDownloader(${activeRevision.uid})`,
+                error,
+            };
+            return;
+        }
+
+        const claimedSha1 = activeRevision.claimedDigests?.sha1;
+        const claimedSizeInBytes = downloader.getClaimedSizeInBytes();
+
+        const integrityVerificationStream = new IntegrityVerificationStream();
+        const controller = downloader.writeToStream(integrityVerificationStream);
+
+        try {
+            await controller.completion();
+
+            const computedSha1 = integrityVerificationStream.computedSha1;
+            const computedSizeInBytes = integrityVerificationStream.computedSizeInBytes;
+            if (claimedSha1 !== computedSha1 || claimedSizeInBytes !== computedSizeInBytes) {
+                yield {
+                    type: 'content_integrity_error',
+                    ...getNodeUids(node),
+                    claimedSha1,
+                    computedSha1,
+                    claimedSizeInBytes,
+                    computedSizeInBytes,
+                };
+            }
+        } catch (error: unknown) {
+            yield {
+                type: 'content_download_error',
+                ...getNodeUids(node),
+                error,
+            };
+        }
+    }
+
+    private async *verifyThumbnails(node: MaybeNode): AsyncGenerator<DiagnosticResult> {
+        if (getNodeType(node) !== NodeType.File) {
+            return;
+        }
+
+        const nodeUid = node.ok ? node.value.uid : node.error.uid;
+
+        try {
+            const result = await Array.fromAsync(
+                this.protonDriveClient.iterateThumbnails([nodeUid], ThumbnailType.Type1),
+            );
+
+            if (result.length === 0) {
+                yield {
+                    type: 'sdk_error',
+                    call: `iterateThumbnails(${nodeUid})`,
+                    error: new Error('No thumbnails found'),
+                };
+            }
+            // TODO: We should have better way to check if the thumbnail is not expected.
+            if (!result[0].ok && result[0].error !== 'Node has no thumbnail') {
+                yield {
+                    type: 'thumbnails_error',
+                    nodeUid,
+                    error: result[0].error,
+                };
+            }
+        } catch (error: unknown) {
+            yield {
+                type: 'sdk_error',
+                call: `iterateThumbnails(${nodeUid})`,
+                error,
+            };
+        }
+    }
+
+    private async *verifyNodeChildren(node: MaybeNode, options?: DiagnosticOptions): AsyncGenerator<DiagnosticResult> {
+        const nodeUid = node.ok ? node.value.uid : node.error.uid;
+        try {
+            for await (const child of this.protonDriveClient.iterateFolderChildren(node)) {
+                yield* this.verifyNodeTree(child, options);
+            }
+        } catch (error: unknown) {
+            yield {
+                type: 'sdk_error',
+                call: `iterateFolderChildren(${nodeUid})`,
+                error,
+            };
+        }
+    }
+}
+
+function getNodeUids(node: MaybeNode): { nodeUid: string; revisionUid?: string } {
+    const activeRevision = getActiveRevision(node);
+    return {
+        nodeUid: node.ok ? node.value.uid : node.error.uid,
+        revisionUid: activeRevision?.uid,
+    };
+}
+
+function getNodeType(node: MaybeNode): NodeType {
+    return node.ok ? node.value.type : node.error.type;
+}
+
+function getActiveRevision(node: MaybeNode): Revision | undefined {
+    if (node.ok) {
+        return node.value.activeRevision;
+    }
+    if (node.error.activeRevision?.ok) {
+        return node.error.activeRevision.value;
+    }
+    return undefined;
+}

package/src/diagnostic/sdkDiagnosticFull.ts

@@ -0,0 +1,37 @@
+import { MaybeNode } from '../interface';
+import { DiagnosticHTTPClient } from './httpClient';
+import { Diagnostic, DiagnosticOptions, DiagnosticResult } from './interface';
+import { DiagnosticTelemetry } from './telemetry';
+import { zipGenerators } from './zipGenerators';
+
+/**
+ * Diagnostic tool that produces full diagnostic, including logs and metrics
+ * by reading the events from the telemetry and HTTP client.
+ */
+export class FullSDKDiagnostic implements Diagnostic {
+    constructor(
+        private diagnostic: Diagnostic,
+        private telemetry: DiagnosticTelemetry,
+        private httpClient: DiagnosticHTTPClient,
+    ) {
+        this.diagnostic = diagnostic;
+        this.telemetry = telemetry;
+        this.httpClient = httpClient;
+    }
+
+    async *verifyMyFiles(options?: DiagnosticOptions): AsyncGenerator<DiagnosticResult> {
+        yield* this.yieldEvents(this.diagnostic.verifyMyFiles(options));
+    }
+
+    async *verifyNodeTree(node: MaybeNode, options?: DiagnosticOptions): AsyncGenerator<DiagnosticResult> {
+        yield* this.yieldEvents(this.diagnostic.verifyNodeTree(node, options));
+    }
+
+    private async *yieldEvents(generator: AsyncGenerator<DiagnosticResult>): AsyncGenerator<DiagnosticResult> {
+        yield* zipGenerators(generator, this.internalGenerator(), { stopOnFirstDone: true });
+    }
+
+    private async *internalGenerator(): AsyncGenerator<DiagnosticResult> {
+        yield* zipGenerators(this.telemetry.iterateEvents(), this.httpClient.iterateEvents());
+    }
+}

package/src/diagnostic/telemetry.ts

@@ -0,0 +1,74 @@
+import { MetricEvent } from '../interface';
+import { LogRecord, LogLevel } from '../telemetry';
+import { EventsGenerator } from './eventsGenerator';
+
+/**
+ * Special telemetry that is compatible with the SDK.
+ *
+ * It is a probe into SDK to observe whats going on and report any suspicious
+ * behavior.
+ *
+ * It should be used only for diagnostic purposes.
+ */
+export class DiagnosticTelemetry extends EventsGenerator {
+    getLogger(name: string): Logger {
+        return new Logger(name, (log) => {
+            this.enqueueEvent({
+                type: log.level === LogLevel.ERROR ? 'log_error' : 'log_warning',
+                log,
+            });
+        });
+    }
+
+    logEvent(event: MetricEvent): void {
+        if (event.eventName === 'download' && !event.error) {
+            return;
+        }
+        if (event.eventName === 'volumeEventsSubscriptionsChanged') {
+            return;
+        }
+
+        this.enqueueEvent({
+            type: 'metric',
+            event,
+        });
+    }
+}
+
+class Logger {
+    constructor(
+        private name: string,
+        private callback?: (log: LogRecord) => void,
+    ) {
+        this.name = name;
+        this.callback = callback;
+    }
+
+    // Debug or info logs are excluded from the diagnostic.
+    // These logs should not include any suspicious behavior.
+
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    debug(message: string) {}
+
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    info(message: string) {}
+
+    warn(message: string) {
+        this.callback?.({
+            time: new Date(),
+            level: LogLevel.WARNING,
+            loggerName: this.name,
+            message,
+        });
+    }
+
+    error(message: string, error?: unknown) {
+        this.callback?.({
+            time: new Date(),
+            level: LogLevel.ERROR,
+            loggerName: this.name,
+            message,
+            error,
+        });
+    }
+}