@protontech/drive-sdk 0.0.10

package/src/internal/download/thumbnailDownloader.ts

@@ -0,0 +1,250 @@
+import { c } from 'ttag';
+
+import { NodeType, ThumbnailType, ProtonDriveTelemetry, Logger, ThumbnailResult } from "../../interface";
+import { ValidationError } from '../../errors';
+import { LoggerWithPrefix } from '../../telemetry';
+import { DownloadAPIService } from "./apiService";
+import { DownloadCryptoService } from "./cryptoService";
+import { NodesService } from "./interface";
+import { getErrorMessage } from '../errors';
+
+/**
+ * Maximum number of thumbnails that can be downloaded at the same time.
+ */
+const MAX_DOWNLOAD_THUMBNAILS = 10;
+
+/**
+ * Maximum number of retries for thumbnail download and decryption.
+ */
+const MAX_THUMBNAIL_DOWNLOAD_ATTEMPTS = 2;
+
+export class ThumbnailDownloader {
+    private logger: Logger;
+
+    private batchThumbnailToNodeUids = new Map<string, string>();
+    private ongoingDownloads = new Map<string, Promise<void>>();
+    private bufferedThumbnails: (
+        { nodeUid: string, ok: true, thumbnail: Uint8Array } |
+        { nodeUid: string, ok: false, error: string }
+    )[] = [];
+
+    constructor(
+        telemetry: ProtonDriveTelemetry,
+        private nodesService: NodesService,
+        private apiService: DownloadAPIService,
+        private cryptoService: DownloadCryptoService,
+    ) {
+        this.logger = telemetry.getLogger("download");
+        this.nodesService = nodesService;
+        this.apiService = apiService;
+        this.cryptoService = cryptoService;
+    }
+
+    async *iterateThumbnails(
+        nodeUids: string[],
+        thumbnailType = ThumbnailType.Type1,
+        signal?: AbortSignal,
+    ): AsyncGenerator<ThumbnailResult> {
+        if (nodeUids.length === 0) {
+            return;
+        }
+
+        for await (const result of this.iterateThumbnailUids(nodeUids, thumbnailType, signal)) {
+            if (!result.ok) {
+                yield result;
+                continue;
+            }
+
+            this.batchThumbnailToNodeUids.set(result.thumbnailUid, result.nodeUid);
+            if (this.batchThumbnailToNodeUids.size >= MAX_DOWNLOAD_THUMBNAILS) {
+                await this.requestBatchedThumbnailDownloads(signal);
+            }
+
+            while (this.ongoingDownloads.size >= MAX_DOWNLOAD_THUMBNAILS) {
+                await Promise.race(this.ongoingDownloads.values());
+                yield* this.bufferedThumbnails;
+                this.bufferedThumbnails = [];
+            }
+        }
+
+        await this.requestBatchedThumbnailDownloads(signal);
+
+        while (this.ongoingDownloads.size > 0) {
+            await Promise.race(this.ongoingDownloads.values());
+            yield* this.bufferedThumbnails;
+            this.bufferedThumbnails = [];
+        }
+
+        yield* this.bufferedThumbnails;
+        this.bufferedThumbnails = [];
+    }
+
+    private async *iterateThumbnailUids(nodeUids: string[], thumbnailType: ThumbnailType, signal?: AbortSignal): AsyncGenerator<
+        { nodeUid: string, ok: true, thumbnailUid: string } |
+        { nodeUid: string, ok: false, error: string }
+    > {
+        for await (const node of this.nodesService.iterateNodes(nodeUids, signal)) {
+            if ('missingUid' in node) {
+                yield {
+                    nodeUid: node.missingUid,
+                    ok: false,
+                    error: c("Error").t`Node not found`,
+                }
+                continue;
+            }
+            if (node.type !== NodeType.File) {
+                yield {
+                    nodeUid: node.uid,
+                    ok: false,
+                    error: c("Error").t`Node is not a file`,
+                }
+                continue;
+            }
+
+            let thumbnail;
+            if (node.activeRevision?.ok) {
+                thumbnail = node.activeRevision.value.thumbnails?.find(
+                    (t) => t.type === thumbnailType,
+                );
+            }
+            if (!thumbnail) {
+                yield {
+                    nodeUid: node.uid,
+                    ok: false,
+                    error: c("Error").t`Node has no thumbnail`,
+                }
+                continue;
+            }
+
+            yield {
+                nodeUid: node.uid,
+                ok: true,
+                thumbnailUid: thumbnail.uid,
+            }
+        }
+    }
+
+    private async requestBatchedThumbnailDownloads(signal?: AbortSignal) {
+        if (this.batchThumbnailToNodeUids.size === 0) {
+            return;
+        }
+
+        this.logger.debug(`Downloading thumbnail batch of size ${this.batchThumbnailToNodeUids.size}`);
+
+        for await (const downloadResult of this.iterateThumbnailDownloads(signal)) {
+            if (!downloadResult.ok) {
+                this.bufferedThumbnails.push({
+                    nodeUid: downloadResult.nodeUid,
+                    ok: false,
+                    error: downloadResult.error,
+                });
+                continue;
+            }
+
+            this.ongoingDownloads.set(
+                downloadResult.nodeUid,
+                downloadResult.downloadPromise
+                    .then((thumbnail) => {
+                        this.bufferedThumbnails.push({
+                            nodeUid: downloadResult.nodeUid,
+                            ok: true,
+                            thumbnail,
+                        });
+                    })
+                    .catch((error) => {
+                        this.bufferedThumbnails.push({
+                            nodeUid: downloadResult.nodeUid,
+                            ok: false,
+                            error: getErrorMessage(error),
+                        });
+                    })
+                    .finally(() => {
+                        this.ongoingDownloads.delete(downloadResult.nodeUid);
+                    }),
+            );
+        }
+        
+        this.batchThumbnailToNodeUids.clear();
+    }
+
+    private async *iterateThumbnailDownloads(signal?: AbortSignal): AsyncGenerator<
+        { nodeUid: string, ok: true, downloadPromise: Promise<Uint8Array> } |
+        { nodeUid: string, ok: false, error: string }
+    > {
+        const missingThumbnailUids = new Set(this.batchThumbnailToNodeUids.keys());
+
+        for await (const result of this.apiService.iterateThumbnails(
+            Array.from(this.batchThumbnailToNodeUids.keys()),
+            signal,
+        )) {
+            const nodeUid = this.batchThumbnailToNodeUids.get(result.uid);
+            if (!nodeUid) {
+                this.logger.warn(`Unexpected thumbnail UID ${result.uid} returned from API`);
+                continue;
+            }
+
+            missingThumbnailUids.delete(result.uid);
+
+            if (!result.ok) {
+                yield {
+                    nodeUid,
+                    ok: false,
+                    error: result.error,
+                }
+                continue;
+            }
+
+            yield {
+                nodeUid,
+                ok: true,
+                downloadPromise: this.downloadThumbnail(nodeUid, result.bareUrl, result.token, signal),
+            }
+        }
+
+        for (const uid of missingThumbnailUids) {
+            const nodeUid = this.batchThumbnailToNodeUids.get(uid)!;
+            this.logger.warn(`Thumbnail UID ${uid} not found in API response`);
+            yield {
+                nodeUid,
+                ok: false,
+                error: c("Error").t`Thumbnail not found`,
+            }
+        }
+    }
+
+    private async downloadThumbnail(nodeUid: string, bareUrl: string, token: string, signal?: AbortSignal): Promise<Uint8Array> {
+        const logger = new LoggerWithPrefix(this.logger, `thumbnail ${token}`);
+        
+        let decryptedBlock: Uint8Array | null = null;
+        let attempt = 0;
+
+        while (!decryptedBlock) {
+            logger.debug(`Downloading`);
+            attempt++;
+
+            try {
+                const [nodeKeys, encryptedBlock] = await Promise.all([
+                    this.nodesService.getNodeKeys(nodeUid),
+                    this.apiService.downloadBlock(bareUrl, token, undefined, signal),
+                ]);
+
+                if (!nodeKeys.contentKeyPacketSessionKey) {
+                    throw new ValidationError(c("Error").t`File has no content key`);
+                }
+
+                logger.debug(`Decrypting`);
+                decryptedBlock = await this.cryptoService.decryptThumbnail(encryptedBlock, nodeKeys.contentKeyPacketSessionKey);
+            } catch (error: unknown) {
+                if (attempt <= MAX_THUMBNAIL_DOWNLOAD_ATTEMPTS) {
+                    logger.warn(`Thumbnail download failed #${attempt}, retrying: ${getErrorMessage(error)}`);
+                    continue;
+                }
+
+                logger.error(`Thumbnail download failed`, error);
+                throw error;
+            }
+        }
+
+        return decryptedBlock;
+    }
+}

package/src/internal/errors.test.ts

@@ -0,0 +1,21 @@
+import { VERIFICATION_STATUS } from '../crypto';
+import { getVerificationMessage } from './errors';
+
+describe('getVerificationMessage', () => {
+    const testCases: [VERIFICATION_STATUS, string | undefined, boolean, string][] = [
+        [VERIFICATION_STATUS.NOT_SIGNED, 'type', false, 'Missing signature for type'],
+        [VERIFICATION_STATUS.NOT_SIGNED, undefined, false, 'Missing signature'],
+        [VERIFICATION_STATUS.NOT_SIGNED, 'type', true, 'Missing signature for type'],
+        [VERIFICATION_STATUS.NOT_SIGNED, undefined, true, 'Missing signature'],
+        [VERIFICATION_STATUS.SIGNED_AND_INVALID, 'type', false, 'Signature verification for type failed'],
+        [VERIFICATION_STATUS.SIGNED_AND_INVALID, undefined, false, 'Signature verification failed'],
+        [VERIFICATION_STATUS.SIGNED_AND_INVALID, 'type', true, 'Verification keys for type are not available'],
+        [VERIFICATION_STATUS.SIGNED_AND_INVALID, undefined, true, 'Verification keys are not available'],
+    ];
+
+    for (const [status, type, notAvailable, expected] of testCases) {
+        it(`returns correct message for status ${status} with type ${type} and notAvailable ${notAvailable}`, () => {
+            expect(getVerificationMessage(status, type, notAvailable)).toBe(expected);
+        });
+    }
+});

package/src/internal/errors.ts

@@ -0,0 +1,28 @@
+import { c } from 'ttag';
+
+import { VERIFICATION_STATUS } from "../crypto";
+
+export function getErrorMessage(error: unknown): string {
+    return error instanceof Error ? error.message : c('Error').t`Unknown error`;
+}
+
+/**
+ * @param signatureType - Must be translated before calling this function.
+ */
+export function getVerificationMessage(verified: VERIFICATION_STATUS, signatureType?: string, notAvailableVerificationKeys = false): string {
+    if (verified === VERIFICATION_STATUS.NOT_SIGNED) {
+        return signatureType
+            ? c('Error').t`Missing signature for ${signatureType}`
+            : c('Error').t`Missing signature`;
+    }
+
+    if (notAvailableVerificationKeys) {
+        return signatureType
+            ? c('Error').t`Verification keys for ${signatureType} are not available`
+            : c('Error').t`Verification keys are not available`;
+    }
+
+    return signatureType
+        ? c('Error').t`Signature verification for ${signatureType} failed`
+        : c('Error').t`Signature verification failed`;
+}

package/src/internal/events/apiService.ts

@@ -0,0 +1,87 @@
+import { Logger } from "../../interface";
+import { DriveAPIService, drivePaths, corePaths } from "../apiService";
+import { makeNodeUid } from "../uids";
+import { DriveEvents, DriveEvent, DriveEventType } from "./interface";
+
+type GetCoreLatestEventResponse = corePaths['/core/{_version}/events/latest']['get']['responses']['200']['content']['application/json'];
+type GetCoreEventResponse = corePaths['/core/{_version}/events/{id}']['get']['responses']['200']['content']['application/json'];
+
+type GetVolumeLatestEventResponse = drivePaths['/drive/volumes/{volumeID}/events/latest']['get']['responses']['200']['content']['application/json'];
+type GetVokumeEventResponse = drivePaths['/drive/v2/volumes/{volumeID}/events/{eventID}']['get']['responses']['200']['content']['application/json'];
+
+const VOLUME_EVENT_TYPE_MAP = {
+    0: DriveEventType.NodeDeleted,
+    1: DriveEventType.NodeCreated,
+    2: DriveEventType.NodeUpdated,
+    3: DriveEventType.NodeUpdatedMetadata,
+}
+
+/**
+ * Provides API communication for fetching events.
+ * 
+ * The service is responsible for transforming local objects to API payloads
+ * and vice versa. It should not contain any business logic.
+ */
+export class EventsAPIService {
+    constructor(private apiService: DriveAPIService, private logger?: Logger) {
+        this.apiService = apiService;
+        this.logger = logger;
+    }
+
+    async getCoreLatestEventId(): Promise<string> {
+        const result = await this.apiService.get<GetCoreLatestEventResponse>(`core/v4/events/latest`);
+        return result.EventID as string;
+    }
+
+    async getCoreEvents(eventId: string): Promise<DriveEvents> {
+        // TODO: Switch to v6 endpoint: DriveShareRefresh doesnt seem to be part of it.
+        const result = await this.apiService.get<GetCoreEventResponse>(`core/v5/events/${eventId}`);
+        const events: DriveEvent[] = result.DriveShareRefresh?.Action === 2 ? [
+            {
+                type: DriveEventType.ShareWithMeUpdated,
+            }
+        ] : [];
+
+        return {
+            lastEventId: result.EventID,
+            more: result.More === 1,
+            refresh: result.Refresh === 1,
+            events,
+        };
+    }
+
+    async getVolumeLatestEventId(volumeId: string): Promise<string> {
+        const result = await this.apiService.get<GetVolumeLatestEventResponse>(`drive/volumes/${volumeId}/events/latest`);
+        return result.EventID;
+    }
+
+    async getVolumeEvents(volumeId: string, eventId: string, isOwnVolume = false): Promise<DriveEvents> {
+        const result = await this.apiService.get<GetVokumeEventResponse>(`drive/v2/volumes/${volumeId}/events/${eventId}`);
+        return {
+            lastEventId: result.EventID,
+            more: result.More,
+            refresh: result.Refresh,
+            events: result.Events.map((event): DriveEvent => {
+                const type = VOLUME_EVENT_TYPE_MAP[event.EventType];
+                const uids = {
+                    nodeUid: makeNodeUid(volumeId, event.Link.LinkID),
+                    parentNodeUid: makeNodeUid(volumeId, event.Link.ParentLinkID as string),
+                }
+                // VOLUME_EVENT_TYPE_MAP will never return this event type.
+                // It is here to satisfy the type checker. It is safe to do.
+                if (type === DriveEventType.ShareWithMeUpdated) {
+                    return {
+                        type,
+                    };
+                }
+                return {
+                    type,
+                    ...uids,
+                    isTrashed: event.Link.IsTrashed,
+                    isShared: event.Link.IsShared,
+                    isOwnVolume,
+                };
+            }),
+        };
+    }
+}

package/src/internal/events/cache.test.ts

@@ -0,0 +1,47 @@
+import { MemoryCache } from "../../cache";
+import { EventsCache } from "./cache";
+
+describe("EventsCache", () => {
+    let memoryCache: MemoryCache<string>;
+    let cache: EventsCache;
+
+    beforeEach(() => {
+        memoryCache = new MemoryCache();
+        cache = new EventsCache(memoryCache);
+    });
+
+    it("should store and retrieve last event ID", async () => {
+        const key = "volume1";
+        await cache.setLastEventId(key, { lastEventId: "eventId1", pollingIntervalInSeconds: 0, isOwnVolume: true });
+        await cache.setLastEventId(key, { lastEventId: "eventId2", pollingIntervalInSeconds: 0, isOwnVolume: true });
+        const result = await cache.getLastEventId(key);
+        expect(result).toBe("eventId2");
+    });
+
+    it("should store and retrieve polling interval", async () => {
+        const key = "volume1";
+        await cache.setLastEventId(key, { lastEventId: "lastEventId", pollingIntervalInSeconds: 10, isOwnVolume: true });
+        await cache.setLastEventId(key, { lastEventId: "lastEventId", pollingIntervalInSeconds: 20, isOwnVolume: true });
+        const result = await cache.getPollingIntervalInSeconds(key);
+        expect(result).toBe(20);
+    });
+
+    it("should store and retrieve subscribed volume IDs", async () => {
+        await cache.setLastEventId("volume1", { lastEventId: "lastEventId", pollingIntervalInSeconds: 0, isOwnVolume: true });
+        await cache.setLastEventId("volume2", { lastEventId: "lastEventId", pollingIntervalInSeconds: 0, isOwnVolume: true });
+        const result = await cache.getSubscribedVolumeIds();
+        expect(result).toStrictEqual(["volume1", "volume2"]);
+    });
+
+    it("should not fail if cache is empty", async () => {
+        const result = await cache.getLastEventId("volume1");
+        expect(result).toBe(undefined);
+    });
+
+    it("should call cache only once", async () => {
+        const spy = jest.spyOn(memoryCache, "getEntity");
+        await cache.getLastEventId("volume1");
+        await cache.getLastEventId("volume1");
+        expect(spy).toHaveBeenCalledTimes(1);
+    });
+});

package/src/internal/events/cache.ts

@@ -0,0 +1,80 @@
+import { ProtonDriveEntitiesCache } from "../../interface";
+
+type CachedEventsData = {
+    // Key is either a volume ID for volume events or 'core' for core events.
+    [key: string]: EventsData;
+};
+
+interface EventsData {
+    lastEventId: string;
+    pollingIntervalInSeconds: number;
+    isOwnVolume: boolean;
+}
+
+/**
+ * Provides caching for events IDs.
+ */
+export class EventsCache {
+    /**
+     * Locally cached events data to avoid unnecessary reads from the cache.
+     * Data about last event ID or interval might be accessed often by events
+     * managers.
+     */
+    private events?: CachedEventsData;
+
+    constructor(private driveCache: ProtonDriveEntitiesCache) {
+        this.driveCache = driveCache;
+    }
+
+    async setLastEventId(volumeIdOrCore: string, eventsData: EventsData): Promise<void> {
+        const events = await this.getEvents();
+        events[volumeIdOrCore] = eventsData;
+        await this.cacheEvents(events);
+    }
+
+    async getLastEventId(volumeIdOrCore: string): Promise<string | undefined> {
+        const events = await this.getEvents();
+        if (events[volumeIdOrCore]) {
+            return events[volumeIdOrCore].lastEventId;
+        }
+    }
+
+    async getPollingIntervalInSeconds(volumeIdOrCore: string): Promise<number | undefined> {
+        const events = await this.getEvents();
+        if (events[volumeIdOrCore]) {
+            return events[volumeIdOrCore].pollingIntervalInSeconds;
+        }
+    }
+
+    async isOwnVolume(volumeIdOrCore: string): Promise<boolean | undefined> {
+        const events = await this.getEvents();
+        if (events[volumeIdOrCore]) {
+            return events[volumeIdOrCore].isOwnVolume;
+        }
+    }
+
+    async getSubscribedVolumeIds(): Promise<string[]> {
+        const events = await this.getEvents();
+        return Object.keys(events).filter((volumeIdOrCore) => volumeIdOrCore !== 'core');
+    }
+
+    private async getEvents(): Promise<CachedEventsData> {
+        if (!this.events) {
+            this.events = await this.getCachedEvents();
+        }
+        return this.events;
+    }
+
+    private async getCachedEvents(): Promise<CachedEventsData> {
+        try {
+            const events = await this.driveCache.getEntity('events');
+            return JSON.parse(events);
+        } catch {};
+        return {};
+    }
+
+    private async cacheEvents(events: CachedEventsData): Promise<void> {
+        this.events = events;
+        await this.driveCache.setEntity('events', JSON.stringify(events));
+    }
+}

package/src/internal/events/coreEventManager.ts

@@ -0,0 +1,69 @@
+import { Logger } from "../../interface";
+import { LoggerWithPrefix } from "../../telemetry";
+import { EventsAPIService } from "./apiService";
+import { EventsCache } from "./cache";
+import { DriveEvent, DriveEventType } from "./interface";
+import { EventManager } from "./eventManager";
+
+/**
+ * Combines API and event manager to provide a service for listening to
+ * core events. Core events are events that are not specific to any volume.
+ * At this moment, Drive listenes only to shares with me updates from core
+ * events. Such even indicates that user was invited to the new share or
+ * that user's membership was removed from existing one and lost access.
+ * 
+ * The client might be already using own core events, thus this service
+ * is here only in case the client is not connected to the Proton services
+ * with own implementation.
+ */
+export class CoreEventManager {
+    private manager: EventManager<DriveEvent>;
+
+    constructor(logger: Logger, private apiService: EventsAPIService, private cache: EventsCache) {
+        this.apiService = apiService;
+
+        this.manager = new EventManager(
+            new LoggerWithPrefix(logger, `core`),
+            () => this.getLastEventId(),
+            (eventId) => this.apiService.getCoreEvents(eventId),
+            (lastEventId) => this.cache.setLastEventId('core', {
+                lastEventId,
+                pollingIntervalInSeconds: this.manager.pollingIntervalInSeconds,
+                isOwnVolume: false,
+            }),
+        );
+    }
+
+    private async getLastEventId(): Promise<string> {
+        const lastEventId = await this.cache.getLastEventId('core');
+        if (lastEventId) {
+            return lastEventId;
+        }
+        return this.apiService.getCoreLatestEventId();
+    }
+
+    async startSubscription(): Promise<void> {
+        await this.manager.start();
+    }
+
+    async stopSubscription(): Promise<void> {
+        await this.manager.stop();
+    }
+
+    addListener(callback: (events: DriveEvent[]) => Promise<void>): void {
+        this.manager.addListener(async (events, fullRefresh) => {
+            if (events) {
+                await callback(events);
+            }
+            if (fullRefresh) {
+                // Because only updates about shares that are shared with me
+                // are listened to from core events, in the case of core full
+                // refresh, we don't have to refresh anything more than this
+                // one specific event.
+                await callback([{
+                    type: DriveEventType.ShareWithMeUpdated,
+                }]);
+            }
+        });
+    }
+}