@protontech/drive-sdk 0.0.10

package/src/internal/events/eventManager.test.ts

@@ -0,0 +1,139 @@
+import { getMockLogger } from "../../tests/logger";
+import { NotFoundAPIError } from "../apiService";
+import { EventManager } from "./eventManager";
+
+jest.useFakeTimers();
+
+describe("EventManager", () => {
+    let manager: EventManager<string>;
+    
+    const getLastEventIdMock = jest.fn();
+    const getEventsMock = jest.fn();
+    const updateLatestEventIdMock = jest.fn();
+    const listenerMock = jest.fn();
+
+    beforeEach(() => {
+        jest.clearAllMocks();
+
+        getLastEventIdMock.mockImplementation(() => Promise.resolve("eventId1"));
+        getEventsMock.mockImplementation(() => Promise.resolve({
+            lastEventId: "eventId2",
+            more: false,
+            refresh: false,
+            events: ["event1", "event2"],
+        }));
+
+        manager = new EventManager(
+            getMockLogger(),
+            getLastEventIdMock,
+            getEventsMock,
+            updateLatestEventIdMock,
+        );
+        manager.addListener(listenerMock);
+    });
+
+    afterEach(async () => {
+        await manager.stop();
+    });
+
+    it("should get latest event ID on first run only", async () => {
+        await manager.start();
+        expect(getLastEventIdMock).toHaveBeenCalledTimes(1);
+        expect(getEventsMock).toHaveBeenCalledTimes(0);
+        expect(listenerMock).toHaveBeenCalledTimes(0);
+        expect(updateLatestEventIdMock).toHaveBeenCalledTimes(1);
+        expect(updateLatestEventIdMock).toHaveBeenCalledWith('eventId1');
+    });
+
+    it("should notify about events in the next run", async () => {
+        await manager.start();
+        expect(getLastEventIdMock).toHaveBeenCalledTimes(1);
+        expect(getEventsMock).toHaveBeenCalledTimes(0);
+        expect(listenerMock).toHaveBeenCalledTimes(0);
+        expect(updateLatestEventIdMock).toHaveBeenCalledTimes(1);
+        expect(updateLatestEventIdMock).toHaveBeenCalledWith('eventId1');
+        updateLatestEventIdMock.mockClear();
+        await jest.runOnlyPendingTimersAsync();
+        expect(getEventsMock).toHaveBeenCalledTimes(1);
+        expect(listenerMock).toHaveBeenCalledTimes(1);
+        expect(updateLatestEventIdMock).toHaveBeenCalledTimes(1);
+        expect(updateLatestEventIdMock).toHaveBeenCalledWith('eventId2');
+    });
+
+    it("should continue with more events", async () => {
+        getEventsMock.mockImplementation((lastEventId: string) => Promise.resolve({
+            lastEventId: lastEventId === "eventId1" ? "eventId2" : "eventId3",
+            more: lastEventId === "eventId1" ? true : false,
+            refresh: false,
+            events: lastEventId === "eventId1" ? ["event1", "event2"] : ["event3"],
+        }));
+        await manager.start();
+        await jest.runOnlyPendingTimersAsync();
+        expect(getEventsMock).toHaveBeenCalledTimes(2);
+        expect(listenerMock).toHaveBeenCalledTimes(2);
+        expect(listenerMock).toHaveBeenCalledWith(["event1", "event2"], false);
+        expect(listenerMock).toHaveBeenCalledWith(["event3"], false);
+        expect(updateLatestEventIdMock).toHaveBeenCalledTimes(3);
+        expect(updateLatestEventIdMock).toHaveBeenCalledWith('eventId1');
+        expect(updateLatestEventIdMock).toHaveBeenCalledWith('eventId2');
+        expect(updateLatestEventIdMock).toHaveBeenCalledWith('eventId3');
+    });
+
+    it("should refresh if event does not exist", async () => {
+        getEventsMock.mockImplementation(() => Promise.reject(new NotFoundAPIError('Event not found', 2501)));
+        await manager.start();
+        await jest.runOnlyPendingTimersAsync();
+        expect(getLastEventIdMock).toHaveBeenCalledTimes(2);
+        expect(listenerMock).toHaveBeenCalledTimes(1);
+        expect(listenerMock).toHaveBeenCalledWith([], true);
+        expect(updateLatestEventIdMock).toHaveBeenCalledTimes(1);
+        expect(updateLatestEventIdMock).toHaveBeenCalledWith('eventId1');
+    });
+
+    it("should retry on error", async () => {
+        let index = 0;
+        getEventsMock.mockImplementation(() => {
+            index++;
+            if (index <= 3) {
+                return Promise.reject(new Error("Error"));
+            }
+            return Promise.resolve({
+                lastEventId: "eventId2",
+                more: false,
+                refresh: false,
+                events: ["event1", "event2"],
+            });
+        });
+        await manager.start();
+        updateLatestEventIdMock.mockClear();
+
+        // First failure.
+        await jest.runOnlyPendingTimersAsync();
+        expect(listenerMock).toHaveBeenCalledTimes(0);
+        expect(manager.nextPollTimeout).toBe(30000);
+
+        // Second failure.
+        await jest.runOnlyPendingTimersAsync();
+        expect(listenerMock).toHaveBeenCalledTimes(0);
+        expect(manager.nextPollTimeout).toBe(60000);
+
+        // Third failure.
+        await jest.runOnlyPendingTimersAsync();
+        expect(listenerMock).toHaveBeenCalledTimes(0);
+        expect(manager.nextPollTimeout).toBe(90000);
+
+        // And now it passes.
+        await jest.runOnlyPendingTimersAsync();
+        expect(listenerMock).toHaveBeenCalledTimes(1);
+        expect(listenerMock).toHaveBeenCalledWith(["event1", "event2"], false);
+        expect(updateLatestEventIdMock).toHaveBeenCalledTimes(1);
+        expect(updateLatestEventIdMock).toHaveBeenCalledWith('eventId2');
+    });
+
+    it("should stop polling", async () => {
+        await manager.start();
+        await manager.stop();
+        await jest.runOnlyPendingTimersAsync();
+        expect(getEventsMock).toHaveBeenCalledTimes(0);
+    });
+});

package/src/internal/events/eventManager.ts

@@ -0,0 +1,168 @@
+import { Logger } from "../../interface";
+import { NotFoundAPIError } from "../apiService";
+import { Events } from "./interface";
+
+const DEFAULT_POLLING_INTERVAL_IN_SECONDS = 30;
+const FIBONACCI_LIST = [1, 1, 2, 3, 5, 8, 13];
+
+/**
+ * `fullRefresh` is true when the event manager has requested a full
+ * refresh of the data. That can happen if there is too many events
+ * to be processed or the last event ID is too old.
+ */
+type Listener<T> = (events: T[], fullRefresh: boolean) => Promise<void>;
+
+/**
+ * Event manager general helper that is responsible for fetching events
+ * from the server and notifying listeners about the events.
+ * 
+ * The specific implementation of fetching the events from the API must
+ * be passed as dependency and can be used for any type of events that
+ * supports the same structure.
+ * 
+ * The manager will not start fetching events until the `start` method is
+ * called. Once started, the manager will fetch events in a loop with
+ * a timeout between each fetch. The default timeout is 30 seconds and
+ * additional jitter is used in case of failure.
+ * 
+ * Example of usage:
+ * 
+ * ```typescript
+ * const manager = new EventManager(
+ *    logger,
+ *    () => apiService.getLatestEventId(),
+ *    (eventId) => apiService.getEvents(eventId),
+ * );
+ * 
+ * manager.addListener((events, fullRefresh) => {
+ *   // Process the events
+ * });
+ * 
+ * manager.start();
+ * ```
+ */
+export class EventManager<T> {
+    private latestEventId?: string;
+    private timeoutHandle?: ReturnType<typeof setTimeout>;
+    private processPromise?: Promise<void>;
+    private listeners: Listener<T>[] = [];
+    private retryIndex: number = 0;
+
+    pollingIntervalInSeconds = DEFAULT_POLLING_INTERVAL_IN_SECONDS;
+
+    constructor(
+        private logger: Logger,
+        private getLatestEventId: () => Promise<string>,
+        private getEvents: (eventId: string) => Promise<Events<T>>,
+        private updateLatestEventId: (lastEventId: string) => Promise<void>,
+    ) {
+        this.logger = logger;
+        this.getLatestEventId = getLatestEventId;
+        this.getEvents = getEvents;
+        this.updateLatestEventId = updateLatestEventId;
+    }
+
+    addListener(callback: Listener<T>): void {
+        this.listeners.push(callback);
+    }
+
+    async start(): Promise<void> {
+        this.logger.info(`Starting event manager with polling interval ${this.pollingIntervalInSeconds} seconds`);
+        await this.stop();
+        this.processPromise = this.processEvents();
+    }
+
+    private async processEvents() {
+        try {
+            if (!this.latestEventId) {
+                this.latestEventId = await this.getLatestEventId();
+                await this.updateLatestEventId(this.latestEventId);
+            } else {
+                while (true) {
+                    let result;
+                    try {
+                        result = await this.getEvents(this.latestEventId);
+                    } catch (error: unknown) {
+                        // If last event ID is not found, we need to refresh the data.
+                        // Caller is notified via standard event update with refresh flag.
+                        if (error instanceof NotFoundAPIError) {
+                            this.logger.warn(`Last event ID not found, refreshing data`);
+                            result = {
+                                lastEventId: await this.getLatestEventId(), 
+                                more: false, 
+                                refresh: true,
+                                events: [],
+                            };
+                        } else {
+                            // Any other error is considered as a failure and we will retry
+                            // with backoff policy.
+                            throw error;
+                        }
+                    }
+                    await this.notifyListeners(result);
+                    if (result.lastEventId !== this.latestEventId) {
+                        await this.updateLatestEventId(result.lastEventId);
+                        this.latestEventId = result.lastEventId;
+                    }
+                    if (!result.more) {
+                        break;
+                    }
+                }
+            }
+            this.retryIndex = 0;
+        } catch (error: unknown) {
+            this.logger.error(`Failed to process events: ${error instanceof Error ? error.message : error} (retry ${this.retryIndex}, last event ID: ${this.latestEventId})`);
+            this.retryIndex++;
+        }
+
+        this.timeoutHandle = setTimeout(() => {
+            this.processPromise = this.processEvents();
+        }, this.nextPollTimeout);
+    };
+
+    private async notifyListeners(result: Events<T>): Promise<void> {
+        if (result.events.length === 0 && !result.refresh) {
+            return;
+        }
+        if (!this.listeners.length) {
+            return;
+        }
+
+        this.logger.debug(`Notifying listeners about ${result.events.length} events`);
+
+        for (const listener of this.listeners) {
+            try {
+                await listener(result.events, result.refresh);
+            } catch (error: unknown) {
+                this.logger.error(`Failed to process events: ${error instanceof Error ? error.message : error} (last event ID: ${result.lastEventId}, refresh: ${result.refresh})`);
+                throw error;
+            }
+        }
+    }
+
+    /**
+     * Polling timeout is using exponential backoff with Fibonacci sequence.
+     * 
+     * The timeout is public for testing purposes only.
+     */
+    get nextPollTimeout(): number {
+        const retryIndex = Math.min(this.retryIndex, FIBONACCI_LIST.length - 1);
+        return this.pollingIntervalInSeconds * 1000 * FIBONACCI_LIST[retryIndex];
+    }
+
+    async stop(): Promise<void> {
+        if (this.processPromise) {
+            this.logger.info(`Stopping event manager`);
+            try {
+                await this.processPromise;
+            } catch {}
+        }
+
+        if (!this.timeoutHandle) {
+            return;
+        }
+
+        clearTimeout(this.timeoutHandle);
+        this.timeoutHandle = undefined;
+    }
+}

package/src/internal/events/index.ts

@@ -0,0 +1,131 @@
+import { ProtonDriveEntitiesCache, Logger, ProtonDriveTelemetry } from "../../interface";
+import { DriveAPIService } from "../apiService";
+import { DriveListener } from "./interface";
+import { EventsAPIService } from "./apiService";
+import { EventsCache } from "./cache";
+import { CoreEventManager } from "./coreEventManager";
+import { VolumeEventManager } from "./volumeEventManager";
+
+export type { DriveEvent, DriveListener } from "./interface";
+export { DriveEventType } from "./interface";
+
+const OWN_VOLUME_POLLING_INTERVAL = 30;
+const OTHER_VOLUME_POLLING_INTERVAL = 60;
+
+/**
+ * Service for listening to drive events. The service is responsible for
+ * managing the subscriptions to the events and notifying the listeners
+ * about the new events.
+ */
+export class DriveEventsService {
+    private apiService: EventsAPIService;
+    private cache: EventsCache;
+    private subscribedToRemoteDataUpdates: boolean = false;
+    private listeners: DriveListener[] = [];
+    private coreEvents: CoreEventManager;
+    private volumesEvents: { [volumeId: string]: VolumeEventManager };
+    private logger: Logger;
+
+    constructor(private telemetry: ProtonDriveTelemetry, apiService: DriveAPIService, driveEntitiesCache: ProtonDriveEntitiesCache) {
+        this.telemetry = telemetry;
+        this.logger = telemetry.getLogger('events');
+        this.apiService = new EventsAPIService(apiService);
+        this.cache = new EventsCache(driveEntitiesCache);
+
+        // FIXME: Allow to pass own core events manager from the public interface.
+        this.coreEvents = new CoreEventManager(this.logger, this.apiService, this.cache);
+        this.volumesEvents = {};
+    }
+
+    /**
+     * Loads all the subscribed volumes (including core events) from the
+     * cache and starts listening to their events. Any additional volume
+     * that is subscribed to later will be automatically started.
+     */
+    async subscribeToRemoteDataUpdates(): Promise<void> {
+        if (this.subscribedToRemoteDataUpdates) {
+            return;
+        }
+
+        await this.loadSubscribedVolumeEventServices();
+        this.sendNumberOfVolumeSubscriptionsToTelemetry();
+
+        this.subscribedToRemoteDataUpdates = true;
+        await this.coreEvents.startSubscription();
+        await Promise.all(
+            Object.values(this.volumesEvents)
+                .map((volumeEvents) => volumeEvents.startSubscription())
+        );
+    }
+
+    /**
+     * Subscribe to given volume. The volume will be polled for events
+     * with the polling interval depending on the type of the volume.
+     * Own volumes are polled with highest frequency, while others are
+     * polled with lower frequency depending on the total number of
+     * subscriptions.
+     * 
+     * @param isOwnVolume - Owned volumes are polled with higher frequency.
+     */
+    async listenToVolume(volumeId: string, isOwnVolume = false): Promise<void> {
+        await this.loadSubscribedVolumeEventServices();
+
+        if (this.volumesEvents[volumeId]) {
+            return;
+        }
+        this.logger.debug(`Creating volume event manager for volume ${volumeId}`);
+        const manager = this.createVolumeEventManager(volumeId, isOwnVolume);
+
+        // FIXME: Use dynamic algorithm to determine polling interval for non-own volumes.
+        manager.setPollingInterval(isOwnVolume ? OWN_VOLUME_POLLING_INTERVAL : OTHER_VOLUME_POLLING_INTERVAL);
+        if (this.subscribedToRemoteDataUpdates) {
+            await manager.startSubscription();
+            this.sendNumberOfVolumeSubscriptionsToTelemetry();
+        }
+    }
+
+    private async loadSubscribedVolumeEventServices() {
+        for (const volumeId of await this.cache.getSubscribedVolumeIds()) {
+            if (!this.volumesEvents[volumeId]) {
+                const isOwnVolume = await this.cache.isOwnVolume(volumeId) || false;
+                this.createVolumeEventManager(volumeId, isOwnVolume);
+            }
+        }
+    }
+
+    private sendNumberOfVolumeSubscriptionsToTelemetry() {
+        this.telemetry.logEvent({
+            eventName: 'volumeEventsSubscriptionsChanged',
+            numberOfVolumeSubscriptions: Object.keys(this.volumesEvents).length,
+        });
+    }
+
+    private createVolumeEventManager(volumeId: string, isOwnVolume: boolean): VolumeEventManager {
+        const manager = new VolumeEventManager(this.logger, this.apiService, this.cache, volumeId, isOwnVolume);
+        for (const listener of this.listeners) {
+            manager.addListener(listener);
+        }
+        this.volumesEvents[volumeId] = manager;
+        return manager;
+    }
+
+    /**
+     * Listen to the drive events. The listener will be called with the
+     * new events as they arrive.
+     * 
+     * One call always provides events from withing the same volume. The
+     * second argument of the callback `fullRefreshVolumeId` is thus single
+     * ID and if multiple volumes must be fully refreshed, client will
+     * receive multiple calls.
+     */
+    addListener(callback: DriveListener): void {
+        // Add new listener to the list for any new event manager.
+        this.listeners.push(callback);
+
+        // Add new listener to all existings managers.
+        this.coreEvents.addListener(callback);
+        for (const volumeEvents of Object.values(this.volumesEvents)) {
+            volumeEvents.addListener(callback);
+        }
+    }
+}

package/src/internal/events/interface.ts

@@ -0,0 +1,51 @@
+/**
+ * Callback that accepts list of Drive events and flag whether no
+ * event should be processed, but rather full cache refresh should be
+ * performed.
+ * 
+ * @param fullRefreshVolumeId - ID of the volume that should be fully refreshed.
+ */
+export type DriveListener = (events: DriveEvent[], fullRefreshVolumeId?: string) => Promise<void>;
+
+/**
+ * Generic internal event interface representing a list of events
+ * with metadata about the last event ID, whether there are more
+ * events to fetch, or whether the listener should refresh its state.
+ */
+export type Events<T> = {
+    lastEventId: string,
+    more: boolean,
+    refresh: boolean,
+    events: T[],
+}
+
+/**
+ * Internal event interface representing a list of specific Drive events.
+ */
+export type DriveEvents = Events<DriveEvent>;
+
+export type DriveEvent = {
+    type: DriveEventType.NodeCreated | DriveEventType.NodeUpdated | DriveEventType.NodeUpdatedMetadata,
+    nodeUid: string,
+    parentNodeUid?: string,
+    isTrashed: boolean,
+    isShared: boolean,
+    isOwnVolume: boolean,
+} | {
+    type: DriveEventType.NodeDeleted,
+    nodeUid: string,
+    parentNodeUid?: string,
+    isTrashed?: boolean,
+    isShared?: boolean,
+    isOwnVolume: boolean,
+} | {
+    type: DriveEventType.ShareWithMeUpdated,
+}
+
+export enum DriveEventType {
+    NodeCreated = 'node_created',
+    NodeUpdated = 'node_updated',
+    NodeUpdatedMetadata = 'node_updated_metadata',
+    NodeDeleted = 'node_deleted',
+    ShareWithMeUpdated = 'share_with_me_updated',
+}

package/src/internal/events/volumeEventManager.ts

@@ -0,0 +1,74 @@
+import { Logger } from "../../interface";
+import { LoggerWithPrefix } from "../../telemetry";
+import { EventsAPIService } from "./apiService";
+import { EventsCache } from "./cache";
+import { DriveEvent, DriveListener } from "./interface";
+import { EventManager } from "./eventManager";
+
+/**
+ * Combines API and event manager to provide a service for listening to
+ * volume events. Volume events are all about nodes updates. Whenever
+ * there is update to the node metadata or content, the event is emitted.
+ */
+export class VolumeEventManager {
+    private manager: EventManager<DriveEvent>;
+
+    constructor(logger: Logger, private apiService: EventsAPIService, private cache: EventsCache, private volumeId: string, isOwnVolume: boolean) {
+        this.apiService = apiService;
+        this.volumeId = volumeId;
+
+        this.manager = new EventManager(
+            new LoggerWithPrefix(logger, `volume ${volumeId}`),
+            () => this.getLastEventId(),
+            (eventId) => this.apiService.getVolumeEvents(volumeId, eventId, isOwnVolume),
+            (lastEventId) => this.cache.setLastEventId(volumeId, {
+                lastEventId,
+                pollingIntervalInSeconds: this.manager.pollingIntervalInSeconds,
+                isOwnVolume
+            }),
+        );
+        this.cache.getPollingIntervalInSeconds(volumeId)
+            .then((pollingIntervalInSeconds) => {
+                if (pollingIntervalInSeconds) {
+                    this.manager.pollingIntervalInSeconds = pollingIntervalInSeconds;
+                }
+            })
+            .catch(() => {});
+    }
+
+    private async getLastEventId(): Promise<string> {
+        const lastEventId = await this.cache.getLastEventId(this.volumeId);
+        if (lastEventId) {
+            return lastEventId;
+        }
+        return this.apiService.getVolumeLatestEventId(this.volumeId);
+    }
+
+    /**
+     * There is a limit how many volume subscribtions can be active at
+     * the same time. The manager of all volume managers should set the
+     * intervals for each volume accordingly depending on the volume
+     * type or the total number of subscriptions.
+     */
+    setPollingInterval(pollingIntervalInSeconds: number): void {
+        this.manager.pollingIntervalInSeconds = pollingIntervalInSeconds;
+    }
+
+    async startSubscription(): Promise<void> {
+        await this.manager.start();
+    }
+
+    async stopSubscription(): Promise<void> {
+        await this.manager.stop();
+    }
+
+    addListener(callback: DriveListener): void {
+        this.manager.addListener(async (events, fullRefresh) => {
+            if (fullRefresh) {
+                await callback([], this.volumeId);
+            } else {
+                await callback(events);
+            }
+        });
+    }
+}