@aerogel/plugin-local-first 0.0.0-next.4c77ee2164e8a4f72bdd920cb56ec059299ccc5f

package/src/services/Cloud.ts

@@ -0,0 +1,450 @@
+import {
+    Semaphore,
+    after,
+    facade,
+    fail,
+    getLocationQueryParameter,
+    hasLocationQueryParameter,
+    isArray,
+    parseBoolean,
+} from '@noeldemartin/utils';
+import { App, Errors, Events, dispatch, translateWithDefault } from '@aerogel/core';
+import { Solid } from '@aerogel/plugin-solid';
+import { SolidModel, Tombstone, isContainer, isContainerClass } from 'soukai-solid';
+import { getTrackedModels, trackModels, trackModelsCollection } from '@aerogel/plugin-soukai';
+import { watchEffect } from 'vue';
+import type { Authenticator } from '@aerogel/plugin-solid';
+import type { Engine } from 'soukai';
+import type { JobListener } from '@aerogel/core';
+import type { SolidContainsRelation, SolidModelConstructor, SolidSchemaDefinition } from 'soukai-solid';
+
+import {
+    clearLocalModelUpdates,
+    createRemoteModel,
+    getLocalModels,
+    getRemoteClass,
+    getRemoteContainersCollection,
+    updateRemoteModel,
+} from '@aerogel/plugin-local-first/lib/mirroring';
+import Backup from '@aerogel/plugin-local-first/jobs/Backup';
+import Migrate from '@aerogel/plugin-local-first/jobs/Migrate';
+import Sync from '@aerogel/plugin-local-first/jobs/Sync';
+import SyncQueue from '@aerogel/plugin-local-first/lib/SyncQueue';
+import {
+    getContainedModels,
+    getContainerRelations,
+    getRelatedClasses,
+} from '@aerogel/plugin-local-first/lib/inference';
+
+import Service, { CloudStatus } from './Cloud.state';
+
+export interface RegisterOptions {
+    path?: string;
+}
+
+export interface SyncOptions extends JobListener {
+    refreshUserProfile?: boolean;
+    model?: SolidModel;
+    models?: SolidModel[];
+}
+
+export class CloudService extends Service {
+
+    protected asyncLock: Semaphore = new Semaphore();
+    protected engine: Engine | null = null;
+    protected pollingInterval: NodeJS.Timeout | null = null;
+
+    public async whenReady<T>(callback: () => T): Promise<T> {
+        if (this.ready) {
+            return callback();
+        }
+
+        return new Promise((resolve) => {
+            const onReady = () => resolve(callback());
+
+            Events.once('cloud:ready', onReady);
+        });
+    }
+
+    public async setup(modelUrlMappings?: WeakMap<SolidModelConstructor, Record<string, string>>): Promise<void> {
+        this.setupOngoing = true;
+
+        try {
+            await this.backup(modelUrlMappings);
+            await this.setReady(true);
+            await this.sync();
+        } finally {
+            this.setupOngoing = false;
+        }
+    }
+
+    public dismissSetup(): void {
+        this.setupDismissed = true;
+    }
+
+    public dismissMigration(): void {
+        this.migrationDismissed = true;
+    }
+
+    public postponeMigration(): void {
+        this.migrationPostponed = true;
+    }
+
+    public async syncIfOnline(options?: SyncOptions): Promise<void>;
+    public async syncIfOnline(model: SolidModel): Promise<void>;
+    public async syncIfOnline(models: SolidModel[]): Promise<void>;
+    public async syncIfOnline(config: SyncOptions | SolidModel | SolidModel[] = {}): Promise<void> {
+        if (!this.online) {
+            return;
+        }
+
+        await this.sync(config as SyncOptions);
+    }
+
+    public async sync(options?: SyncOptions): Promise<void>;
+    public async sync(model: SolidModel): Promise<void>;
+    public async sync(models: SolidModel[]): Promise<void>;
+    public async sync(config: SyncOptions | SolidModel | SolidModel[] = {}): Promise<void> {
+        const options = isArray(config)
+            ? { models: config }
+            : config instanceof SolidModel
+                ? { model: config }
+                : config;
+
+        if (!Solid.isLoggedIn() || !this.ready || !this.online) {
+            return;
+        }
+
+        await this.asyncLock.run(async () => {
+            const start = Date.now();
+            const models = options.model ? [options.model] : options.models;
+
+            this.syncError = null;
+            this.status = CloudStatus.Syncing;
+
+            SyncQueue.clear(models);
+
+            try {
+                await Events.emit('cloud:sync-started', models);
+
+                if (options.refreshUserProfile) {
+                    await Solid.refreshUserProfile();
+                }
+
+                this.syncJob = new Sync(models);
+
+                this.syncJob.listeners.add(options);
+
+                await dispatch(this.syncJob);
+
+                if (this.syncJob.cancelled) {
+                    return;
+                }
+
+                if (!App.testing) {
+                    await after({ milliseconds: Math.max(500, 1000 - (Date.now() - start)) });
+                }
+
+                this.syncJob = null;
+            } catch (error) {
+                if (App.testing) {
+                    throw error;
+                }
+
+                this.syncJob = null;
+                this.syncError = error;
+
+                await Errors.report(error, translateWithDefault('cloud.syncFailed', 'Sync failed'));
+            } finally {
+                this.status = CloudStatus.Online;
+
+                await Events.emit('cloud:sync-completed', models);
+            }
+        });
+    }
+
+    public shouldMigrate(): boolean {
+        return this.schemaMigrations.size > 0;
+    }
+
+    public async migrate(listener?: JobListener): Promise<void> {
+        if (!this.online || !Solid.isLoggedIn() || !this.ready) {
+            throw new Error('Data cannot be migrated at the moment');
+        }
+
+        if (!this.shouldMigrate()) {
+            return;
+        }
+
+        await this.asyncLock.run(async () => {
+            this.status = CloudStatus.Migrating;
+            this.migrationJob ??= new Migrate();
+
+            const start = Date.now();
+            const removeListeners: Function[] = [];
+            const models = (this.migrationJob.models ??= getLocalModels());
+
+            SyncQueue.clear(models);
+
+            try {
+                listener && removeListeners.push(this.migrationJob.listeners.add(listener));
+
+                await Events.emit('cloud:migration-started', models);
+                await dispatch(this.migrationJob);
+
+                if (this.migrationJob.cancelled) {
+                    await Events.emit('cloud:migration-cancelled', models);
+
+                    return;
+                }
+
+                this.migrationJob = null;
+                this.migrationDismissed = false;
+                this.migrationPostponed = false;
+
+                await after({ milliseconds: Math.max(0, 1000 - (Date.now() - start)) });
+                await Events.emit('cloud:migration-completed', models);
+            } catch (error) {
+                await Errors.report(error, translateWithDefault('cloud.migrationFailed', 'Migration failed'));
+            } finally {
+                this.status = CloudStatus.Online;
+
+                removeListeners.forEach((removeListener) => removeListener?.());
+            }
+        });
+    }
+
+    public async register(modelClass: SolidModelConstructor, options: RegisterOptions = {}): Promise<void> {
+        if (this.registeredModels.find((registered) => modelClass === registered.modelClass)) {
+            return;
+        }
+
+        const engine = this.engine;
+
+        if (engine) {
+            getRelatedClasses(modelClass).forEach((relatedClass) => getRemoteClass(relatedClass).setEngine(engine));
+        }
+
+        this.whenReady(() => {
+            if (!isContainerClass(modelClass)) {
+                return;
+            }
+
+            modelClass.collection = getRemoteContainersCollection(options.path);
+        });
+
+        await trackModels(modelClass, {
+            created: (model) => createRemoteModel(model),
+            updated: (model) => updateRemoteModel(model),
+            deleted: (model) => clearLocalModelUpdates([model]),
+        });
+
+        if (isContainerClass(modelClass)) {
+            getContainerRelations(modelClass).forEach((relation) => {
+                const relatedClass = modelClass
+                    .instance()
+                    .requireRelation<SolidContainsRelation>(relation).relatedClass;
+
+                relatedClass.on('created', (model) => createRemoteModel(model));
+                relatedClass.on('updated', (model) => updateRemoteModel(model));
+                relatedClass.on('deleted', (model) => clearLocalModelUpdates([model]));
+            });
+        }
+
+        this.registeredModels.push({ modelClass, path: options.path });
+
+        for (const collection of this.modelCollections[modelClass.modelName] ?? []) {
+            await trackModelsCollection(modelClass, collection);
+        }
+    }
+
+    public registerSchemaMigration(
+        modelClass: SolidModelConstructor,
+        schema: SolidModelConstructor | SolidSchemaDefinition,
+    ): void {
+        this.schemaMigrations.set(modelClass, schema);
+    }
+
+    public requireRemoteCollection(modelClass: SolidModelConstructor): string {
+        for (const registration of this.registeredModels) {
+            if (modelClass !== registration.modelClass) {
+                continue;
+            }
+
+            return getRemoteContainersCollection(registration.path);
+        }
+
+        throw new Error(`Failed resolving remote collection for '${modelClass.modelName}' model`);
+    }
+
+    protected override async boot(): Promise<void> {
+        await Solid.booted;
+
+        Solid.isLoggedIn() && this.login(Solid.authenticator);
+
+        Events.on('auth:login', ({ authenticator }) => this.login(authenticator));
+        Events.on('auth:logout', () => this.logout());
+        Events.once('application-ready', () => this.onApplicationReady());
+        Events.once('application-mounted', () => this.onApplicationMounted());
+
+        watchEffect(() => {
+            this.pollingInterval && clearInterval(this.pollingInterval);
+
+            if (!this.pollingEnabled || this.migrationJob) {
+                this.pollingInterval = null;
+
+                return;
+            }
+
+            this.pollingInterval = setInterval(() => this.sync(), this.pollingMinutes * 60 * 1000);
+        });
+    }
+
+    protected getDirtyLocalModels(): SolidModel[] {
+        const urls = Object.keys(this.localModelUpdates);
+        const dirtyLocalModels = [];
+
+        for (const { modelClass } of this.registeredModels) {
+            for (const model of getTrackedModels(modelClass)) {
+                dirtyLocalModels.push(...this.getModelsIn(urls, model));
+            }
+        }
+
+        return dirtyLocalModels;
+    }
+
+    protected getModelsIn(urls: string[], model: SolidModel): SolidModel[] {
+        const models = [];
+
+        if (urls.includes(model.url)) {
+            models.push(model);
+        }
+
+        if (isContainer(model)) {
+            models.push(
+                ...getContainedModels(model)
+                    .map((containedModel) => this.getModelsIn(urls, containedModel))
+                    .flat(),
+            );
+        }
+
+        return models;
+    }
+
+    protected requireEngine(): Engine {
+        return this.engine ?? fail('Could not get required Engine');
+    }
+
+    protected async setReady(ready: boolean): Promise<void> {
+        if (this.ready || !ready) {
+            return;
+        }
+
+        await Events.emit('cloud:ready');
+
+        this.ready = true;
+    }
+
+    protected async backup(modelUrlMappings?: WeakMap<SolidModelConstructor, Record<string, string>>): Promise<void> {
+        await Events.emit('cloud:backup-started');
+
+        modelUrlMappings ??= new WeakMap();
+
+        const job = new Backup();
+
+        for (const { modelClass, path } of this.registeredModels) {
+            if (!isContainerClass(modelClass)) {
+                continue;
+            }
+
+            for (const [local, remote] of Object.entries(modelUrlMappings.get(modelClass) ?? {})) {
+                job.migrateUrl(modelClass, local, remote);
+            }
+
+            const localCollection = modelClass.collection;
+            const remoteCollection = getRemoteContainersCollection(path);
+
+            modelClass.collection = remoteCollection;
+
+            job.migrateCollection(modelClass, localCollection, remoteCollection);
+        }
+
+        await dispatch(job);
+        await Events.emit('cloud:backup-completed');
+    }
+
+    protected async onApplicationReady(): Promise<void> {
+        if (this.ready || !Solid.isLoggedIn()) {
+            return;
+        }
+
+        await this.setReady(!getLocalModels().some((model) => model.url.startsWith('solid://')));
+    }
+
+    protected async onApplicationMounted(): Promise<void> {
+        if (!this.syncOnStartup()) {
+            return;
+        }
+
+        await this.sync();
+    }
+
+    protected login(authenticator: Authenticator): void {
+        const relatedClasses = [...this.registeredModels]
+            .map(({ modelClass }) => getRelatedClasses(modelClass))
+            .flat()
+            .concat([Tombstone]);
+
+        this.status = CloudStatus.Online;
+        this.engine = authenticator.engine;
+
+        for (const relatedClass of relatedClasses) {
+            getRemoteClass(relatedClass).setEngine(this.engine);
+        }
+    }
+
+    protected logout(): void {
+        this.setState({
+            localModelUpdates: {},
+            migrationDismissed: false,
+            migrationJob: null,
+            migrationPostponed: false,
+            modelCollections: {},
+            pollingEnabled: true,
+            pollingMinutes: 10,
+            ready: false,
+            setupDismissed: false,
+            setupOngoing: false,
+            startupSync: true,
+            status: CloudStatus.Disconnected,
+        });
+    }
+
+    private syncOnStartup(): boolean {
+        if (hasLocationQueryParameter('startupSync')) {
+            return parseBoolean(getLocationQueryParameter('startupSync'));
+        }
+
+        if (this.migrationJob) {
+            return false;
+        }
+
+        return this.startupSync;
+    }
+
+}
+
+export default facade(CloudService);
+
+declare module '@aerogel/core' {
+    export interface EventsPayload {
+        'cloud:ready': void;
+        'cloud:backup-started': void;
+        'cloud:backup-completed': void;
+        'cloud:migration-started': SolidModel[];
+        'cloud:migration-cancelled': SolidModel[];
+        'cloud:migration-completed': SolidModel[];
+        'cloud:sync-started': SolidModel[] | undefined;
+        'cloud:sync-completed': SolidModel[] | undefined;
+    }
+}

package/src/services/DocumentsCache.ts

@@ -0,0 +1,35 @@
+import { Events, Service } from '@aerogel/core';
+import { IndexedDBMap, facade } from '@noeldemartin/utils';
+
+export class DocumentsCacheService extends Service {
+
+    private cache = new IndexedDBMap<{ modifiedAt: number }>('documents-cache');
+
+    public async isFresh(documentUrl: string, modifiedAt: Date | number): Promise<boolean> {
+        const meta = await this.cache.get(documentUrl);
+        const modifiedAtTime = typeof modifiedAt !== 'number' ? modifiedAt.getTime() : modifiedAt;
+
+        return !!meta?.modifiedAt && meta.modifiedAt >= modifiedAtTime;
+    }
+
+    public async remember(documentUrl: string, modifiedAt: Date | number): Promise<void> {
+        await this.cache.set(documentUrl, {
+            modifiedAt: typeof modifiedAt !== 'number' ? modifiedAt.getTime() : modifiedAt,
+        });
+    }
+
+    public async forget(documentUrl: string): Promise<void> {
+        await this.cache.delete(documentUrl);
+    }
+
+    protected override async boot(): Promise<void> {
+        Events.on('auth:logout', async () => {
+            await this.cache.clear();
+
+            this.cache.close();
+        });
+    }
+
+}
+
+export default facade(DocumentsCacheService);

package/src/testing/cloud.ts

@@ -0,0 +1,193 @@
+import { App } from '@aerogel/core';
+import { bootSolidModels } from 'soukai-solid';
+import { uuid } from '@noeldemartin/utils';
+import { InMemoryEngine, bootModels, resetModelListeners, setEngine } from 'soukai';
+import { it } from 'vitest';
+import { resetTrackedModels } from '@aerogel/plugin-soukai';
+import { Solid } from '@aerogel/plugin-solid';
+import type { App as AppInstance } from 'vue';
+
+import Cloud from '@aerogel/plugin-local-first/services/Cloud';
+
+import Movie from '@aerogel/plugin-local-first/testing/stubs/Movie';
+import MoviesContainer from '@aerogel/plugin-local-first/testing/stubs/MoviesContainer';
+import SolidMock from '@aerogel/plugin-local-first/testing/mocks/Solid.mock';
+import Task from '@aerogel/plugin-local-first/testing/stubs/Task';
+import TaskSchema from '@aerogel/plugin-local-first/testing/stubs/Task.schema';
+import TasksList from '@aerogel/plugin-local-first/testing/stubs/TasksList';
+import Workspace from '@aerogel/plugin-local-first/testing/stubs/Workspace';
+import { FakeResponse, mock } from '@noeldemartin/testing';
+
+export async function setupCloudTests(): Promise<void> {
+    bootSolidModels();
+    bootModels({ Movie, MoviesContainer, Task, TasksList, Workspace });
+    setEngine(new InMemoryEngine());
+    resetTrackedModels();
+    resetModelListeners();
+
+    await Task.updateSchema(TaskSchema);
+
+    App.instance = mock<AppInstance>({ config: { globalProperties: { $cloud: Cloud } } });
+
+    Solid.mock();
+    Cloud.reset();
+}
+
+export function testVariants<T, Variants extends Record<string, T>>(
+    description: string,
+    variants: Variants,
+    test: (variantData: T, variantName: keyof Variants) => unknown,
+    options: { skip?: boolean; only?: boolean | keyof Variants } = {},
+): void {
+    const only = typeof options.only === 'string' ? [options.only] : options.only ? Object.keys(variants) : [];
+    const skip = typeof options.skip === 'string' ? [options.skip] : options.skip ? Object.keys(variants) : [];
+
+    for (const [name, data] of Object.entries(variants)) {
+        if (only.includes(name)) {
+            it.only(`[${name}] ${description}`, () => test(data, name));
+
+            continue;
+        }
+
+        if (skip.includes(name)) {
+            it.skip(`[${name}] ${description}`, () => test(data, name));
+
+            continue;
+        }
+
+        it(`[${name}] ${description}`, () => test(data, name));
+    }
+}
+
+export function testRegisterVariants(
+    description: string,
+    test: (registerModels: () => unknown, variant: 'model registration' | 'container registration') => unknown,
+    options: { skip?: boolean; only?: boolean | 'model registration' | 'container registration' } = {},
+): void {
+    testVariants(
+        description,
+        {
+            'model registration': () => Cloud.register(Movie),
+            'container registration': () => Cloud.register(MoviesContainer),
+        },
+        test,
+        options,
+    );
+}
+
+export function typeIndexResponse(containers?: Record<string, string>): Response {
+    const typeIndexUrl = SolidMock.requireUser().privateTypeIndexUrl ?? '';
+    const registrations = Object.entries(containers ?? []).map(
+        ([containerUrl, forClass]) => `
+            <#${uuid()}>
+                a solid:TypeRegistration ;
+                solid:forClass ${forClass} ;
+                solid:instanceContainer <${containerUrl}> .
+        `,
+    );
+
+    return FakeResponse.success(`
+        @prefix solid: <http://www.w3.org/ns/solid/terms#>.
+
+        <${typeIndexUrl}> a solid:TypeIndex .
+
+        ${registrations.join('\n')}
+    `);
+}
+
+export function containerResponse(
+    options: { name?: string; documentUrls?: string[]; createdAt?: Date; updatedAt?: Date; append?: string } = {},
+): Response {
+    const name = options.name ?? 'Movies';
+    const documentUrls = options.documentUrls ?? [];
+    const createdAt = options.createdAt ?? new Date();
+    const updatedAt = options.updatedAt ?? new Date();
+
+    return FakeResponse.success(`
+        @prefix dc: <http://purl.org/dc/terms/>.
+        @prefix ldp: <http://www.w3.org/ns/ldp#>.
+        @prefix posix: <http://www.w3.org/ns/posix/stat#>.
+        @prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+        @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
+        @prefix crdt: <https://vocab.noeldemartin.com/crdt/>.
+
+        <./> a ldp:Container, ldp:BasicContainer, ldp:Resource;
+            rdfs:label "${name}";
+            dc:created "2024-02-05T12:28:42Z"^^xsd:dateTime;
+            dc:modified "2024-02-05T12:28:42Z"^^xsd:dateTime;
+            ${documentUrls.length > 0 ? `ldp:contains ${documentUrls.map((url) => `<${url}>`).join(',')};` : ''}
+            posix:mtime 1707136122.
+
+        <./#metadata>
+            a crdt:Metadata ;
+            crdt:resource <./> ;
+            crdt:createdAt "${createdAt.toISOString()}"^^xsd:dateTime ;
+            crdt:updatedAt "${updatedAt.toISOString()}"^^xsd:dateTime .
+
+        ${options.append ?? ''}
+    `);
+}
+
+export function movieResponse(title: string): Response {
+    return FakeResponse.success(`
+        @prefix schema: <https://schema.org/>.
+        @prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+        @prefix crdt: <https://vocab.noeldemartin.com/crdt/> .
+
+        <#it>
+            a schema:Movie;
+            schema:datePublished "2001-07-20T00:00:00Z"^^xsd:dateTime;
+            schema:name "${title}".
+
+        <#it-metadata> a crdt:Metadata;
+            crdt:resource <#it>;
+            crdt:createdAt "2024-02-10T00:00:00Z"^^xsd:dateTime;
+            crdt:updatedAt "2024-02-10T00:00:00Z"^^xsd:dateTime.
+    `);
+}
+
+export function taskResponse(name?: string): Response {
+    return FakeResponse.success(`
+        @prefix schema: <https://schema.org/>.
+        @prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+        @prefix crdt: <https://vocab.noeldemartin.com/crdt/> .
+
+        <#it>
+            a schema:Action;
+            schema:name "${name ?? 'Task'}".
+
+        <#it-metadata> a crdt:Metadata;
+            crdt:resource <#it>;
+            crdt:createdAt "2024-02-10T00:00:00Z"^^xsd:dateTime;
+            crdt:updatedAt "2024-02-10T00:00:00Z"^^xsd:dateTime.
+    `);
+}
+
+export function legacyTaskResponse(name?: string): Response {
+    return FakeResponse.success(`
+        @prefix ical: <http://www.w3.org/2002/12/cal/ical#>.
+        @prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+        @prefix crdt: <https://vocab.noeldemartin.com/crdt/> .
+
+        <#it>
+            a ical:Vtodo;
+            ical:summary "${name ?? 'Task'}".
+
+        <#it-metadata> a crdt:Metadata;
+            crdt:resource <#it>;
+            crdt:createdAt "2024-02-10T00:00:00Z"^^xsd:dateTime;
+            crdt:updatedAt "2024-02-10T00:00:00Z"^^xsd:dateTime.
+    `);
+}
+
+export function tombstoneResponse(): Response {
+    return FakeResponse.success(`
+        @prefix crdt: <https://vocab.noeldemartin.com/crdt/> .
+        @prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+
+        <#it-metadata>
+            a crdt:Tombstone ;
+            crdt:resource <#it> ;
+            crdt:deletedAt  "2025-03-03T00:00:00Z"^^xsd:dateTime .
+    `);
+}