@aerogel/plugin-local-first 0.0.0

package/src/jobs/Backup.ts

@@ -0,0 +1,98 @@
+import { Job } from '@aerogel/core';
+import { requireEngine } from 'soukai';
+import { requireUrlParentDirectory } from '@noeldemartin/utils';
+import { ignoreModelsCollection, trackModelsCollection } from '@aerogel/plugin-soukai';
+import type { EngineDocument, IndexedDBEngine } from 'soukai';
+import type { SolidModelConstructor } from 'soukai-solid';
+
+interface Migration {
+    modelClass: SolidModelConstructor;
+    local: string;
+    remote: string;
+}
+
+export default class Backup extends Job {
+
+    private urlMigrations: Migration[] = [];
+    private collectionMigrations: Migration[] = [];
+
+    public migrateUrl(modelClass: SolidModelConstructor, local: string, remote: string): void {
+        this.urlMigrations.push({
+            modelClass,
+            local,
+            remote,
+        });
+    }
+
+    public migrateCollection(modelClass: SolidModelConstructor, local: string, remote: string): void {
+        this.collectionMigrations.push({
+            modelClass,
+            local,
+            remote,
+        });
+    }
+
+    protected async run(): Promise<void> {
+        const engine = requireEngine<IndexedDBEngine>();
+        const collections = await engine.getCollections();
+
+        for (const collection of collections) {
+            const documents = await engine.readMany(collection);
+
+            for (const [id, document] of Object.entries(documents)) {
+                const url = this.replaceUrl(id);
+
+                await engine.create(requireUrlParentDirectory(url), this.replaceDocumentUrls(document), url);
+                await engine.delete(collection, id);
+            }
+        }
+
+        for (const { modelClass, remote } of this.urlMigrations) {
+            trackModelsCollection(modelClass, requireUrlParentDirectory(remote), { refresh: false });
+        }
+
+        for (const { modelClass, local, remote } of this.collectionMigrations) {
+            ignoreModelsCollection(modelClass, local);
+            trackModelsCollection(modelClass, remote, { refresh: false });
+        }
+    }
+
+    protected replaceDocumentUrls(document: EngineDocument): EngineDocument {
+        for (const [key, value] of Object.entries(document)) {
+            if (key === '@id' && typeof value === 'string') {
+                document[key] = this.replaceUrl(value);
+
+                continue;
+            }
+
+            if (typeof value !== 'object' || value === null) {
+                continue;
+            }
+
+            document[key] = this.replaceDocumentUrls(value as EngineDocument);
+        }
+
+        return document;
+    }
+
+    protected replaceUrl(url: string): string {
+        for (const { local, remote } of this.urlMigrations) {
+            if (!url.startsWith(local)) {
+                continue;
+            }
+
+            return url.replace(local, remote);
+        }
+
+        for (const { local, remote } of this.collectionMigrations) {
+            if (!url.startsWith(local)) {
+                continue;
+            }
+
+            return url.replace(local, remote);
+        }
+
+        return url;
+    }
+
+}

package/src/jobs/Migrate.test.ts

@@ -0,0 +1,217 @@
+import { beforeEach, describe, expect, it } from 'vitest';
+import { Events } from '@aerogel/core';
+import { FakeServer, fakeContainerUrl, fakeDocumentUrl } from '@noeldemartin/testing';
+import { range } from '@noeldemartin/utils';
+import { Solid } from '@aerogel/plugin-solid';
+
+import Cloud from '@aerogel/plugin-local-first/services/Cloud';
+import { getRemoteClass } from '@aerogel/plugin-local-first/lib/mirroring';
+
+import Workspace from '@aerogel/plugin-local-first/testing/stubs/Workspace';
+import SolidMock from '@aerogel/plugin-local-first/testing/mocks/Solid.mock';
+import LegacyTaskSchema from '@aerogel/plugin-local-first/testing/stubs/LegacyTask.schema';
+import TaskSchema from '@aerogel/plugin-local-first/testing/stubs/Task.schema';
+import Task from '@aerogel/plugin-local-first/testing/stubs/Task';
+import { legacyTaskResponse, setupCloudTests, typeIndexResponse } from '@aerogel/plugin-local-first/testing/cloud';
+
+import Migrate from './Migrate';
+
+describe('Migrate', () => {
+
+    beforeEach(setupCloudTests);
+
+    it('Migrates containers', async () => {
+        // Arrange
+        const updates: number[] = [];
+        const {
+            workspace,
+            typeIndexUrl,
+            documentUrls: [documentUrl],
+        } = await setupMigration(1);
+
+        // Act
+        await Cloud.migrate({ onUpdated: (progress) => updates.push(progress) });
+
+        // Assert - Progress
+        expect(updates).toEqual([0, 1]);
+
+        // Assert - Local models
+        expect(Task.rdfsClasses).toEqual(['https://schema.org/Action']);
+
+        const freshWorkspace = await workspace.fresh();
+
+        await freshWorkspace.loadRelation('tasks');
+
+        expect(freshWorkspace.tasks).toHaveLength(1);
+
+        // Assert - Remote models
+        expect(getRemoteClass(Task).rdfsClasses).toEqual(['https://schema.org/Action']);
+
+        expect(FakeServer.getRequests(documentUrl)).toHaveLength(3);
+        expect(FakeServer.getRequests(typeIndexUrl)).toHaveLength(3);
+        expect(FakeServer.getRequests()).toHaveLength(6);
+
+        expect(FakeServer.getRequest('PATCH', documentUrl)?.body).toEqualSparql(`
+            DELETE DATA {
+                @prefix ical: <http://www.w3.org/2002/12/cal/ical#>.
+
+                <#it>
+                    a ical:Vtodo ;
+                    ical:summary "Migrate schemas" .
+            } ;
+
+            INSERT DATA {
+                @prefix schema: <https://schema.org/>.
+
+                <#it>
+                    a schema:Action ;
+                    schema:name "Migrate schemas" .
+            } .
+        `);
+
+        expect(FakeServer.getRequest('PATCH', typeIndexUrl)?.body).toEqualSparql(`
+            DELETE DATA {
+                @prefix solid: <http://www.w3.org/ns/solid/terms#>.
+                @prefix ical: <http://www.w3.org/2002/12/cal/ical#>.
+
+                <#[[registration][.*]]> solid:forClass ical:Vtodo .
+            } ;
+
+            INSERT DATA {
+                @prefix solid: <http://www.w3.org/ns/solid/terms#>.
+                @prefix schema: <https://schema.org/>.
+
+                <#[[registration][.*]]> solid:forClass schema:Action .
+            } .
+        `);
+    });
+
+    it('Tracks progress', async () => {
+        // Arrange
+        const updates: number[] = [];
+
+        await setupMigration(5);
+
+        // Act
+        await Cloud.migrate({ onUpdated: (progress) => updates.push(progress) });
+
+        // Assert
+        expect(updates).toEqual([0, 0.2, 0.4, 0.6, 0.8, 1]);
+    });
+
+    it('Resumes after cancellation', async () => {
+        // Arrange
+        const updates: number[] = [];
+
+        await setupMigration(5);
+
+        // Act - Cancel
+        await Cloud.migrate({
+            onUpdated: (progress) => {
+                updates.push(progress);
+
+                if (progress > 0.5) {
+                    Cloud.migrationJob?.cancel();
+                }
+            },
+        });
+
+        // Assert - Cancel
+        const job = Cloud.migrationJob;
+
+        expect(job?.cancelled).toBe(true);
+        expect(updates).toEqual([0, 0.2, 0.4, 0.6]);
+
+        // Act - Resume
+        await Cloud.migrate({ onUpdated: (progress) => updates.push(progress) });
+
+        // Assert - Resume
+        expect(job?.cancelled).toBe(false);
+        expect(Cloud.migrationJob).toBe(null);
+        expect(updates).toEqual([0, 0.2, 0.4, 0.6, 0.6, 0.8, 1]);
+    });
+
+    it('Resumes after going through serialization', async () => {
+        // Arrange
+        const updates: number[] = [];
+
+        await setupMigration(5);
+
+        // Act - Cancel and serialize
+        await Cloud.migrate({
+            onUpdated: (progress) => {
+                updates.push(progress);
+
+                if (progress > 0.5) {
+                    Cloud.migrationJob?.cancel();
+                }
+            },
+        });
+
+        Cloud.migrationJob = Migrate.restore(JSON.parse(JSON.stringify(Cloud.migrationJob?.serialize())));
+
+        // Assert - Cancel and serialize
+        expect(Cloud.migrationJob?.models).toBeUndefined();
+        expect(updates).toEqual([0, 0.2, 0.4, 0.6]);
+
+        // Act - Resume
+        await Cloud.migrate({
+            onUpdated: (progress) => updates.push(progress),
+        });
+
+        // Assert - Resume
+        expect(Cloud.migrationJob).toBe(null);
+        expect(updates).toEqual([0, 0.2, 0.4, 0.6, 0.6, 0.8, 1]);
+    });
+
+});
+
+async function setupMigration(count: number = 1) {
+    // Arrange - Mint urls
+    const parentContainerUrl = Solid.requireUser().storageUrls[0];
+    const containerUrl = fakeContainerUrl({ baseUrl: parentContainerUrl });
+    const documentUrls = range(count).map(() => fakeDocumentUrl({ containerUrl })) as [string, ...string[]];
+
+    // Arrange - Prepare models
+    await Task.updateSchema(LegacyTaskSchema);
+
+    const workspace = await Workspace.at(parentContainerUrl).create({
+        url: containerUrl,
+        name: 'Tasks',
+    });
+
+    for (const documentUrl of documentUrls) {
+        await workspace.relatedTasks.create({
+            url: `${documentUrl}#it`,
+            name: 'Migrate schemas',
+        });
+    }
+
+    // Arrange - Prepare responses
+    const typeIndexUrl = SolidMock.requireUser().privateTypeIndexUrl ?? '';
+    const typeIndex = typeIndexResponse({ [containerUrl]: '<http://www.w3.org/2002/12/cal/ical#Vtodo>' });
+
+    FakeServer.respondOnce(typeIndexUrl, typeIndex);
+    documentUrls.forEach((documentUrl) => {
+        FakeServer.respondOnce(documentUrl, legacyTaskResponse('Migrate schemas'));
+        FakeServer.respondOnce(documentUrl, legacyTaskResponse('Migrate schemas'));
+        FakeServer.respondOnce(documentUrl);
+    });
+    FakeServer.respondOnce(typeIndexUrl, typeIndex);
+    FakeServer.respondOnce(typeIndexUrl);
+
+    // Arrange - Prepare service
+    Cloud.ready = true;
+
+    Cloud.registerSchemaMigration(Task, TaskSchema);
+
+    await Cloud.launch();
+    await Cloud.register(Workspace);
+    await Events.emit('application-ready');
+
+    return {
+        workspace,
+        typeIndexUrl,
+        documentUrls,
+    };
+}

package/src/jobs/Migrate.ts

@@ -0,0 +1,179 @@
+import { PropertyOperation, isContainer } from 'soukai-solid';
+import { Job } from '@aerogel/core';
+import { isInstanceOf, map, mixed, objectMap, tap } from '@noeldemartin/utils';
+import type { JobListener, JobStatus } from '@aerogel/core';
+import type { Nullable } from '@noeldemartin/utils';
+import type { SolidModel, SolidTypeIndex } from 'soukai-solid';
+
+import {
+    clearSchemaMigrations,
+    getRemoteClass,
+    getSchemaMigration,
+    getSchemaMigrations,
+} from '@aerogel/plugin-local-first/lib/mirroring';
+import { getContainerRegisteredClasses } from '@aerogel/plugin-local-first/lib/inference';
+
+import LoadsChildren from './mixins/LoadsChildren';
+import LoadsTypeIndex from './mixins/LoadsTypeIndex';
+import TracksLocalModels from './mixins/TracksLocalModels';
+
+const BaseJob = Job as typeof Job<JobListener, MigrateJobStatus, MigrateJobStatus>;
+
+export interface MigrateJobStatus extends JobStatus {
+    children?: MigrateModelJobStatus[];
+}
+
+export interface MigrateModelJobStatus extends JobStatus {
+    modelUrl: string;
+    children?: MigrateModelJobStatus[];
+}
+
+export default class Migrate extends mixed(BaseJob, [LoadsChildren, LoadsTypeIndex, TracksLocalModels]) {
+
+    public static restore(status: MigrateJobStatus): Migrate {
+        return tap(new Migrate(), (job) => {
+            job.status = status;
+        });
+    }
+
+    protected _models?: SolidModel[];
+
+    public get models(): SolidModel[] | undefined {
+        return this._models;
+    }
+
+    public set models(models: SolidModel[] | undefined) {
+        if (!models) {
+            delete this._models;
+            delete this.localModels;
+
+            return;
+        }
+
+        const childrenStatus = objectMap(this.status.children ?? [], 'modelUrl');
+
+        this._models = models;
+        this.localModels = map([], 'url');
+        this.status.children = models.map(
+            (model) =>
+                childrenStatus[model.url] ?? {
+                    modelUrl: model.url,
+                    completed: false,
+                },
+        );
+    }
+
+    protected async run(): Promise<void> {
+        if (!this.models) {
+            throw new Error('Cant start migration without models');
+        }
+
+        await this.indexLocalModels(this.models);
+
+        const modelStatuses = this.status.children ?? [];
+        const typeIndex = await this.loadTypeIndex();
+
+        for (const modelStatus of modelStatuses) {
+            const model = this.models.find(({ url }) => modelStatus.modelUrl === url);
+
+            if (!model || modelStatus.completed) {
+                continue;
+            }
+
+            await this.migrateRegisteredModel(model, modelStatus, typeIndex);
+            await this.updateProgress(() => (modelStatus.completed = true));
+        }
+
+        for (const [modelClass, schema] of getSchemaMigrations()) {
+            await modelClass.updateSchema(schema);
+        }
+
+        clearSchemaMigrations();
+    }
+
+    protected async migrateRegisteredModel(
+        model: SolidModel,
+        status: MigrateModelJobStatus,
+        typeIndex?: Nullable<SolidTypeIndex>,
+    ): Promise<void> {
+        await this.migrate(model, status);
+
+        if (!isContainer(model)) {
+            return;
+        }
+
+        const registeredClasses = getContainerRegisteredClasses(model.static());
+        const registration = typeIndex?.registrations.find(({ instanceContainer }) => instanceContainer === model.url);
+
+        if (!typeIndex || !registration || registeredClasses.length === 0) {
+            return;
+        }
+
+        registration.setAttribute(
+            'forClass',
+            registeredClasses
+                .map((modelClass) => getSchemaMigration(modelClass)?.rdfsClasses ?? modelClass.rdfsClass)
+                .flat(),
+        );
+
+        await typeIndex.save();
+    }
+
+    protected async migrate(localModel: SolidModel, status: MigrateModelJobStatus): Promise<void> {
+        this.assertNotCancelled();
+
+        await this.migrateChildren(localModel, status);
+
+        const schema = getSchemaMigration(localModel.static());
+
+        if (!schema) {
+            return;
+        }
+
+        const remoteModel = await getRemoteClass(localModel.static()).find(localModel.url);
+        const migratedModel = await remoteModel?.migrateSchema(schema);
+
+        await localModel.migrateSchema(schema, {
+            mintOperationUrl: (operation) => {
+                return migratedModel?.operations?.find((migratedOperation) => {
+                    if (
+                        !isInstanceOf(operation, PropertyOperation) ||
+                        !isInstanceOf(migratedOperation, PropertyOperation)
+                    ) {
+                        return false;
+                    }
+
+                    return (
+                        operation.property === migratedOperation.property &&
+                        operation.date.getTime() === migratedOperation.date.getTime()
+                    );
+                })?.url;
+            },
+        });
+    }
+
+    protected async migrateChildren(localModel: SolidModel, status: MigrateModelJobStatus): Promise<void> {
+        if (!isContainer(localModel)) {
+            return;
+        }
+
+        const children = await this.loadContainedModels(localModel, { ignoreTombstones: true });
+
+        status.children ??= children.map((child) => ({
+            modelUrl: child.url,
+            completed: false,
+        }));
+
+        for (const childStatus of status.children) {
+            const child = children.find(({ url }) => childStatus.modelUrl === url);
+
+            if (!child || childStatus.completed) {
+                continue;
+            }
+
+            await this.migrate(child, childStatus);
+            await this.updateProgress(() => (childStatus.completed = true));
+        }
+    }
+
+}