@aerogel/plugin-local-first 0.0.0-next.4c77ee2164e8a4f72bdd920cb56ec059299ccc5f

package/src/jobs/mixins/LoadsChildren.ts

@@ -0,0 +1,210 @@
+import { arrayChunk, arrayFrom, isInstanceOf, map, required } from '@noeldemartin/utils';
+import { DocumentNotFound } from 'soukai';
+import { MalformedSolidDocumentError } from '@noeldemartin/solid-utils';
+import { Tombstone } from 'soukai-solid';
+import type { JobStatus } from '@aerogel/core';
+import type { ObjectsMap } from '@noeldemartin/utils';
+import type { SolidContainer, SolidContainsRelation, SolidDocument, SolidModel } from 'soukai-solid';
+
+import DocumentsCache from '@aerogel/plugin-local-first/services/DocumentsCache';
+import { cloneLocalModel, isLocalModel } from '@aerogel/plugin-local-first/lib/mirroring';
+import { getContainerRelations } from '@aerogel/plugin-local-first/lib/inference';
+import { lazy } from '@aerogel/plugin-local-first/lib/utils';
+
+export interface ResourceJobStatus extends JobStatus {
+    documentUrl?: string;
+    children?: ResourceJobStatus[];
+}
+
+export default class LoadsChildren {
+
+    declare protected localModels?: ObjectsMap<SolidModel>;
+
+    protected async loadContainedModels(
+        model: SolidContainer,
+        options: {
+            ignoreTombstones?: boolean;
+            status?: ResourceJobStatus;
+            onLoaded?: (urls: string[]) => unknown;
+            onMalformedDocument?: (url: string) => unknown;
+        } = {},
+    ): Promise<SolidModel[]> {
+        if (isLocalModel(model)) {
+            const children = [];
+
+            for (const relation of getContainerRelations(model.static())) {
+                const relationModels =
+                    (await model.loadRelationIfUnloaded<SolidModel | SolidModel[] | null>(relation)) ?? [];
+
+                children.push(...arrayFrom(relationModels));
+            }
+
+            return children;
+        }
+
+        const children: Record<string, SolidModel[]> = {};
+        const tombstones: Tombstone[] = [];
+        const documents = map(model.documents, 'url');
+        const documentUrlChunks = arrayChunk(model.resourceUrls, 10);
+        const statusChildren = documentUrlChunks.flat().map((url) => ({
+            documentUrl: url,
+            completed: false,
+        }));
+        const statusChildrenMap = map(statusChildren, 'documentUrl');
+
+        if (options.status) {
+            options.status.children = statusChildren;
+        }
+
+        for (const documentUrls of documentUrlChunks) {
+            await Promise.all(
+                documentUrls.map(async (documentUrl) => {
+                    const { children: documentChildren, tombstones: documentTombstones } =
+                        await this.loadDocumentChildren(model, documentUrl, documents, options.onMalformedDocument);
+
+                    tombstones.push(...documentTombstones);
+
+                    for (const [relation, documentModels] of Object.entries(documentChildren)) {
+                        const relationChildren = (children[relation] ??= []);
+
+                        relationChildren.push(...documentModels);
+                    }
+                }),
+            );
+
+            for (const documentUrl of documentUrls) {
+                if (documentUrl.endsWith('/')) {
+                    continue;
+                }
+
+                required(statusChildrenMap.get(documentUrl)).completed = true;
+            }
+
+            await options.onLoaded?.(documentUrls);
+        }
+
+        for (const [relation, relationModels] of Object.entries(children)) {
+            model.setRelationModels(relation, relationModels);
+        }
+
+        const childrenArray = options.ignoreTombstones
+            ? Object.values(children).flat()
+            : Object.values(children).flat().concat(tombstones);
+
+        if (options.status) {
+            options.status.children = childrenArray.map((child) => {
+                return required(statusChildrenMap.get(child.requireDocumentUrl()));
+            });
+        }
+
+        return childrenArray;
+    }
+
+    protected async loadDocumentChildren(
+        model: SolidModel,
+        documentUrl: string,
+        documents: ObjectsMap<SolidDocument>,
+        onMalformedDocument?: (url: string) => unknown,
+    ): Promise<{ children: Record<string, SolidModel[]>; tombstones: Tombstone[] }> {
+        const document = documents.get(documentUrl);
+
+        if (
+            document &&
+            (await DocumentsCache.isFresh(document.url, document.updatedAt ?? new Date())) &&
+            this.localModels?.hasKey(model.url)
+        ) {
+            return {
+                children: await this.loadDocumentChildrenFromLocal(model, document),
+                tombstones: [],
+            };
+        }
+
+        return this.loadDocumentChildrenFromRemote(model, documentUrl, onMalformedDocument);
+    }
+
+    protected async loadDocumentChildrenFromLocal(
+        model: SolidModel,
+        document: SolidDocument,
+    ): Promise<Record<string, SolidModel[]>> {
+        const children: Record<string, SolidModel[]> = {};
+
+        for (const relation of getContainerRelations(model.static())) {
+            const relationInstance = model.requireRelation<SolidContainsRelation>(relation);
+
+            if (relationInstance.loaded) {
+                children[relation] = relationInstance.getLoadedModels();
+
+                continue;
+            }
+
+            const relatedLocalModels = this.localModels?.get(model.url)?.getRelationModels(relation) ?? [];
+
+            children[relation] = relatedLocalModels
+                .filter((relatedLocalModel) => relatedLocalModel.getDocumentUrl() === document.url)
+                .map((relatedLocalModel) => cloneLocalModel(relatedLocalModel, { clean: true }));
+        }
+
+        return children;
+    }
+
+    protected async loadDocumentChildrenFromRemote(
+        model: SolidModel,
+        documentUrl: string,
+        onMalformedDocument?: (url: string) => unknown,
+    ): Promise<{ children: Record<string, SolidModel[]>; tombstones: Tombstone[] }> {
+        const children: Record<string, SolidModel[]> = {};
+        const tombstones: Tombstone[] = [];
+        const readDocument = lazy(async () => {
+            try {
+                const document = await model.requireEngine().readOne(model.static('collection'), documentUrl);
+                const documentTombstones = await Tombstone.createManyFromEngineDocuments({
+                    [documentUrl]: document,
+                });
+
+                tombstones.push(...documentTombstones);
+
+                return document;
+            } catch (error) {
+                if (isInstanceOf(error, DocumentNotFound)) {
+                    return null;
+                }
+
+                if (isInstanceOf(error, MalformedSolidDocumentError)) {
+                    // eslint-disable-next-line no-console
+                    console.warn(error.message);
+
+                    await onMalformedDocument?.(documentUrl);
+
+                    return null;
+                }
+
+                throw error;
+            }
+        });
+
+        for (const relation of getContainerRelations(model.static())) {
+            const relationInstance = model.requireRelation<SolidContainsRelation>(relation);
+
+            if (relationInstance.loaded) {
+                children[relation] = relationInstance.getLoadedModels();
+
+                continue;
+            }
+
+            const document = await readDocument();
+
+            if (!document) {
+                continue;
+            }
+
+            const documentChildren = await relationInstance.relatedClass.createManyFromEngineDocuments({
+                [documentUrl]: document,
+            });
+
+            children[relation] = documentChildren;
+        }
+
+        return { children, tombstones };
+    }
+
+}

package/src/jobs/mixins/LoadsTypeIndex.ts

@@ -0,0 +1,23 @@
+import { Solid } from '@aerogel/plugin-solid';
+import type { SolidTypeIndex } from 'soukai-solid';
+import type { SolidUserProfile } from '@noeldemartin/solid-utils';
+
+const typeIndexes: WeakMap<SolidUserProfile, SolidTypeIndex | null> = new WeakMap();
+
+export default class LoadsTypeIndex {
+
+    protected async loadTypeIndex(options: { create: true }): Promise<SolidTypeIndex>;
+    protected async loadTypeIndex(options?: { create: false }): Promise<SolidTypeIndex | null>;
+    protected async loadTypeIndex(options: { create?: boolean } = {}): Promise<SolidTypeIndex | null> {
+        const user = Solid.requireUser();
+
+        if (options.create && !typeIndexes.get(user)) {
+            typeIndexes.set(user, await Solid.findOrCreatePrivateTypeIndex());
+        } else if (!typeIndexes.has(user)) {
+            typeIndexes.set(user, await Solid.findPrivateTypeIndex());
+        }
+
+        return typeIndexes.get(user) ?? null;
+    }
+
+}

package/src/jobs/mixins/TracksLocalModels.ts

@@ -0,0 +1,32 @@
+import type { ObjectsMap } from '@noeldemartin/utils';
+import type { SolidModel } from 'soukai-solid';
+
+import { getContainerRelations } from '@aerogel/plugin-local-first/lib/inference';
+
+export default class TracksLocalModels {
+
+    declare protected localModels?: ObjectsMap<SolidModel>;
+
+    protected async indexLocalModels(models: SolidModel[]): Promise<void> {
+        for (const localModel of models) {
+            await this.addLocalModel(localModel);
+        }
+    }
+
+    protected async addLocalModel(localModel: SolidModel): Promise<void> {
+        if (!this.localModels || this.localModels.hasKey(localModel.url)) {
+            return;
+        }
+
+        this.localModels.add(localModel);
+
+        for (const relation of getContainerRelations(localModel.static())) {
+            await localModel.loadRelationIfUnloaded(relation);
+        }
+
+        for (const relatedModel of localModel.getRelatedModels()) {
+            await this.addLocalModel(relatedModel);
+        }
+    }
+
+}

package/src/lib/SyncQueue.ts

@@ -0,0 +1,94 @@
+import { after, arrayRemove, facade, required } from '@noeldemartin/utils';
+import { App } from '@aerogel/core';
+import { type SolidModel, isContainer } from 'soukai-solid';
+
+import { getContainedModels } from '@aerogel/plugin-local-first/lib/inference';
+
+const Cloud = required(() => App.service('$cloud'));
+
+class SyncQueue {
+
+    private models: Set<SolidModel> = new Set();
+    private promises: Promise<unknown>[] = [];
+
+    public push(model: SolidModel): void {
+        this.models.add(model);
+        this.syncAfter(after({ seconds: 1 }));
+    }
+
+    public clear(models?: SolidModel[]): void {
+        if (!models) {
+            this.models = new Set();
+
+            return;
+        }
+
+        for (const model of models) {
+            this.models.delete(model);
+        }
+    }
+
+    protected async syncAfter<T = void>(operationOrPromise: Promise<T> | (() => Promise<T>)): Promise<T> {
+        const promise = typeof operationOrPromise === 'function' ? operationOrPromise() : operationOrPromise;
+
+        this.promises.push(promise);
+
+        promise.then(() => arrayRemove(this.promises, promise));
+
+        if (this.promises.length === 1) {
+            this.schedule();
+        }
+
+        return promise as Promise<T>;
+    }
+
+    private schedule(): void {
+        Promise.all(this.promises)
+            .then(() => after())
+            .then(() => {
+                if (this.promises.length !== 0) {
+                    this.schedule();
+
+                    return;
+                }
+
+                const models = this.consume();
+
+                if (models.length === 0) {
+                    return;
+                }
+
+                Cloud.syncIfOnline(models);
+            });
+    }
+
+    private consume(): SolidModel[] {
+        const models: SolidModel[] = [];
+        const children: Set<SolidModel> = new Set();
+
+        for (const model of this.models) {
+            if (!isContainer(model)) {
+                continue;
+            }
+
+            this.models.delete(model);
+            getContainedModels(model).forEach((child) => children.add(child));
+            models.push(model);
+        }
+
+        for (const model of this.models) {
+            this.models.delete(model);
+
+            if (children.has(model)) {
+                continue;
+            }
+
+            models.push(model);
+        }
+
+        return models;
+    }
+
+}
+
+export default facade(SyncQueue);

package/src/lib/inference.ts

@@ -0,0 +1,124 @@
+import {
+    Metadata,
+    Operation,
+    SolidACLAuthorization,
+    SolidContainer,
+    SolidContainsRelation,
+    SolidDocument,
+    SolidModel,
+    isContainerClass,
+    isSolidDocumentRelation,
+} from 'soukai-solid';
+import { arrayFilter, isInstanceOf } from '@noeldemartin/utils';
+
+import type { SolidContainerConstructor, SolidModelConstructor } from 'soukai-solid';
+
+const containerRelations: WeakMap<typeof SolidModel, string[]> = new WeakMap();
+const sameDocumentRelations: WeakMap<typeof SolidModel, string[]> = new WeakMap();
+
+export function getContainerRegisteredClasses(modelClass: SolidContainerConstructor): SolidModelConstructor[] {
+    return arrayFilter(
+        getContainerRelations(modelClass).map((relation) => {
+            const relatedClass = modelClass.instance().requireRelation<SolidContainsRelation>(relation).relatedClass;
+
+            if (isContainerClass(relatedClass)) {
+                return false;
+            }
+
+            return relatedClass;
+        }),
+    );
+}
+
+export function getContainerRelations(modelClass: typeof SolidModel): string[] {
+    if (!containerRelations.has(modelClass)) {
+        containerRelations.set(
+            modelClass,
+            modelClass.relations.filter((relation) => {
+                if (SolidModel.reservedRelations.includes(relation)) {
+                    return false;
+                }
+
+                const relationInstance = modelClass.instance().requireRelation(relation);
+
+                return relationInstance instanceof SolidContainsRelation;
+            }),
+        );
+    }
+
+    return containerRelations.get(modelClass) ?? [];
+}
+
+export function getContainedModels(model: SolidContainer): SolidModel[] {
+    const relations = getContainerRelations(model.static());
+
+    return relations.reduce(
+        (models, relation) => models.concat(model.getRelationModels(relation) ?? []),
+        [] as SolidModel[],
+    );
+}
+
+export function getRelatedClasses(modelClass: SolidModelConstructor): SolidModelConstructor[] {
+    const relatedClasses = new Set<SolidModelConstructor>();
+    const instance = modelClass.instance();
+
+    relatedClasses.add(modelClass);
+
+    for (const relation of modelClass.relations) {
+        if (modelClass.reservedRelations.includes(relation)) {
+            continue;
+        }
+
+        relatedClasses.add(instance.requireRelation(relation).relatedClass);
+    }
+
+    return [...relatedClasses.values()];
+}
+
+export function getRelatedAppModels(model: SolidModel): SolidModel[] {
+    return model
+        .getRelatedModels()
+        .filter(
+            (related) =>
+                !isInstanceOf(related, SolidACLAuthorization) &&
+                !isInstanceOf(related, SolidDocument) &&
+                !isInstanceOf(related, Metadata) &&
+                !isInstanceOf(related, Operation) &&
+                related.static() !== SolidContainer,
+        );
+}
+
+export function getSameDocumentRelations(modelClass: typeof SolidModel): string[] {
+    if (!sameDocumentRelations.has(modelClass)) {
+        sameDocumentRelations.set(
+            modelClass,
+            modelClass.relations.filter((relation) => {
+                if (SolidModel.reservedRelations.includes(relation)) {
+                    return false;
+                }
+
+                const relationInstance = modelClass.instance().requireRelation(relation);
+
+                return isSolidDocumentRelation(relationInstance) && relationInstance.useSameDocument;
+            }),
+        );
+    }
+
+    return sameDocumentRelations.get(modelClass) ?? [];
+}
+
+export function isDirtyOrHasDirtyChildren(model: SolidModel): boolean {
+    if (model.isDirty()) {
+        return true;
+    }
+
+    if (!(model instanceof SolidContainer)) {
+        return false;
+    }
+
+    const relationHasDirtyModels = (relation: string) => {
+        return model.getRelationModels(relation)?.some((relatedModel) => isDirtyOrHasDirtyChildren(relatedModel));
+    };
+
+    return getContainerRelations(model.static()).some(relationHasDirtyModels);
+}