@agentshouse/mdruntime 0.1.0-cacb4c3f6734e → 0.1.0-caee7332d2235

package/dist/overlay.js

@@ -1,4 +1,20 @@
-import { apply, parse, serialize } from '@agentshouse/mdmodel';
+import { apply, frontmatterValue, parse, serialize, } from '@agentshouse/mdmodel';
+import { differences, drafted, patching, settling } from './shared.js';
+import { ROW_WEIGHT } from './turn.js';
+export function acceptedBinaries(files) {
+    return Object.fromEntries(Object.entries(files).map(([path, file]) => [path, { state: 'accepted', ...file }]));
+}
+export function sameBinary(left, right) {
+    if (left === undefined || right === undefined)
+        return left === right;
+    return (left.state === right.state &&
+        left.byteLength === right.byteLength &&
+        left.mediaType === right.mediaType &&
+        left.sha256 === right.sha256 &&
+        (left.state === 'accepted'
+            ? right.state === 'accepted' && left.fileRef === right.fileRef
+            : right.state === 'pending' && left.localId === right.localId));
+}
 export function sameChange(left, right) {
     if (left === undefined || right === undefined)
         return left === right;
@@ -8,10 +24,13 @@ export function sameChange(left, right) {
         left.author?.name === right.author?.name);
 }
 const EMPTY_FRONTMATTER = '---\n---\n';
+function isPart(change) {
+    return change.kind === 'field' || change.kind === 'section' || change.kind === 'preamble';
+}
 function at(path, name) {
     return `${path} ${name}`;
 }
-function priorsOf(preconditions) {
+function* priorsOf(preconditions) {
     const fields = new Map();
     const sections = new Map();
     const preambles = new Map();
@@ -25,6 +44,7 @@ function priorsOf(preconditions) {
         if (precondition.kind === 'preamble') {
             preambles.set(precondition.path, precondition.value);
         }
+        yield ROW_WEIGHT;
     }
     return {
         field: (path, name) => fields.get(at(path, name)),
@@ -40,49 +60,50 @@ function compose(fields, sections) {
     const opened = parse(skeleton);
     if (!opened.ok)
         return skeleton;
-    let document = opened.document;
-    for (const [name, value] of Object.entries(fields)) {
-        const next = apply(document, { kind: 'field', name, value, prior: undefined });
-        if (next.outcome === 'applied')
-            document = next.document;
-    }
-    return serialize(document);
+    const next = apply(opened.document, Object.entries(fields).flatMap(([name, value]) => frontmatterValue(value) ? [{ kind: 'field', name, value, prior: undefined }] : []));
+    return serialize(next.outcome === 'applied' ? next.document : opened.document);
 }
-function edited(document, change, priors) {
-    if (change.kind === 'field') {
-        return apply(document, {
+function edited(change, priors) {
+    if (change.kind === 'field')
+        return {
             kind: 'field',
             name: change.name,
             value: change.value,
             prior: priors.field(change.path, change.name),
-        });
-    }
-    if (change.kind === 'preamble') {
-        return apply(document, {
-            kind: 'preamble',
-            value: change.value,
-            prior: priors.preamble(change.path),
-        });
-    }
-    if (change.kind !== 'section')
-        return null;
-    return apply(document, {
+        };
+    if (change.kind === 'preamble')
+        return { kind: 'preamble', value: change.value, prior: priors.preamble(change.path) };
+    return {
         kind: 'section',
         name: change.name,
         value: change.value,
         prior: priors.section(change.path, change.name),
-    });
+    };
 }
-function edit(staged, change, priors) {
-    const text = staged.documents[change.path];
+function edit(staged, changes, priors) {
+    const path = changes[0].path;
+    delete staged.revisions[path];
+    const text = staged.documents[path];
     if (text === undefined)
-        return;
-    const parsed = parse(text, change.path);
+        return changes.length;
+    const parsed = parse(text, path);
     if (!parsed.ok)
-        return;
-    const next = edited(parsed.document, change, priors);
-    if (next?.outcome === 'applied')
-        staged.documents[change.path] = serialize(next.document);
+        return changes.length;
+    const wanted = changes.map((change) => edited(change, priors));
+    const first = apply(parsed.document, wanted);
+    if (first.outcome === 'refuse')
+        return changes.length;
+    if (first.outcome === 'applied') {
+        staged.documents[path] = serialize(first.document);
+        return 0;
+    }
+    const stale = new Set(first.stale);
+    const kept = wanted.filter((_, at) => !stale.has(at));
+    const rest = apply(parsed.document, kept);
+    if (rest.outcome !== 'applied')
+        return changes.length;
+    staged.documents[path] = serialize(rest.document);
+    return first.stale.length;
 }
 function relocate(staged, from, to) {
     const text = staged.documents[from];
@@ -90,194 +111,255 @@ function relocate(staged, from, to) {
         delete staged.documents[from];
         staged.documents[to] = text;
     }
-    else if (staged.binaryTargets.delete(from)) {
-        staged.binaryTargets.add(to);
+    else if (staged.binaries[from] !== undefined) {
+        staged.binaries[to] = staged.binaries[from];
+        delete staged.binaries[from];
     }
     else {
-        return;
+        return false;
     }
     moveHead(staged, from, to);
+    return true;
 }
 function moveHead(heads, from, to) {
     const revision = heads.revisions[from];
+    const fileId = heads.fileIds[from];
     const change = heads.changes[from];
     delete heads.revisions[from];
+    delete heads.fileIds[from];
     delete heads.changes[from];
+    delete heads.fileIds[to];
     delete heads.changes[to];
     if (revision !== undefined)
         heads.revisions[to] = revision;
+    if (fileId !== undefined)
+        heads.fileIds[to] = fileId;
     if (change !== undefined)
         heads.changes[to] = change;
 }
 function erase(staged, path) {
     delete staged.documents[path];
     delete staged.revisions[path];
+    delete staged.fileIds[path];
     delete staged.changes[path];
-    staged.binaryTargets.delete(path);
+    delete staged.binaries[path];
 }
-function stage(staged, change, priors) {
-    if (change.kind === 'field' || change.kind === 'section' || change.kind === 'preamble') {
-        edit(staged, change, priors);
-        return;
-    }
+function stage(staged, change) {
     if (change.kind === 'create') {
         staged.documents[change.path] =
             'content' in change ? change.content : compose(change.fields, change.sections);
+        delete staged.fileIds[change.path];
         delete staged.changes[change.path];
-        return;
+        return true;
     }
     if (change.kind === 'replace') {
+        delete staged.revisions[change.path];
         if (staged.documents[change.path] === undefined)
-            return;
+            return false;
         staged.documents[change.path] = change.content;
-        return;
+        return true;
+    }
+    if (change.kind === 'binary') {
+        if (staged.documents[change.path] !== undefined)
+            return false;
+        staged.binaries[change.path] = {
+            state: 'pending',
+            localId: change.localId,
+            byteLength: change.byteLength,
+            mediaType: change.mediaType,
+            sha256: change.sha256,
+        };
+        delete staged.revisions[change.path];
+        delete staged.changes[change.path];
+        return true;
     }
     if (change.kind === 'move') {
-        relocate(staged, change.path, change.to);
-        return;
+        return relocate(staged, change.path, change.to);
     }
     erase(staged, change.path);
+    return true;
 }
-export function overlay(base, queue) {
+export function* overlaying(base, queue, unapplied) {
     const staged = {
-        documents: { ...base.documents },
-        binaryTargets: new Set(base.binaryTargets),
-        revisions: { ...base.revisions },
-        changes: { ...base.changes },
+        documents: drafted(base.documents),
+        binaries: drafted(base.binaries),
+        revisions: drafted(base.revisions),
+        fileIds: drafted(base.fileIds),
+        changes: drafted(base.changes),
     };
     for (const batch of queue) {
-        const priors = priorsOf(batch.preconditions);
-        for (const change of batch.changes)
-            stage(staged, change, priors);
+        const priors = yield* priorsOf(batch.preconditions);
+        for (let index = 0; index < batch.changes.length; index += 1) {
+            const change = batch.changes[index];
+            if (!isPart(change)) {
+                if (!stage(staged, change))
+                    unapplied?.();
+                yield ROW_WEIGHT;
+                continue;
+            }
+            const run = [change];
+            while (index + 1 < batch.changes.length) {
+                const next = batch.changes[index + 1];
+                if (!isPart(next) || next.path !== change.path)
+                    break;
+                run.push(next);
+                index += 1;
+            }
+            const text = staged.documents[change.path] ?? '';
+            for (let left = edit(staged, run, priors); left > 0; left -= 1)
+                unapplied?.();
+            yield ROW_WEIGHT + text.length;
+        }
     }
     return {
-        documents: staged.documents,
-        binaryTargets: [...staged.binaryTargets].sort(),
-        revisions: staged.revisions,
-        changes: staged.changes,
+        documents: yield* settling(staged.documents),
+        binaries: yield* settling(staged.binaries),
+        revisions: yield* settling(staged.revisions),
+        fileIds: yield* settling(staged.fileIds),
+        changes: yield* settling(staged.changes),
     };
 }
-function pathsOf(batch) {
-    return batch.changes.flatMap((change) => change.kind === 'move' ? [change.path, change.to] : [change.path]);
+function* pathsOf(batch) {
+    const paths = new Set();
+    for (const change of batch.changes) {
+        paths.add(change.path);
+        if (change.kind === 'move')
+            paths.add(change.to);
+        yield ROW_WEIGHT;
+    }
+    return paths;
 }
-export function sameLayerText(left, right) {
-    const paths = Object.keys(left.documents);
-    return (paths.length === Object.keys(right.documents).length &&
-        paths.every((path) => left.documents[path] === right.documents[path]) &&
-        left.binaryTargets.length === right.binaryTargets.length &&
-        left.binaryTargets.every((path) => right.binaryTargets.includes(path)));
+export function* sameText(left, right) {
+    return ((yield* differences(left.documents, right.documents)).size === 0 &&
+        (yield* differences(left.binaries, right.binaries, sameBinary)).size === 0);
 }
-function replayHeads(base, queue) {
-    const heads = { revisions: { ...base.revisions }, changes: { ...base.changes } };
-    for (const batch of queue) {
-        for (const change of batch.changes) {
-            if (change.kind === 'remove') {
-                delete heads.revisions[change.path];
-                delete heads.changes[change.path];
-            }
-            else if (change.kind === 'create') {
-                delete heads.changes[change.path];
-            }
-            else if (change.kind === 'move') {
-                moveHead(heads, change.path, change.to);
-            }
+export function* changedPaths(left, right) {
+    return new Set([
+        ...(yield* differences(left.documents, right.documents)),
+        ...(yield* differences(left.revisions, right.revisions)),
+        ...(yield* differences(left.fileIds, right.fileIds)),
+        ...(yield* differences(left.changes, right.changes, sameChange)),
+        ...(yield* differences(left.binaries, right.binaries, sameBinary)),
+    ]);
+}
+function* sameItems(left, right) {
+    if (left.length !== right.length)
+        return false;
+    for (let index = 0; index < left.length; index += 1) {
+        const item = JSON.stringify(left[index]);
+        if (item !== JSON.stringify(right[index]))
+            return false;
+        yield ROW_WEIGHT + item.length;
+    }
+    return true;
+}
+function* sameBatch(left, right) {
+    if (left === right)
+        return true;
+    const { changes, preconditions, ...rest } = left;
+    const { changes: others, preconditions: priors, ...held } = right;
+    return (JSON.stringify(rest) === JSON.stringify(held) &&
+        (yield* sameItems(changes, others)) &&
+        (yield* sameItems(preconditions, priors)));
+}
+function* replayHeads(base, batch) {
+    const heads = {
+        revisions: drafted(base.revisions),
+        fileIds: drafted(base.fileIds),
+        changes: drafted(base.changes),
+    };
+    for (const change of batch.changes) {
+        if (change.kind === 'remove') {
+            delete heads.revisions[change.path];
+            delete heads.fileIds[change.path];
+            delete heads.changes[change.path];
         }
+        else if (change.kind === 'create') {
+            delete heads.fileIds[change.path];
+            delete heads.changes[change.path];
+        }
+        else if (change.kind === 'replace' ||
+            change.kind === 'field' ||
+            change.kind === 'section' ||
+            change.kind === 'preamble') {
+            delete heads.revisions[change.path];
+        }
+        else if (change.kind === 'binary') {
+            delete heads.revisions[change.path];
+            delete heads.changes[change.path];
+        }
+        else if (change.kind === 'move') {
+            moveHead(heads, change.path, change.to);
+        }
+        yield ROW_WEIGHT;
     }
-    return heads;
+    return {
+        revisions: yield* settling(heads.revisions),
+        fileIds: yield* settling(heads.fileIds),
+        changes: yield* settling(heads.changes),
+    };
 }
-export function overlayChanged(base, queue, previousBase, previousQueue, previousVisible) {
-    const before = previousQueue.map((batch) => JSON.stringify(batch));
-    const after = queue.map((batch) => JSON.stringify(batch));
-    if (before.length === after.length &&
-        before.every((id, index) => id === after[index]) &&
-        sameLayerText(base, previousBase)) {
+function* repatched(held, paths, value) {
+    const set = [];
+    const removed = [];
+    for (const path of paths) {
+        const found = value(path);
+        if (found === undefined)
+            removed.push(path);
+        else
+            set.push([path, found]);
+        yield ROW_WEIGHT;
+    }
+    return yield* patching(held, set, removed);
+}
+export function* overlayChanged(base, batch, previousBase, previousBatch, previousVisible) {
+    const same = yield* sameBatch(batch, previousBatch);
+    if (same && (yield* sameText(base, previousBase))) {
         return {
             documents: previousVisible.documents,
-            binaryTargets: previousVisible.binaryTargets,
-            ...replayHeads(base, queue),
+            binaries: previousVisible.binaries,
+            ...(yield* replayHeads(base, batch)),
         };
     }
-    const affected = new Set();
-    const paths = new Set([
-        ...Object.keys(base.documents),
-        ...Object.keys(previousBase.documents),
-        ...Object.keys(base.revisions),
-        ...Object.keys(previousBase.revisions),
-        ...Object.keys(base.changes),
-        ...Object.keys(previousBase.changes),
-        ...base.binaryTargets,
-        ...previousBase.binaryTargets,
-    ]);
-    for (const path of paths) {
-        if (base.documents[path] !== previousBase.documents[path] ||
-            base.revisions[path] !== previousBase.revisions[path] ||
-            !sameChange(base.changes[path], previousBase.changes[path]) ||
-            base.binaryTargets.includes(path) !== previousBase.binaryTargets.includes(path))
+    const affected = new Set(yield* changedPaths(base, previousBase));
+    const touched = yield* pathsOf(batch);
+    let replayed = !same;
+    if (replayed)
+        for (const path of yield* pathsOf(previousBatch))
             affected.add(path);
+    for (const path of touched) {
+        if (affected.has(path))
+            replayed = true;
+        yield ROW_WEIGHT;
     }
-    const oldIds = new Set(before);
-    const newIds = new Set(after);
-    for (const batch of previousQueue) {
-        if (!newIds.has(JSON.stringify(batch)))
-            for (const path of pathsOf(batch))
-                affected.add(path);
-    }
-    for (const batch of queue) {
-        if (!oldIds.has(JSON.stringify(batch)))
-            for (const path of pathsOf(batch))
-                affected.add(path);
-    }
-    let expanded = true;
-    while (expanded) {
-        expanded = false;
-        for (const batch of queue) {
-            const touched = pathsOf(batch);
-            if (!touched.some((path) => affected.has(path)))
-                continue;
-            for (const path of touched) {
-                if (affected.has(path))
-                    continue;
-                affected.add(path);
-                expanded = true;
-            }
-        }
-    }
-    const documents = {};
-    const revisions = {};
-    const changes = {};
-    for (const path of affected) {
+    const restricted = {
+        documents: {},
+        revisions: {},
+        fileIds: {},
+        changes: {},
+        binaries: {},
+    };
+    for (const path of replayed ? touched : []) {
+        affected.add(path);
         if (base.documents[path] !== undefined)
-            documents[path] = base.documents[path];
+            restricted.documents[path] = base.documents[path];
         if (base.revisions[path] !== undefined)
-            revisions[path] = base.revisions[path];
+            restricted.revisions[path] = base.revisions[path];
+        if (base.fileIds[path] !== undefined)
+            restricted.fileIds[path] = base.fileIds[path];
         if (base.changes[path] !== undefined)
-            changes[path] = base.changes[path];
-    }
-    const staged = overlay({
-        documents,
-        revisions,
-        changes,
-        binaryTargets: base.binaryTargets.filter((path) => affected.has(path)),
-    }, queue.filter((batch) => pathsOf(batch).some((path) => affected.has(path))));
-    const mergedDocuments = { ...previousVisible.documents };
-    const mergedRevisions = { ...previousVisible.revisions };
-    const mergedChanges = { ...previousVisible.changes };
-    const mergedTargets = new Set(previousVisible.binaryTargets);
-    for (const path of affected) {
-        delete mergedDocuments[path];
-        delete mergedRevisions[path];
-        delete mergedChanges[path];
-        mergedTargets.delete(path);
+            restricted.changes[path] = base.changes[path];
+        if (base.binaries[path] !== undefined)
+            restricted.binaries[path] = base.binaries[path];
+        yield ROW_WEIGHT;
     }
-    Object.assign(mergedDocuments, staged.documents);
-    Object.assign(mergedRevisions, staged.revisions);
-    Object.assign(mergedChanges, staged.changes);
-    for (const path of staged.binaryTargets)
-        mergedTargets.add(path);
+    const staged = replayed ? yield* overlaying(restricted, [batch]) : base;
+    const from = (path) => (touched.has(path) ? staged : base);
     return {
-        documents: mergedDocuments,
-        revisions: mergedRevisions,
-        changes: mergedChanges,
-        binaryTargets: [...mergedTargets].sort(),
+        documents: yield* repatched(previousVisible.documents, affected, (path) => from(path).documents[path]),
+        revisions: yield* repatched(previousVisible.revisions, affected, (path) => from(path).revisions[path]),
+        fileIds: yield* repatched(previousVisible.fileIds, affected, (path) => from(path).fileIds[path]),
+        changes: yield* repatched(previousVisible.changes, affected, (path) => from(path).changes[path]),
+        binaries: yield* repatched(previousVisible.binaries, affected, (path) => from(path).binaries[path]),
     };
 }

package/dist/query.d.ts

@@ -1,5 +1,7 @@
 import { type Definition } from '@agentshouse/mdmodel';
 import type { HeadChange } from './adapter.js';
+import type { BinaryFileRecord } from './binary.js';
+import type { Open } from './open.js';
 export type Value = string | number | boolean | null;
 export type Predicate = Value | {
     $eq: Value;
@@ -36,6 +38,8 @@ export type Filter = {
     types?: readonly string[];
     where?: Where;
     pathPrefix?: string;
+    folder?: string;
+    paths?: readonly string[];
     hop?: Related & {
         field: string;
     };
@@ -43,7 +47,7 @@ export type Filter = {
         field?: string;
     };
 };
-export type Key = 'room' | 'path' | 'type' | {
+export type Key = 'room' | 'path' | 'type' | 'folder' | {
     field: string;
 };
 export type Select = 'all' | {
@@ -53,6 +57,7 @@ export type Select = 'all' | {
     markdown?: true;
     backlinks?: true;
     change?: true;
+    fileId?: true;
 };
 export type OrderOptions = {
     nulls?: 'first' | 'last';
@@ -68,6 +73,8 @@ export type DocumentQuery = Filter & {
         direction?: Direction;
     } & OrderOptions)[];
     limit?: number;
+    after?: string;
+    total?: true;
 };
 export type GroupQuery = Filter & {
     groupBy: readonly Exclude<Key, 'path'>[];
@@ -76,6 +83,8 @@ export type GroupQuery = Filter & {
         direction?: Direction;
     }[];
     limit?: number;
+    after?: string;
+    total?: true;
 };
 export type LocalQuery = DocumentQuery | GroupQuery;
 export type Backlink = {
@@ -93,18 +102,49 @@ export type QueryDocument = {
     markdown?: string;
     backlinks?: readonly Backlink[];
     change?: HeadChange | null;
+    fileId?: string | null;
+    binaryFile?: BinaryFileRecord;
 };
 export type QueryGroup = {
     key: readonly unknown[];
     count: number;
 };
-export type DocumentResult = {
+export type CoveredRoom = {
+    room: string;
+    generation: string;
+    current: boolean;
+};
+export type QueryCoverage = {
+    rooms: readonly CoveredRoom[];
+    indexes: readonly string[];
+    overlay: readonly string[];
+};
+export type Awaits = Open<'acquisition' | 'preparation' | 'index'>;
+export type PendingRoom = {
+    room: string;
+    awaits: Awaits;
+};
+export type PendingResult = {
+    pending: readonly PendingRoom[];
+};
+export type DocumentRows = {
     documents: readonly QueryDocument[];
+    next?: string;
+    total?: number;
 };
-export type GroupResult = {
+export type GroupRows = {
     groups: readonly QueryGroup[];
+    next?: string;
+    total?: number;
+};
+export type QueryRows = DocumentRows | GroupRows;
+export type DocumentResult = DocumentRows & {
+    coverage: QueryCoverage;
+};
+export type GroupResult = GroupRows & {
+    coverage: QueryCoverage;
 };
-export type QueryResult = DocumentResult | GroupResult;
+export type QueryResult = DocumentResult | GroupResult | PendingResult;
 export type QueryErrorCode = 'invalid-query' | 'unknown-type' | 'unknown-field' | 'wrong-kind';
 export type RoomFacts = {
     locator: string;
@@ -159,6 +199,8 @@ export type KeyPlan = {
     of: 'path';
 } | {
     of: 'type';
+} | {
+    of: 'folder';
 } | {
     of: 'field';
     field: string;
@@ -171,6 +213,7 @@ export type DocumentShape = {
     markdown: boolean;
     backlinks: boolean;
     change: boolean;
+    fileId: boolean;
     orderBy: readonly {
         key: KeyPlan;
         options: OrderOptions & {
@@ -190,10 +233,17 @@ export type PlanReach = {
     rooms: readonly string[];
     documents: readonly string[];
 };
+export type Continuation = {
+    text: string;
+    values: readonly unknown[] | null;
+};
 export type Plan = {
+    key: string;
     rooms: readonly string[] | null;
     reach: PlanReach | null;
     root: Scope;
+    folder: string | null;
+    paths: readonly string[] | null;
     hop: (Scope & {
         field: string;
     }) | null;
@@ -202,17 +252,23 @@ export type Plan = {
     }) | null;
     shape: DocumentShape | GroupShape;
     limit: number | null;
+    after: Continuation | null;
+    total: boolean;
 };
 export declare class QueryError extends Error {
+    readonly code: QueryErrorCode;
+    readonly retryable = false;
     constructor(code: QueryErrorCode, at: string, problem: string);
 }
 type Entries = Readonly<Record<string, unknown>>;
 export declare function invalid(at: string, problem: string): never;
 export declare function plain(value: unknown): value is Entries;
 export declare function known(value: Entries, allowed: readonly string[], at: string): void;
+export declare function flag(input: unknown, at: string): boolean;
 export declare function planOf(input: unknown, facts: HeldRooms, reach?: PlanReach | null): Plan;
 export declare function limitOf(input: unknown): number | null;
 export declare function filterOf(input: unknown, facts: HeldRooms, reach?: PlanReach | null): Plan;
+export declare function queryKey(query: unknown): string;
 export declare function identityOf(plan: Plan): string;
 export {};
 //# sourceMappingURL=query.d.ts.map