@earendil-works/pi-durable 0.87.1 → 0.99.0

package/dist/storage/memory.js

@@ -1,4 +1,15 @@
-import { applyImmutable } from "@earendil-works/chord/delta";
+import { applyImmutableBatches } from "@earendil-works/chord/delta";
+import { StorageRejected } from "../errors.js";
+import { idFromNumber, seqFromNumber } from "../ids.js";
+function* documentDeltaBatches(id, version, revisions, start) {
+    for (let index = start; index < revisions.length; index++) {
+        const revision = revisions[index];
+        if (revision.kind !== "delta" || revision.version !== version) {
+            throw new Error(`Document ${id} crosses a stored version boundary without a base`);
+        }
+        yield revision.ops;
+    }
+}
 const clone = (value) => {
     if (value === null || typeof value !== "object")
         return value;
@@ -23,7 +34,21 @@ const clone = (value) => {
     }
     return result;
 };
-const cursorId = (cursor) => cursor?.after;
+const freeze = (value) => {
+    if (value === null || typeof value !== "object" || Object.isFrozen(value))
+        return value;
+    for (const child of Object.values(value))
+        freeze(child);
+    return Object.freeze(value);
+};
+const cursorId = (cursor) => {
+    const after = cursor?.after;
+    if (after === undefined)
+        return undefined;
+    if (typeof after !== "number" || !Number.isSafeInteger(after))
+        throw new TypeError("Invalid storage cursor");
+    return idFromNumber(after);
+};
 const lowerBound = (ids, target) => {
     let low = 0;
     let high = ids.length;
@@ -59,6 +84,14 @@ const removeSorted = (ids, id) => {
     if (ids[index] === id)
         ids.splice(index, 1);
 };
+const insertMapId = (index, key, id) => {
+    let ids = index.get(key);
+    if (ids === undefined) {
+        ids = [];
+        index.set(key, ids);
+    }
+    insertSorted(ids, id);
+};
 const scopeKey = (scope) => {
     switch (scope.kind) {
         case "session":
@@ -81,19 +114,7 @@ const isAliveAt = (record, at) => {
     return record.createdAt <= at && (record.retiredAt === undefined || at < record.retiredAt);
 };
 const isCurrentOnly = (record) => record.scope.kind !== "conversation" || record.history === "latest";
-const tableContaining = (state, id) => {
-    if (state.conversations.has(id))
-        return "conversation";
-    if (state.entries.has(id))
-        return "entry";
-    if (state.tasks.has(id))
-        return "task";
-    if (state.submissions.has(id))
-        return "submission";
-    if (state.documents.has(id))
-        return "document";
-    return undefined;
-};
+const tableContaining = (state, id) => state.recordTypes.get(id);
 const page = (values, limit) => {
     const items = values.slice(0, limit);
     if (values.length <= limit)
@@ -108,8 +129,11 @@ const page = (values, limit) => {
  */
 export class MemoryStorage {
     state = {
+        recordTypes: new Map(),
         conversations: new Map(),
         conversationIds: [],
+        conversationIdsByOwnerConversation: new Map(),
+        conversationIdsByOwnerTask: new Map(),
         entries: new Map(),
         entryIds: new Map(),
         headEntryIds: new Map(),
@@ -118,6 +142,8 @@ export class MemoryStorage {
         taskIds: [],
         taskIdsByStatus: { pending: [], running: [], terminal: [] },
         submissions: new Map(),
+        submissionIds: [],
+        submissionIdsByStatus: { queued: [], placed: [], done: [], unanswered: [] },
         submissionIdsByRequest: new Map(),
         documents: new Map(),
         documentAddresses: new Map(),
@@ -127,20 +153,91 @@ export class MemoryStorage {
     nextSeq = 1;
     closed = false;
     async commit(writes, _context) {
+        return this.prepareCommit(writes).apply();
+    }
+    /** Validate and detach one commit without changing observable state. */
+    prepareCommit(writes, seq = seqFromNumber(this.nextSeq)) {
         this.assertOpen();
-        const prepared = writes.map((write) => clone(write));
-        const seq = this.nextSeq;
-        this.checkGlobalIds(prepared);
-        const documentActions = this.prepareDocumentActions(prepared);
+        if (!Number.isSafeInteger(seq) || seq < this.nextSeq) {
+            throw new Error(`Commit sequence ${seq} does not strictly increase`);
+        }
+        const detachedWrites = freeze(this.resolveDocumentCopies(writes.map((write) => clone(write))));
+        this.checkGlobalIds(detachedWrites);
+        const documentActions = this.prepareDocumentActions(detachedWrites);
         this.checkDocumentActions(documentActions);
+        let applied = false;
+        return {
+            seq,
+            writes: detachedWrites,
+            apply: () => {
+                if (!applied) {
+                    applied = true;
+                    this.applyPreparedCommit(detachedWrites, documentActions, seq);
+                }
+                return seq;
+            },
+        };
+    }
+    resolveDocumentCopies(writes) {
+        if (!writes.some((write) => write.type === "document.copy"))
+            return writes;
+        const changedDocumentIds = new Set();
+        for (const write of writes) {
+            if (write.type === "document.create" || write.type === "document.copy") {
+                changedDocumentIds.add(write.record.id);
+            }
+            else if (write.type === "document.change" || write.type === "document.retire") {
+                changedDocumentIds.add(write.id);
+            }
+        }
+        return writes.map((write) => {
+            if (write.type !== "document.copy")
+                return write;
+            try {
+                if (changedDocumentIds.has(write.source.id)) {
+                    throw new Error(`Fork source document ${write.source.id} is changed in the copy batch`);
+                }
+                const stored = this.materializeDocument(write.source.id, write.source.at);
+                if (stored === undefined)
+                    throw new Error(`Fork source document ${write.source.id} cannot be read`);
+                if (stored.record.scope.kind !== "conversation" ||
+                    write.record.scope.kind !== "conversation" ||
+                    stored.record.kind !== write.record.kind ||
+                    stored.record.key !== write.record.key ||
+                    stored.record.history !== write.record.history ||
+                    stored.record.fork !== write.record.fork) {
+                    throw new Error(`Fork source document ${write.source.id} does not match the copied record`);
+                }
+                return {
+                    type: "document.create",
+                    record: write.record,
+                    content: { kind: "base", version: stored.version, value: stored.value },
+                };
+            }
+            catch (error) {
+                if (error instanceof StorageRejected)
+                    throw error;
+                throw new StorageRejected(`Document copy ${write.record.id} was rejected`, { cause: error });
+            }
+        });
+    }
+    applyPreparedCommit(prepared, documentActions, seq) {
         for (const write of prepared) {
             switch (write.type) {
-                case "conversation":
+                case "conversation": {
+                    this.state.recordTypes.set(write.value.id, "conversation");
                     this.state.conversations.set(write.value.id, write.value);
                     insertSorted(this.state.conversationIds, write.value.id);
+                    const owner = write.value.owner;
+                    if (owner !== undefined) {
+                        insertMapId(this.state.conversationIdsByOwnerConversation, owner.conversationId, write.value.id);
+                        insertMapId(this.state.conversationIdsByOwnerTask, owner.taskId, write.value.id);
+                    }
                     this.nextId = Math.max(this.nextId, write.value.id + 1);
                     break;
+                }
                 case "entry": {
+                    this.state.recordTypes.set(write.value.id, "entry");
                     this.state.entries.set(write.value.id, write.value);
                     this.state.entryCommitSeqs.set(write.value.id, seq);
                     let ids = this.state.entryIds.get(write.value.conversationId);
@@ -161,6 +258,7 @@ export class MemoryStorage {
                     break;
                 }
                 case "task": {
+                    this.state.recordTypes.set(write.value.id, "task");
                     const previous = this.state.tasks.get(write.value.id);
                     if (previous === undefined) {
                         insertSorted(this.state.taskIds, write.value.id);
@@ -175,7 +273,16 @@ export class MemoryStorage {
                     break;
                 }
                 case "submission": {
+                    this.state.recordTypes.set(write.value.id, "submission");
                     const previous = this.state.submissions.get(write.value.id);
+                    if (previous === undefined) {
+                        insertSorted(this.state.submissionIds, write.value.id);
+                        insertSorted(this.state.submissionIdsByStatus[write.value.status], write.value.id);
+                    }
+                    else if (previous.status !== write.value.status) {
+                        removeSorted(this.state.submissionIdsByStatus[previous.status], write.value.id);
+                        insertSorted(this.state.submissionIdsByStatus[write.value.status], write.value.id);
+                    }
                     if (previous?.requestId !== undefined) {
                         const previousRequests = this.state.submissionIdsByRequest.get(previous.conversationId);
                         if (previousRequests?.get(previous.requestId) === write.value.id) {
@@ -196,6 +303,8 @@ export class MemoryStorage {
                     this.nextId = Math.max(this.nextId, write.value.id + 1);
                     break;
                 }
+                case "document.copy":
+                    throw new Error("Prepared document copy was not resolved");
                 case "document.create":
                 case "document.change":
                 case "document.retire":
@@ -203,32 +312,53 @@ export class MemoryStorage {
             }
         }
         this.applyDocumentActions(documentActions, seq);
-        this.nextSeq++;
+        this.nextSeq = seq + 1;
         return seq;
     }
     async mintId() {
         this.assertOpen();
         if (!Number.isSafeInteger(this.nextId))
             throw new Error("ID space is exhausted");
-        return this.nextId++;
+        return idFromNumber(this.nextId++);
     }
     async conversation(id, _context) {
         this.assertOpen();
         const value = this.state.conversations.get(id);
         return value === undefined ? undefined : clone(value);
     }
-    async scanConversations(cursor, limit, _context) {
+    async scanConversations(query, limit, cursor, _context) {
         this.assertOpen();
+        const ids = query.ownerTaskId !== undefined
+            ? (this.state.conversationIdsByOwnerTask.get(query.ownerTaskId) ?? [])
+            : query.ownerConversationId !== undefined
+                ? (this.state.conversationIdsByOwnerConversation.get(query.ownerConversationId) ?? [])
+                : this.state.conversationIds;
         const after = cursorId(cursor);
-        const start = after === undefined ? 0 : upperBound(this.state.conversationIds, after);
-        const values = this.state.conversationIds
-            .slice(start, start + limit + 1)
-            .map((id) => this.state.conversations.get(id));
+        const start = after === undefined ? 0 : upperBound(ids, after);
+        const values = [];
+        for (let index = start; index < ids.length && values.length <= limit; index++) {
+            const value = this.state.conversations.get(ids[index]);
+            if (query.ownerConversationId !== undefined && value.owner?.conversationId !== query.ownerConversationId) {
+                continue;
+            }
+            values.push(value);
+        }
         return page(values, limit);
     }
-    async entry(id, _context) {
+    async entry(idOrConversationId, idOrContext, context) {
         this.assertOpen();
-        const entry = this.state.entries.get(id);
+        if (context === undefined) {
+            const id = idFromNumber(idOrConversationId);
+            const entry = this.state.entries.get(id);
+            if (entry === undefined)
+                return undefined;
+            return { entry: clone(entry), commitSeq: this.state.entryCommitSeqs.get(id) };
+        }
+        if (typeof idOrContext !== "number")
+            throw new TypeError("Storage.entry() requires an entry ID");
+        const conversationId = idFromNumber(idOrConversationId);
+        const id = idFromNumber(idOrContext);
+        const entry = this.visibleEntries(conversationId, id, id).next().value;
         if (entry === undefined)
             return undefined;
         return { entry: clone(entry), commitSeq: this.state.entryCommitSeqs.get(id) };
@@ -254,7 +384,7 @@ export class MemoryStorage {
             currentId = conversation.parent.conversationId;
         }
     }
-    async scanEntries(query, cursor, limit, _context) {
+    async scanEntries(query, limit, cursor, _context) {
         this.assertOpen();
         const after = cursorId(cursor);
         const maxEntryId = after === undefined ? query.maxEntryId : Math.min(query.maxEntryId ?? Number.POSITIVE_INFINITY, after - 1);
@@ -271,7 +401,7 @@ export class MemoryStorage {
         const value = this.state.tasks.get(id);
         return value === undefined ? undefined : clone(value);
     }
-    async scanTasks(query, cursor, limit, _context) {
+    async scanTasks(query, limit, cursor, _context) {
         this.assertOpen();
         const after = cursorId(cursor);
         const ids = query.status === undefined ? this.state.taskIds : this.state.taskIdsByStatus[query.status];
@@ -296,6 +426,20 @@ export class MemoryStorage {
         const value = this.state.submissions.get(id);
         return value === undefined ? undefined : clone(value);
     }
+    async scanSubmissions(query, limit, cursor, _context) {
+        this.assertOpen();
+        const after = cursorId(cursor);
+        const ids = query.status === undefined ? this.state.submissionIds : this.state.submissionIdsByStatus[query.status];
+        const start = after === undefined ? 0 : upperBound(ids, after);
+        const values = [];
+        for (let index = start; index < ids.length && values.length <= limit; index++) {
+            const value = this.state.submissions.get(ids[index]);
+            if (query.conversationId !== undefined && value.conversationId !== query.conversationId)
+                continue;
+            values.push(value);
+        }
+        return page(values, limit);
+    }
     async submissionByRequest(conversationId, requestId, _context) {
         this.assertOpen();
         const id = this.state.submissionIdsByRequest.get(conversationId)?.get(requestId);
@@ -318,32 +462,17 @@ export class MemoryStorage {
     }
     async document(id, at, _context) {
         this.assertOpen();
-        const stored = this.state.documents.get(id);
-        if (stored === undefined)
-            return undefined;
-        if (at !== "current" && isCurrentOnly(stored.record)) {
-            throw new Error(`Document ${id} does not retain historical content`);
-        }
-        if (!isAliveAt(stored.record, at))
-            return undefined;
-        const revisions = at === "current" ? stored.revisions : stored.revisions.filter((revision) => revision.seq <= at);
-        let baseIndex = revisions.length - 1;
-        while (baseIndex >= 0 && revisions[baseIndex].kind !== "base")
-            baseIndex--;
-        const base = revisions[baseIndex];
-        if (base?.kind !== "base")
-            throw new Error(`Document ${id} is missing a required base`);
-        let value = base.value;
-        for (let index = baseIndex + 1; index < revisions.length; index++) {
-            const revision = revisions[index];
-            if (revision.kind !== "delta" || revision.version !== base.version) {
-                throw new Error(`Document ${id} crosses a stored version boundary without a base`);
-            }
-            value = applyImmutable(value, revision.ops);
-        }
-        return { record: clone(stored.record), version: base.version, value: clone(value) };
+        const stored = this.materializeDocument(id, at);
+        return stored === undefined
+            ? undefined
+            : {
+                record: clone(stored.record),
+                version: stored.version,
+                value: clone(stored.value),
+                deltasSinceBase: stored.deltasSinceBase,
+            };
     }
-    async scanDocuments(query, cursor, limit, _context) {
+    async scanDocuments(query, limit, cursor, _context) {
         this.assertOpen();
         const ids = this.state.documentIdsByScope.get(scopeKey(query.scope)) ?? [];
         const after = cursorId(cursor);
@@ -361,6 +490,25 @@ export class MemoryStorage {
     async close(_context) {
         this.closed = true;
     }
+    materializeDocument(id, at) {
+        const stored = this.state.documents.get(id);
+        if (stored === undefined)
+            return undefined;
+        if (at !== "current" && isCurrentOnly(stored.record)) {
+            throw new Error(`Document ${id} does not retain historical content`);
+        }
+        if (!isAliveAt(stored.record, at))
+            return undefined;
+        const revisions = at === "current" ? stored.revisions : stored.revisions.filter((revision) => revision.seq <= at);
+        let baseIndex = revisions.length - 1;
+        while (baseIndex >= 0 && revisions[baseIndex].kind !== "base")
+            baseIndex--;
+        const base = revisions[baseIndex];
+        if (base?.kind !== "base")
+            throw new Error(`Document ${id} is missing a required base`);
+        const value = applyImmutableBatches(base.value, documentDeltaBatches(id, base.version, revisions, baseIndex + 1));
+        return { record: stored.record, version: base.version, value, deltasSinceBase: revisions.length - baseIndex - 1 };
+    }
     *visibleEntries(conversationId, minEntryId = Number.NEGATIVE_INFINITY, maxEntryId = Number.POSITIVE_INFINITY) {
         if (!this.state.conversations.has(conversationId)) {
             throw new Error(`Unknown conversation: ${conversationId}`);
@@ -389,8 +537,9 @@ export class MemoryStorage {
         for (const write of writes) {
             if (write.type === "document.change" || write.type === "document.retire")
                 continue;
-            const table = write.type === "document.create" ? "document" : write.type;
-            const id = write.type === "document.create" ? write.record.id : write.value.id;
+            const document = write.type === "document.create" || write.type === "document.copy";
+            const table = document ? "document" : write.type;
+            const id = document ? write.record.id : write.value.id;
             const existing = tableContaining(this.state, id);
             const earlier = claimed.get(id);
             if (table === "conversation" || table === "entry" || table === "document") {
@@ -453,9 +602,6 @@ export class MemoryStorage {
                 throw new Error(`Document ${id} already exists`);
             if (existing?.record.retiredAt !== undefined)
                 throw new Error(`Document ${id} is retired`);
-            if (action.create !== undefined && action.content?.kind !== "base") {
-                throw new Error(`Document ${id} creation requires a base`);
-            }
             const previous = existing?.revisions.at(-1);
             if (action.content?.kind === "delta") {
                 if (previous === undefined)
@@ -490,6 +636,7 @@ export class MemoryStorage {
                     ...(action.retire ? { retiredAt: seq } : {}),
                 };
                 stored = { record, revisions: [{ ...action.content, seq }] };
+                this.state.recordTypes.set(id, "document");
                 this.state.documents.set(id, stored);
                 const key = recordAddressKey(record);
                 let address = this.state.documentAddresses.get(key);