@x12i/memorix-service 3.12.4 → 3.14.0

package/dist/routes/data.js

@@ -2,13 +2,18 @@ import { parseLiveDeclaration } from "@x12i/memorix-data";
 import { resolveScope } from "../request-scope.js";
 import { ServiceError } from "../errors.js";
 import { requireCallbackAccess, requireDeploymentAccess } from "../access-gate.js";
-import { normalizeWritePrecondition } from "../write-precondition.js";
-import { assertAddressMatchesEnvelope, resolveRecordIdAlias, } from "../identity-alias.js";
+import { runContentBatch, runContentWrite } from "../writes/content-write.js";
+import { pendingRecord, publicQueuedReceipt, queueKeyIdFor, readQueueFlag, runQueuedBatch, runQueuedWrite, } from "../writes/queued-write.js";
+import { commitWriteEffects } from "../writes/effects.js";
+import { patchGuardCheck } from "../patches/service.js";
+import { resolveRecordIdAlias } from "../identity-alias.js";
 import { maybeEnqueueNexumAnalysis } from "../nexum-analysis/enqueue.js";
+import { listKnowledgeGraphsForSource } from "../nexum-analysis/knowledge-read.js";
 import { memorixActivityRing, recordLooksLikeKnowledgeGraph, } from "../activity-ring.js";
 import { getRequestCaller, requireCaller } from "../caller-identity.js";
-import { runHookTrigger } from "../hooks/request-trigger.js";
 import { parseSafeFilters, parseSafeFiltersValue, parseSafeSort, parseSafeSortValue } from "../safe-query.js";
+import { declaredDiscriminatorValues } from "../catalog-bridge.js";
+import { singleReadDiscriminatorValue, withDiscriminatorHelp } from "../single-read-discriminator.js";
 function parseIncludeObject(raw) {
     if (raw === undefined)
         return undefined;
@@ -45,13 +50,27 @@ function subjectRecordIdFromRequest(req) {
     }
     return recordId;
 }
-function readSuppressHooks(value) {
-    if (value === undefined)
-        return false;
-    if (typeof value !== "boolean") {
-        throw new ServiceError("VALIDATION", "suppressHooks must be a boolean", 400);
-    }
-    return value;
+function publicWriteReceipt(row) {
+    return {
+        receiptId: row.receiptId,
+        status: row.status,
+        ...(row.writeId ? { writeId: row.writeId } : {}),
+        createdAt: row.createdAt,
+        updatedAt: row.updatedAt,
+        ...(row.receipt ? { receipt: row.receipt } : {}),
+        ...(row.error
+            ? {
+                error: {
+                    code: row.error.code,
+                    field: row.error.field,
+                    message: row.error.message,
+                    retryable: row.error.retryable,
+                    ...(row.error.retryAfterMs !== undefined ? { retryAfterMs: row.error.retryAfterMs } : {}),
+                },
+            }
+            : {}),
+        ...(row.status === "pending" ? { retryAfterMs: row.error?.retryAfterMs ?? 1_000 } : {}),
+    };
 }
 function contentDatasource(req, body) {
     const service = requireCaller(req);
@@ -75,7 +94,134 @@ function draftForDatasourceStamp(record) {
         ...(record.insights ? { insights: record.insights } : {}),
     };
 }
+function readRunStatus(raw) {
+    if (raw === undefined || raw === "" || raw === "all")
+        return undefined;
+    if (raw === "queued" || raw === "stored" || raw === "rejected")
+        return raw;
+    throw new ServiceError("VALIDATION", "status must be queued, stored or rejected", 400, {
+        field: "status",
+        retryable: false,
+    });
+}
+/** The cursor is the accept number of the last row on the page. Callers treat it as opaque. */
+function writeReceiptCursor(seq) {
+    return Buffer.from(String(seq), "utf8").toString("base64url");
+}
+function readReceiptCursor(raw) {
+    if (raw === undefined || raw === "")
+        return undefined;
+    const seq = Number(Buffer.from(raw, "base64url").toString("utf8"));
+    // Only a cursor this service wrote decodes and encodes back to itself. Anything else would
+    // silently decode to a number and restart the list from the top.
+    if (!Number.isInteger(seq) || seq < 1 || writeReceiptCursor(seq) !== raw) {
+        throw new ServiceError("VALIDATION", "cursor is not valid", 400, { field: "cursor", retryable: false });
+    }
+    return seq;
+}
+function isReadNotFound(err) {
+    return Boolean(err && typeof err === "object" && err.code === "READ_NOT_FOUND");
+}
+/** One content instance by recordId; `filter` may only select the discriminator. */
+async function getSingleContent(platform, scope, q, recordId, include) {
+    const ports = platform.portsFor(scope);
+    const discriminatorPath = ports.catalog.getContentProfile?.(q.contentType)?.discriminator?.path;
+    const discriminatorValue = singleReadDiscriminatorValue({
+        discriminatorValue: q.discriminatorValue ?? q.discriminator,
+        filters: parseSafeFilters(q.filter),
+        discriminatorPath,
+    });
+    const queuedFor = async () => {
+        const keyId = queueKeyIdFor(ports.catalog, {
+            objectType: q.objectType,
+            contentType: q.contentType,
+            recordId,
+            discriminatorValue,
+        });
+        return keyId ? platform.writeReceiptsFor(scope.orgId).latestQueuedForKey(scope.orgId, keyId) : null;
+    };
+    let record;
+    try {
+        record = (await ports.reader.getContent(scope, {
+            objectType: q.objectType,
+            contentType: q.contentType,
+            recordId,
+            ...(discriminatorValue ? { discriminatorValue } : {}),
+        }, include ? { include } : undefined));
+    }
+    catch (err) {
+        if (isReadNotFound(err)) {
+            const pending = await queuedFor();
+            if (pending) {
+                return {
+                    record: pendingRecord(pending),
+                    stored: false,
+                    pending,
+                    ...(discriminatorValue ? { discriminatorValue } : {}),
+                };
+            }
+        }
+        throw withDiscriminatorHelp(err, () => discriminatorPath
+            ? declaredDiscriminatorValues(platform.metadata, scope.agentIds, q.objectType, q.contentType, discriminatorPath)
+            : undefined);
+    }
+    const pending = await queuedFor();
+    return {
+        record,
+        stored: true,
+        ...(discriminatorValue ? { discriminatorValue } : {}),
+        ...(pending ? { pending } : {}),
+    };
+}
 export async function registerDataRoutes(app, platform) {
+    app.get("/api/data/knowledge-graphs", async (req) => {
+        requireCallbackAccess(req, "explore");
+        const scope = resolveScope(req);
+        const q = req.query;
+        const sourceRecordId = q.sourceRecordId?.trim() ?? "";
+        if (!sourceRecordId) {
+            throw new ServiceError("VALIDATION", "sourceRecordId required", 400);
+        }
+        const caller = getRequestCaller(req);
+        const ports = platform.portsFor(scope);
+        const listed = await listKnowledgeGraphsForSource({
+            reader: ports.reader,
+            scope,
+            sourceRecordId,
+            limit: Number(q.limit ?? 50) || 50,
+            cursor: q.cursor,
+        });
+        memorixActivityRing.recordFetch({
+            orgId: scope.orgId,
+            caller,
+            mode: "list",
+            objectType: listed.addressing.objectType,
+            contentType: listed.addressing.contentType,
+            recordId: sourceRecordId,
+            hit: listed.records.length > 0,
+            count: listed.records.length,
+            knowledgeGraph: listed.records.length > 0,
+        });
+        return { scope, ...listed };
+    });
+    app.get("/api/data/content/context", async (req) => {
+        requireCallbackAccess(req, "explore");
+        const scope = resolveScope(req);
+        const q = req.query;
+        if (!q.objectType || !q.contentType) {
+            throw new ServiceError("VALIDATION", "objectType and contentType required", 400);
+        }
+        const recordId = resolveRecordIdAlias(q.id, q.recordId);
+        if (!recordId) {
+            throw new ServiceError("VALIDATION", "recordId required", 400);
+        }
+        const { record } = await getSingleContent(platform, scope, { ...q, objectType: q.objectType, contentType: q.contentType }, recordId);
+        const context = record.context;
+        if (!context || typeof context !== "object" || Array.isArray(context) || Object.keys(context).length === 0) {
+            throw new ServiceError("NOT_FOUND", "context is not stored on this record", 404);
+        }
+        return context;
+    });
     app.get("/api/data/content", async (req) => {
         requireCallbackAccess(req, "explore");
         const scope = resolveScope(req);
@@ -90,12 +236,11 @@ export async function registerDataRoutes(app, platform) {
         const recordId = resolveRecordIdAlias(q.id, q.recordId);
         if (recordId) {
             let record;
+            let discriminatorValue;
+            let pending;
+            let stored = true;
             try {
-                record = (await ports.reader.getContent(scope, {
-                    objectType: q.objectType,
-                    contentType: q.contentType,
-                    recordId,
-                }, { include }));
+                ({ record, discriminatorValue, pending, stored } = await getSingleContent(platform, scope, { ...q, objectType: q.objectType, contentType: q.contentType }, recordId, include));
             }
             catch (err) {
                 const msg = err instanceof Error ? err.message : String(err);
@@ -141,34 +286,44 @@ export async function registerDataRoutes(app, platform) {
                     contentType: q.contentType,
                     id: recordId,
                     recordId,
+                    ...(discriminatorValue ? { discriminatorValue } : {}),
                 },
                 record,
-                currentRevision: record.revision,
+                // A key with only a queued submission has no revision yet. null cannot be sent as exact-revision.
+                currentRevision: stored ? record.revision : null,
+                ...(pending ? { pending: true, receiptId: pending.receiptId } : {}),
                 unresolvedRelationships,
             };
         }
         const limit = Math.min(Math.max(Number(q.limit ?? 50) || 50, 1), 200);
         const filters = parseSafeFilters(q.filter);
         const sort = parseSafeSort(q.sort);
-        if (q.count === "true" && platform.analytics) {
-            const count = await platform.analytics.countIndexed(scope, {
-                objectType: q.objectType,
-                contentType: q.contentType,
-                filters,
-            });
-            if (count !== null) {
-                return {
-                    scope,
-                    count,
-                    addressing: {
-                        orgId: scope.orgId,
-                        objectType: q.objectType,
-                        contentType: q.contentType,
-                        id: null,
-                        recordId: null,
-                    },
-                };
-            }
+        if (q.count === "true") {
+            const indexed = platform.analytics
+                ? await platform.analytics.countIndexed(scope, {
+                    objectType: q.objectType,
+                    contentType: q.contentType,
+                    filters,
+                })
+                : null;
+            const count = indexed !== null
+                ? indexed
+                : await ports.reader.countContent(scope, {
+                    objectType: q.objectType,
+                    contentType: q.contentType,
+                    filters,
+                });
+            return {
+                scope,
+                count,
+                addressing: {
+                    orgId: scope.orgId,
+                    objectType: q.objectType,
+                    contentType: q.contentType,
+                    id: null,
+                    recordId: null,
+                },
+            };
         }
         const listed = await ports.reader.listContent(scope, {
             objectType: q.objectType,
@@ -237,178 +392,122 @@ export async function registerDataRoutes(app, platform) {
             pageInfo: { nextCursor: listed.nextCursor ?? null, limit },
         };
     });
-    app.post("/api/data/content", async (req) => {
+    app.post("/api/data/content/batch", async (req) => {
         requireDeploymentAccess(req, "write");
         const scope = resolveScope(req);
-        const body = req.body;
-        if (!body.objectType || !body.contentType) {
-            throw new ServiceError("VALIDATION", "objectType, contentType required", 400);
-        }
         await platform.analytics?.prepare(scope.orgId);
-        assertAddressMatchesEnvelope({
-            objectType: body.objectType,
-            contentType: body.contentType,
-            record: body.record,
-        });
-        const precondition = normalizeWritePrecondition(body.precondition, body.expectedRevision);
-        const datasource = contentDatasource(req, body);
-        const suppressHooks = readSuppressHooks(body.suppressHooks);
-        const ports = platform.portsFor(scope);
-        const op = body.op ?? (precondition.form === "absent" ? "add" : "upsert");
-        const expectedRevision = precondition.form === "exact-revision" ? precondition.expectedRevision : undefined;
-        try {
-            if (op === "remove") {
-                if (!body.record?.recordId || expectedRevision === undefined) {
-                    throw new ServiceError("VALIDATION", "record.recordId and expectedRevision required for remove", 400);
-                }
-                const pinned = await ports.publications.list();
-                const blocking = pinned.filter((p) => (p.status === "committed" || p.status === "draft") &&
-                    p.members.some((m) => m.recordId === body.record.recordId &&
-                        m.objectType === body.objectType &&
-                        m.contentType === body.contentType));
-                if (blocking.length) {
-                    throw new ServiceError("PUBLICATION_PINNED", "cannot remove a content instance referenced by a draft or committed publication", 409, { publicationIds: blocking.map((p) => p.publicationId) });
-                }
-                const objectType = body.objectType;
-                const contentType = body.contentType;
-                const recordId = body.record.recordId;
-                const discriminatorValue = typeof body.discriminatorValue === "string" && body.discriminatorValue.trim()
-                    ? body.discriminatorValue.trim()
-                    : undefined;
-                const noted = await runHookTrigger(() => ports.writer.remove({
-                    scope,
-                    objectType,
-                    contentType,
-                    recordId,
-                    ...(discriminatorValue ? { discriminatorValue } : {}),
-                    precondition: { kind: "revision", revision: expectedRevision },
-                    datasource,
-                    suppressHooks,
-                }));
-                const result = noted.value;
-                memorixActivityRing.recordLand({
-                    orgId: scope.orgId,
-                    caller: getRequestCaller(req),
-                    objectType: body.objectType,
-                    contentType: body.contentType,
-                    recordId: result.recordId,
-                    revision: result.previousRevision,
-                    outcome: "removed",
-                    hooks: noted.hooks,
-                });
-                return { scope, written: true, outcome: "updated", result, precondition: body.precondition };
-            }
-            if (!body.record) {
-                throw new ServiceError("VALIDATION", "record required", 400);
-            }
-            const record = {
-                ...body.record,
-                objectType: body.objectType,
-                contentType: body.contentType,
-            };
-            if (record._system && typeof record._system === "object") {
-                const system = { ...record._system };
-                delete system._datasource;
-                record._system = system;
-            }
-            let result;
-            let hooks;
-            const objectType = body.objectType;
-            const contentType = body.contentType;
-            const recordId = body.record.recordId;
-            if (op === "add" || precondition.form === "absent") {
-                const noted = await runHookTrigger(() => ports.writer.add({
-                    scope,
-                    precondition: { kind: "absent" },
-                    record,
-                    datasource,
-                    suppressHooks,
-                }));
-                result = noted.value;
-                hooks = noted.hooks;
-            }
-            else if (op === "patch") {
-                const noted = await runHookTrigger(() => ports.writer.patch({
-                    scope,
-                    precondition: { kind: "revision", revision: Number(expectedRevision) },
-                    objectType,
-                    contentType,
-                    recordId,
-                    ops: body.patch ?? [],
-                    datasource,
-                    suppressHooks,
-                }));
-                result = noted.value;
-                hooks = noted.hooks;
-            }
-            else if (op === "replace") {
-                const noted = await runHookTrigger(() => ports.writer.replace({
-                    scope,
-                    precondition: { kind: "revision", revision: Number(expectedRevision) },
-                    record,
-                    datasource,
-                    suppressHooks,
-                }));
-                result = noted.value;
-                hooks = noted.hooks;
-            }
-            else {
-                // exact-revision upsert — never silently treat as create
-                const noted = await runHookTrigger(() => ports.writer.upsert({
-                    scope,
-                    precondition: { kind: "revision", revision: Number(expectedRevision) },
-                    record,
-                    datasource,
-                    suppressHooks,
-                }));
-                result = noted.value;
-                hooks = noted.hooks;
-            }
-            const indexing = await maybeEnqueueNexumAnalysis({
-                platform,
-                scope,
-                objectType: body.objectType,
-                contentType: body.contentType,
-                recordId: result.record.recordId,
-                revision: result.record.revision,
-                data: result.record.data &&
-                    typeof result.record.data === "object" &&
-                    !Array.isArray(result.record.data)
-                    ? result.record.data
-                    : body.record?.data &&
-                        typeof body.record.data === "object" &&
-                        !Array.isArray(body.record.data)
-                        ? body.record.data
-                        : undefined,
-            });
-            memorixActivityRing.recordLand({
+        const caller = requireCaller(req);
+        const queue = readQueueFlag(req.body?.queue);
+        return (queue ? runQueuedBatch : runContentBatch)({
+            scope,
+            caller,
+            datasource: contentDatasource(req, {}),
+            ports: platform.portsFor(scope),
+            receipts: platform.writeReceiptsFor(scope.orgId),
+            onCommitted: (event) => commitWriteEffects({ platform, scope, caller, event }),
+            checkWrite: patchGuardCheck(platform, scope),
+        }, req.body, (item) => contentDatasource(req, item));
+    });
+    app.get("/api/data/receipts", async (req) => {
+        requireCallbackAccess(req, "explore");
+        const scope = resolveScope(req);
+        const caller = requireCaller(req);
+        const q = req.query;
+        const limit = Math.min(Math.max(Number(q.limit ?? 50) || 50, 1), 200);
+        const runId = q.runId?.trim();
+        const receipts = platform.writeReceiptsFor(scope.orgId);
+        if (runId) {
+            // Queued writes of one run, in accept order. Omitting status returns every status.
+            const status = readRunStatus(q.status);
+            const stepId = q.stepId?.trim() || undefined;
+            const after = readReceiptCursor(q.cursor);
+            const rows = await receipts.listByRun({
                 orgId: scope.orgId,
-                caller: getRequestCaller(req),
-                objectType: body.objectType,
-                contentType: body.contentType,
-                recordId: result.record.recordId,
-                revision: result.record.revision,
-                outcome: result.created ? "created" : "updated",
-                indexing,
-                hooks,
+                caller,
+                runId,
+                ...(stepId ? { stepId } : {}),
+                ...(status ? { status } : {}),
+                ...(after !== undefined ? { after } : {}),
+                limit: limit + 1,
             });
+            const page = rows.slice(0, limit);
+            const last = page[page.length - 1];
             return {
                 scope,
-                written: true,
-                outcome: result.created ? "created" : "updated",
-                result,
-                currentRevision: result.record.revision,
-                precondition: body.precondition,
-                indexing,
+                receipts: page.map(publicQueuedReceipt),
+                pageInfo: {
+                    nextCursor: rows.length > limit && last?.queue ? writeReceiptCursor(last.queue.seq) : null,
+                    limit,
+                },
             };
         }
-        catch (err) {
-            const msg = err instanceof Error ? err.message : String(err);
-            if (/stale|revision|WRITE_STALE/i.test(msg)) {
-                throw new ServiceError("REVISION_CONFLICT", msg, 409);
-            }
-            throw err;
-        }
+        if (q.stepId?.trim()) {
+            throw new ServiceError("VALIDATION", "stepId needs runId", 400, { field: "runId", retryable: false });
+        }
+        const status = q.status === "pending" ||
+            q.status === "queued" ||
+            q.status === "stored" ||
+            q.status === "rejected" ||
+            q.status === "all"
+            ? q.status
+            : "rejected";
+        const rows = await receipts.list({
+            orgId: scope.orgId,
+            caller,
+            ...(status === "all" ? {} : { status }),
+            limit,
+        });
+        return { scope, receipts: rows.map((row) => (row.queue ? publicQueuedReceipt(row) : publicWriteReceipt(row))) };
+    });
+    // A static segment wins over `:receiptId`, so `summary` is never read as a receipt id.
+    app.get("/api/data/receipts/summary", async (req) => {
+        requireCallbackAccess(req, "explore");
+        const scope = resolveScope(req);
+        const caller = requireCaller(req);
+        const q = req.query;
+        const runId = q.runId?.trim();
+        if (!runId)
+            throw new ServiceError("VALIDATION", "runId required", 400, { field: "runId", retryable: false });
+        const stepId = q.stepId?.trim() || undefined;
+        const counts = await platform
+            .writeReceiptsFor(scope.orgId)
+            .summarizeRun({ orgId: scope.orgId, caller, runId, ...(stepId ? { stepId } : {}) });
+        return { scope, runId, ...(stepId ? { stepId } : {}), ...counts };
+    });
+    app.get("/api/data/receipts/:receiptId", async (req) => {
+        requireCallbackAccess(req, "explore");
+        const scope = resolveScope(req);
+        const caller = requireCaller(req);
+        const receiptId = req.params.receiptId?.trim() ?? "";
+        if (!receiptId)
+            throw new ServiceError("VALIDATION", "receiptId required", 400, { field: "receiptId" });
+        const row = await platform.writeReceiptsFor(scope.orgId).get(scope.orgId, receiptId);
+        if (!row || row.caller !== caller) {
+            throw new ServiceError("NOT_FOUND", "receipt not found", 404, { field: "receiptId", retryable: false });
+        }
+        return row.queue ? publicQueuedReceipt(row) : publicWriteReceipt(row);
+    });
+    app.post("/api/data/content", async (req, reply) => {
+        requireDeploymentAccess(req, "write");
+        const scope = resolveScope(req);
+        const body = (req.body ?? {});
+        await platform.analytics?.prepare(scope.orgId);
+        const caller = requireCaller(req);
+        const queue = readQueueFlag(body.queue);
+        const outcome = await (queue ? runQueuedWrite : runContentWrite)({
+            scope,
+            caller,
+            datasource: contentDatasource(req, body),
+            ports: platform.portsFor(scope),
+            receipts: platform.writeReceiptsFor(scope.orgId),
+            onCommitted: (event) => commitWriteEffects({ platform, scope, caller, event }),
+            checkWrite: patchGuardCheck(platform, scope),
+        }, body);
+        if (outcome.kind === "rejected")
+            throw outcome.error;
+        if (outcome.kind === "pending" || outcome.kind === "queued")
+            return reply.code(202).send(outcome.body);
+        return outcome.body;
     });
     app.post("/api/data/content/move-fields", async (req) => {
         requireDeploymentAccess(req, "write");