@plurnk/plurnk-service 1.9.1 → 1.10.0

package/dist/core/TurnRunner.js

@@ -1,5 +1,5 @@
 var _a;
-import { PathSyntax, PlurnkParser, PlurnkParseError, UNKNOWN_POSITION } from "@plurnk/plurnk-contracts";
+import { PlurnkParser, PlurnkParseError, UNKNOWN_POSITION } from "@plurnk/plurnk-contracts";
 const TERMINAL_SEND_SIGNALS = new Set([102, 200, 202, 499]);
 const comparePosition = (a, b) => a.line - b.line || a.column - b.column;
 import Meta from "@plurnk/plurnk-meta";
@@ -8,7 +8,8 @@ import GitMembership from "./git-membership.js";
 import GitState from "./git-state.js";
 import WorkspaceSettings from "./workspace-settings.js";
 import { editedSpan } from "../content/index.js";
-import { authorityParts, generatedPathname, promptPathname, promptLoopPrefix, renderAddress } from "./plurnk-uri.js";
+import ReadResolve from "../content/read-resolve.js";
+import { authorityParts, generatedPathname, promptPathname, promptLoopPrefix } from "./plurnk-uri.js";
 import { readFile } from "node:fs/promises";
 // {§grammar-rail-registration} — bare names are the built-in rail namespace;
 // everything else is an import specifier resolved through the Node chain.
@@ -43,18 +44,6 @@ import { readOptimisticSettlementMs } from "./optimistic-settlement.js";
 import ModelCall, { ModelCallPersistenceError, ProviderAccountingIntegrityError, } from "./ModelCall.js";
 import OverflowTurn from "./OverflowTurn.js";
 import TurnOps from "./TurnOps.js";
-const RECORD_STREAM_MIMETYPES = new Set(["application/jsonl", "application/x-ndjson"]);
-const baseMimetype = (mimetype) => mimetype.split(";", 1)[0].trim().toLowerCase();
-// {§exec-stream} Active streams publish only independently meaningful units.
-// Atomic documents wait for close; JSONL stops after its last complete record.
-const streamPublicationEnd = (content, mimetype, cursor, closed) => {
-    const type = baseMimetype(mimetype);
-    if (closed || type.startsWith("text/"))
-        return content.length;
-    if (!RECORD_STREAM_MIMETYPES.has(type))
-        return cursor;
-    return Math.max(cursor, content.lastIndexOf("\n") + 1);
-};
 const ENGINE_PROBLEMS = Object.freeze({
     max_commands_exceeded: {
         status: 429,
@@ -182,6 +171,7 @@ class TurnRunner {
     #liveSubscriptions;
     #streamEventNotify;
     #reasoningEventNotify;
+    #loopPacketNotify;
     #wakeWorkerNotify;
     #executors;
     #loopSignal;
@@ -190,7 +180,7 @@ class TurnRunner {
     #dispatch;
     #resolveWorkerProviderIdentity;
     #gbnfCache = new Map();
-    constructor({ db, schemes, mimetypes, weigh, notices, problems, strikes, packets, dispatcher, liveSubscriptions, streamEventNotify, reasoningEventNotify, wakeWorkerNotify, executors, loopSignal, interactions, warmWorkspace, dispatch, resolveWorkerProviderIdentity, }) {
+    constructor({ db, schemes, mimetypes, weigh, notices, problems, strikes, packets, dispatcher, liveSubscriptions, streamEventNotify, reasoningEventNotify, loopPacketNotify, wakeWorkerNotify, executors, loopSignal, interactions, warmWorkspace, dispatch, resolveWorkerProviderIdentity, }) {
         this.#db = db;
         this.#schemes = schemes;
         this.#mimetypes = mimetypes;
@@ -203,6 +193,7 @@ class TurnRunner {
         this.#liveSubscriptions = liveSubscriptions;
         this.#streamEventNotify = streamEventNotify;
         this.#reasoningEventNotify = reasoningEventNotify;
+        this.#loopPacketNotify = loopPacketNotify;
         this.#wakeWorkerNotify = wakeWorkerNotify;
         this.#executors = executors;
         this.#loopSignal = loopSignal;
@@ -211,6 +202,19 @@ class TurnRunner {
         this.#dispatch = dispatch;
         this.#resolveWorkerProviderIdentity = resolveWorkerProviderIdentity;
     }
+    async #recordInference(args) {
+        await Turn.recordInference(this.#db, args.turnId, args.evidence);
+        if (this.#loopPacketNotify === undefined)
+            return;
+        const packetCount = await this.#db.engine_loop_packet_count.get({ loop_id: args.loopId });
+        if (packetCount === undefined)
+            throw new Error(`loop ${args.loopId}: packet count row missing`);
+        this.#loopPacketNotify(args.workspaceId, {
+            workerId: args.workerId,
+            loopId: args.loopId,
+            packetCount: packetCount.count,
+        });
+    }
     // {§rail-truth-engine-verdict} — the verify GAP (a configured grammar @plurnk/gbnf can't
     // parse): warn once per message, never per turn; the turn records railsVerdict "unverifiable".
     static #railGapWarned = new Set();
@@ -653,6 +657,29 @@ class TurnRunner {
                 requestInteraction: (request) => this.#interactions.request(request, { workspaceId, workerId, loopId, turnId: contextTurnId }, this.#loopSignal(loopId)),
             });
             let systemCtx = systemContext(initializationTurn?.id ?? modelTurn.id);
+            // {§prompt-entry} — the prompt entry exists before any turn of the loop
+            // runs, so the initialization COPY archives a real source; its `prompt`
+            // log row is published to the model turn below (one durable publication
+            // per loop, decided by that row).
+            const promptPublication = turnNumber === 1 && loopRow?.prompt_published === 0
+                && typeof loopRow.prompt === "string" && loopRow.prompt.length > 0
+                ? {
+                    content: loopRow.prompt,
+                    source: loopRow.prompt_source,
+                    path: promptTarget(loopRow.sequence, 1),
+                    openPaths: assertOpenPaths(JSON.parse(loopRow.open_paths), `Loop ${loopId} open_paths`),
+                }
+                : null;
+            if (promptPublication !== null) {
+                const entry = {
+                    channels: { body: { content: promptPublication.content, mimetype: "text/markdown" } },
+                    attributes: {
+                        openPaths: promptPublication.openPaths,
+                        ...(promptPublication.source === null ? {} : { source: promptPublication.source }),
+                    },
+                };
+                await EntryCrud.writeEntry({ authority: "", pathname: promptPublication.path.pathname }, entry, systemCtx, "prompt", workerId);
+            }
             const initializationStatements = [];
             // {§worker-initialization-entry} — the worker's first turn is the worked
             // example itself: an ordinary PLAN, the actual orienting operations,
@@ -663,12 +690,28 @@ class TurnRunner {
                     signal: null, target: null, lineMarker: null,
                     body: [{
                             content: "Discover the tooling available and survey the workspace file root.",
-                            priority: "medium",
                             status: "in_progress",
                         }],
                     position: UNKNOWN_POSITION,
                 };
                 initializationStatements.push(plan);
+                // {§worker-initialization-entry} — the prompt is archived into the
+                // worker's private space: the worked COPY specimen, and the private
+                // space shown as scratch. An append onto an absent entry creates it.
+                // Authored first among the operations because it executes first
+                // ({§op-mode-phases}: Mutate precedes Observe).
+                if (promptPublication !== null) {
+                    const archive = {
+                        op: "COPY", delimiter: "", annotation: "append latest prompt", signal: ["+_plurnk", "+backup"],
+                        target: promptPublication.path, lineMarker: null,
+                        body: {
+                            target: { kind: "url", raw: "worker://~/prompts.md", scheme: "worker", username: null, password: null, hostname: "~", port: null, pathname: "/prompts.md", query: null, fragment: null },
+                            lineMarker: { marks: [-1] },
+                        },
+                        position: UNKNOWN_POSITION,
+                    };
+                    initializationStatements.push(archive);
+                }
                 // {§turn0-agents-stunt} — the project AGENTS.md (materialized by LoopDocs as
                 // worker://~/_plurnk/agents.md) gets one foisted READ on the worker's first
                 // loop, so local repo guidance is visible turn-0 content. Global policy
@@ -738,11 +781,11 @@ class TurnRunner {
                     const catalogSchemes = await this.#db.engine_scheme_catalog_summary.all({ workspace_id: workspaceId });
                     const fileItems = catalogSchemes.find(({ scheme }) => scheme === "file")?.shallow_items ?? 0;
                     const fileCap = filesItems > 0 && fileItems > 0 ? Math.min(filesItems, fileItems) : null;
-                    // {§tools-resource-materialization} — PLURNK_MCP_EXPANDED servers
-                    // contribute one complete family-document READ each, directly
-                    // after the family survey, so turn 0 names every tool they
-                    // expose in order. (The family document is the whole tool
-                    // teaching — per-target child documents do not exist.)
+                    // {§tools-resource-materialization} — a PLURNK_MCP_EXPANDED server
+                    // contributes one FIND over its family document matching the
+                    // `## EXEC0` headings: one row per tool — heading, annotation,
+                    // signature — paged like every survey. No document is delivered
+                    // unasked; per-target child documents do not exist.
                     const registry = this.#executors();
                     const toolExpansions = [];
                     for (const tag of registry?.availableRuntimes(workerId) ?? []) {
@@ -751,9 +794,10 @@ class TurnRunner {
                             continue;
                         toolExpansions.push({
                             statement: {
-                                op: "READ", delimiter: "", annotation: `enabled tools: ${tag}`, signal: ["+_plurnk", "+init", "+tools"],
+                                op: "FIND", delimiter: "", annotation: `enabled tools: ${tag}`, signal: ["+_plurnk", "+init", "+tools"],
                                 target: { kind: "url", raw: `worker://~/_plurnk/tools/${tag}.md`, scheme: "worker", username: null, password: null, hostname: "~", port: null, pathname: generatedPathname(`/tools/${tag}.md`), query: null, fragment: null },
-                                body: null, lineMarker: { marks: [1, -1] }, position: UNKNOWN_POSITION,
+                                body: { dialect: "regex", raw: "/^## EXEC0 .*\\n.*$/m", pattern: "^## EXEC0 .*\\n.*$", flags: "m" },
+                                lineMarker: null, position: UNKNOWN_POSITION,
                             },
                         });
                     }
@@ -864,32 +908,18 @@ class TurnRunner {
             // Model operations continue the same turn sequence after these rows.
             let nextActionIndex = 1;
             const turnOpenPaths = [];
-            if (turnNumber === 1 && loopRow?.prompt_published === 0) {
-                const promptRow = loopRow; // {§prompt-entry} — one durable publication per loop
-                if (promptRow !== undefined && typeof promptRow.prompt === "string" && promptRow.prompt.length > 0) {
-                    const openPaths = assertOpenPaths(JSON.parse(promptRow.open_paths), `Loop ${loopId} open_paths`);
-                    const promptLoopSeq = promptRow.sequence;
-                    const promptPath = promptTarget(promptLoopSeq, 1);
-                    const entry = {
-                        channels: { body: { content: promptRow.prompt, mimetype: "text/markdown" } },
-                        attributes: {
-                            openPaths,
-                            ...(promptRow.prompt_source === null ? {} : { source: promptRow.prompt_source }),
-                        },
-                    };
-                    await EntryCrud.writeEntry({ authority: "", pathname: promptPath.pathname }, entry, systemCtx, "prompt", workerId);
-                    turnOpenPaths.push(...openPaths);
-                    const promptLogId = await this.#writePromptLog({
-                        workerId,
-                        loopId,
-                        turnId,
-                        sequence: nextActionIndex++,
-                        target: promptPath,
-                        content: promptRow.prompt,
-                        source: promptRow.prompt_source,
-                    });
-                    onDispatch?.(promptLogId);
-                }
+            if (promptPublication !== null) { // {§prompt-entry} — one durable publication per loop
+                turnOpenPaths.push(...promptPublication.openPaths);
+                const promptLogId = await this.#writePromptLog({
+                    workerId,
+                    loopId,
+                    turnId,
+                    sequence: nextActionIndex++,
+                    target: promptPublication.path,
+                    content: promptPublication.content,
+                    source: promptPublication.source,
+                });
+                onDispatch?.(promptLogId);
             }
             // {§prompt-loop-containment}: the loop contains every prompt that arrived
             // while it ran. Publish each undelivered frame oldest-first exactly once.
@@ -1270,12 +1300,15 @@ class TurnRunner {
                 // runLoop/Daemon settle the exact 504/499 loop result.
                 if (providerSignal?.aborted) {
                     const status = providerSignal.reason === LOOP_TIMEOUT_REASON ? 504 : 499;
-                    await Turn.recordInference(this.#db, turnId, {
-                        packet: StoredPacket.stringify(requestPacket),
-                        usageCurationBudget: this.#packets.curationBudgetFor(provider),
-                        finishReason: splitResponse?.callMetadata.finishReason ?? null,
-                        model: splitResponse?.callMetadata.model ?? provider.model,
-                        meta: JSON.stringify(response?.meta ?? {}),
+                    await this.#recordInference({
+                        workspaceId, workerId, loopId, turnId,
+                        evidence: {
+                            packet: StoredPacket.stringify(requestPacket),
+                            usageCurationBudget: this.#packets.curationBudgetFor(provider),
+                            finishReason: splitResponse?.callMetadata.finishReason ?? null,
+                            model: splitResponse?.callMetadata.model ?? provider.model,
+                            meta: JSON.stringify(response?.meta ?? {}),
+                        },
                     });
                     await Turn.complete(this.#db, turnId, status);
                     throw err;
@@ -1292,12 +1325,15 @@ class TurnRunner {
                 // The provider call was attempted, but no completed exchange exists.
                 // Persist the exact request half and failure status; omitting assistant
                 // is materially different from fabricating an empty model turn.
-                await Turn.recordInference(this.#db, turnId, {
-                    packet: StoredPacket.stringify(requestPacket),
-                    usageCurationBudget: this.#packets.curationBudgetFor(provider),
-                    finishReason: splitResponse?.callMetadata.finishReason ?? null,
-                    model: splitResponse?.callMetadata.model ?? provider.model,
-                    meta: JSON.stringify(response?.meta ?? {}),
+                await this.#recordInference({
+                    workspaceId, workerId, loopId, turnId,
+                    evidence: {
+                        packet: StoredPacket.stringify(requestPacket),
+                        usageCurationBudget: this.#packets.curationBudgetFor(provider),
+                        finishReason: splitResponse?.callMetadata.finishReason ?? null,
+                        model: splitResponse?.callMetadata.model ?? provider.model,
+                        meta: JSON.stringify(response?.meta ?? {}),
+                    },
                 });
                 await Turn.complete(this.#db, turnId, recorded.result.status);
                 if (capacityFailure) {
@@ -1338,20 +1374,31 @@ class TurnRunner {
                             ? { reasoningItems: response.assistant.reasoningEncrypted }
                             : {}),
                     });
+                    // {§invalid-emission-attempts} — the informed turn carries the parser's own
+                    // diagnostic and position: the model sees WHY, not only that it was refused.
+                    const diagnostic = splitResponse.parseErrors[0];
                     this.#notices.push(workspaceId, loopId, {
                         source: "engine:grammar",
                         kind: "invalid_emission",
                         level: "error",
-                        message: INVALID_EMISSION_RECOVERY_MESSAGE,
+                        message: diagnostic === undefined
+                            ? INVALID_EMISSION_RECOVERY_MESSAGE
+                            : `${INVALID_EMISSION_RECOVERY_MESSAGE} Parser: ${diagnostic.message}`,
+                        ...(diagnostic !== undefined && diagnostic.line > 0
+                            ? { position: { type: "content-offset", line: diagnostic.line, column: diagnostic.column } }
+                            : {}),
                     });
                 }
                 const status = allowInvalidEmissionRecovery ? TURN_STATUS_IMPLICIT_CONTINUE : 500;
-                await Turn.recordInference(this.#db, turnId, {
-                    packet: StoredPacket.stringify(requestPacket),
-                    usageCurationBudget: this.#packets.curationBudgetFor(provider),
-                    finishReason: splitResponse.callMetadata.finishReason,
-                    model: splitResponse.callMetadata.model,
-                    meta: JSON.stringify(response.meta ?? {}),
+                await this.#recordInference({
+                    workspaceId, workerId, loopId, turnId,
+                    evidence: {
+                        packet: StoredPacket.stringify(requestPacket),
+                        usageCurationBudget: this.#packets.curationBudgetFor(provider),
+                        finishReason: splitResponse.callMetadata.finishReason,
+                        model: splitResponse.callMetadata.model,
+                        meta: JSON.stringify(response.meta ?? {}),
+                    },
                 });
                 await Turn.complete(this.#db, turnId, status);
                 return {
@@ -1436,14 +1483,17 @@ class TurnRunner {
             // the producer-neutral admitted-turn executor settles every operation
             // and its exact source artifact.
             const packet = StoredPacket.admit(requestPacket, packetAssistant, response.assistantRaw);
-            await Turn.recordInference(this.#db, turnId, {
-                packet: StoredPacket.stringify(packet),
-                usageCurationBudget: this.#packets.curationBudgetFor(provider), // {§tokenomics-client-gauge}
-                finishReason: callMetadata.finishReason,
-                model: callMetadata.model,
-                // Opaque provider metadata plus engine-authored rail keys.
-                // {§meta-passthrough}, {§rail-truth-engine-verdict}
-                meta: JSON.stringify({ ...(response.meta ?? {}), ...(railKeys ?? {}) }),
+            await this.#recordInference({
+                workspaceId, workerId, loopId, turnId,
+                evidence: {
+                    packet: StoredPacket.stringify(packet),
+                    usageCurationBudget: this.#packets.curationBudgetFor(provider), // {§tokenomics-client-gauge}
+                    finishReason: callMetadata.finishReason,
+                    model: callMetadata.model,
+                    // Opaque provider metadata plus engine-authored rail keys.
+                    // {§meta-passthrough}, {§rail-truth-engine-verdict}
+                    meta: JSON.stringify({ ...(response.meta ?? {}), ...(railKeys ?? {}) }),
+                },
             });
             // {§operator-config-workspace-max-commands} — workspace maxCommands
             // narrows the operator ceiling before the admitted program reaches the
@@ -1736,90 +1786,33 @@ class TurnRunner {
                 && ch.channel === this.#schemes.defaultChannelFor(ch.runtime, workerId)
                 ? null
                 : ch.channel;
-            const streamBase = renderAddress({
-                scheme: ch.runtime,
-                authority: ch.authority,
-                pathname: ch.coord,
-            });
-            const streamTarget = visibleFragment === null
-                ? streamBase
-                : `${streamBase}#${PathSyntax.escapeTarget(visibleFragment)}`;
             const targetParts = authorityParts(ch.authority);
-            const cursor = ch.published_end;
-            const closed = ch.state === "closed" || ch.state === "errored";
-            const terminal = closed
-                ? Results.assert(JSON.parse(ch.producer_result ?? "null"))
-                : null;
-            const terminalResult = async (fields, sequence) => {
-                if (terminal === null)
-                    throw new Error(`closed subscription ${ch.subscription_id} has no terminal result`);
-                const result = Results.assert({
-                    ...terminal,
-                    ...(terminal.problem === undefined ? {} : { problem: { ...terminal.problem } }),
-                    ...fields,
-                });
-                if (result.problem !== undefined) {
-                    const seqs = await this.#db.engine_loop_turn_seqs.get({
-                        loop_id: loopId,
-                        turn_id: turnId,
-                    });
-                    if (seqs === undefined)
-                        throw new Error(`stream delta has no log coordinate for loop=${loopId} turn=${turnId}`);
-                    Results.attachInstance(result, `log:///${seqs.loop_seq}/${seqs.turn_seq}/${sequence}/READ`);
-                }
-                return result;
-            };
-            const publishEnd = streamPublicationEnd(ch.content, ch.mimetype, cursor, closed);
-            if (publishEnd <= cursor) {
-                // The cursor-terminal race: a channel written in one final
-                // burst gets fully shown FOLDED while still active; the close then has zero new
-                // content and the auto-OPEN terminal observation never fired — the model was never shown
-                // the conclusion of a stream whose result it already holds folded. The same
-                // observation is required when the channel produced no publishable content:
-                // completion is information independently of payload. A text stream that delivered
-                // content retains its terse revisit marker; an empty text stream and every structured
-                // channel emit a bodyless typed conclusion.
-                if (terminal !== null) {
-                    const sequence = fromSequence + written;
-                    const content = cursor > 0 && baseMimetype(ch.mimetype).startsWith("text/")
-                        ? `[ stream closed (${terminal.status}) - full output already delivered above; READ ${streamTarget} to revisit ]`
-                        : "";
-                    const rx = JSON.stringify(await terminalResult({
-                        content,
-                        mimetype: ch.mimetype,
-                    }, sequence));
-                    await this.#db.engine_insert_stream_delta.run({
-                        worker_id: workerId, loop_id: loopId, turn_id: turnId, sequence,
-                        subscription_publication_id: ch.publication_id,
-                        scheme: ch.runtime, hostname: targetParts.hostname, port: targetParts.port,
-                        pathname: ch.coord, fragment: visibleFragment,
-                        rx,
-                        weight: LogBody.weight({
-                            op: "READ",
-                            attrs: {},
-                            tx: "",
-                            rx,
-                            mimetypeTx: "text/plain",
-                            mimetypeRx: "application/json",
-                        }, this.#weighContent),
-                        status: terminal?.status ?? 200,
-                        attrs: JSON.stringify({ streamEnd: ch.content.length, terminal: true }),
-                        folded: LogVisibility.serialize(LogVisibility.OPEN),
-                    });
-                    written++;
-                }
+            // {§exec-stream} — nothing publishes while a stream is active: the Child Streams
+            // section reports its size and growth ({§child-orientation}); the model READs any
+            // range it wants. At close, ONE foisted READ that is exactly a markerless READ —
+            // the first page, the extent, the terminal status and Problem — born OPEN. {§exec-stream-page}
+            if (ch.state !== "closed" && ch.state !== "errored")
                 continue;
-            }
-            // startLine continues the line count across turns: a multi-turn stream's deltas number
-            // into one sequence (lines N..M, then M+1..), not N independent "1:" restarts. {§exec-stream}
-            const startLine = (ch.content.slice(0, cursor).match(/\n/g)?.length ?? 0) + 1;
+            const terminal = Results.assert(JSON.parse(ch.producer_result ?? "null"));
             const sequence = fromSequence + written;
-            const content = ch.content.slice(cursor, publishEnd);
-            const terminalDelivery = closed && publishEnd === ch.content.length;
-            const fields = { content, mimetype: ch.mimetype, startLine };
-            const result = terminalDelivery
-                ? await terminalResult(fields, sequence)
-                : { status: 200, ...fields };
+            const page = await ReadResolve.resolve({ content: ch.content, mimetype: ch.mimetype, lineMarker: null });
+            const result = Results.assert({
+                ...terminal,
+                ...(terminal.problem === undefined ? {} : { problem: { ...terminal.problem } }),
+                content: page.content ?? "",
+                mimetype: page.mimetype,
+                ...(page.startLine === undefined || page.startLine === null ? {} : { startLine: page.startLine }),
+                ...(page.range === undefined ? {} : { range: page.range }),
+            });
+            if (result.problem !== undefined) {
+                const seqs = await this.#db.engine_loop_turn_seqs.get({
+                    loop_id: loopId,
+                    turn_id: turnId,
+                });
+                if (seqs === undefined)
+                    throw new Error(`stream delta has no log coordinate for loop=${loopId} turn=${turnId}`);
+                Results.attachInstance(result, `log:///${seqs.loop_seq}/${seqs.turn_seq}/${sequence}/READ`);
+            }
             const rx = JSON.stringify(result);
             await this.#db.engine_insert_stream_delta.run({
                 worker_id: workerId, loop_id: loopId, turn_id: turnId, sequence,
@@ -1835,9 +1828,9 @@ class TurnRunner {
                     mimetypeTx: "text/plain",
                     mimetypeRx: "application/json",
                 }, this.#weighContent),
-                status: result.status,
-                attrs: JSON.stringify({ streamEnd: publishEnd, terminal: terminalDelivery }),
-                folded: LogVisibility.serialize(terminalDelivery ? LogVisibility.OPEN : LogVisibility.FOLDED), // {§exec-stream} — terminal observation auto-OPENs; ongoing folds
+                status: terminal.status,
+                attrs: JSON.stringify({ streamEnd: ch.content.length, terminal: true }),
+                folded: LogVisibility.serialize(LogVisibility.OPEN), // {§exec-stream} — the conclusion is born OPEN
             });
             written++;
         }