@plurnk/plurnk-service 0.54.0 → 0.56.0

package/dist/core/Engine.js

@@ -9,7 +9,7 @@ import GitState from "./git-state.js";
 import Fork from "./fork.js";
 import RunCap from "./run-cap.js";
 import { teachingLine, docsExcludeSet } from "./teaching.js";
-import { readPacketInject } from "./packet-inject.js";
+import { readPacketInject, readSystemPolicy, readProjectPolicy } from "./packet-inject.js";
 import SessionSettings from "./session-settings.js";
 import { decodePathParens } from "./path-decode.js";
 import { DEFAULT_LOOP_FLAGS } from "./scheme-types.js";
@@ -315,6 +315,9 @@ class Engine {
             // #263 — the last turn's prompt tokens = current window occupancy (gauge numerator), NOT the
             // summed promptTokens above, which overcounts a context that grows across turns.
             contextTokens: row?.context ?? 0,
+            // #252 — the latest turn's opaque provider blob, parsed for the wire. Empty {} when the
+            // provider returned no meta. The service forwards it; it never reads a field within.
+            meta: JSON.parse(row?.meta ?? "{}"),
         };
     }
     #pushTelemetry(sessionId, loopId, event) {
@@ -357,6 +360,26 @@ class Engine {
         }
         return slice.join("\n");
     }
+    // A @plurnk/gbnf divergence position (providers#24) is a CODE-POINT offset into the
+    // model's content; the snippet/telemetry surface speaks 1-based line + 0-based column.
+    // Convert over code points (not UTF-16 units) so an astral char doesn't skew the line,
+    // clamping out-of-range offsets to the content's end.
+    #offsetToLineColumn(content, offset) {
+        const cps = Array.from(content);
+        const clamped = Math.max(0, Math.min(offset, cps.length));
+        let line = 1;
+        let column = 0;
+        for (let i = 0; i < clamped; i++) {
+            if (cps[i] === "\n") {
+                line++;
+                column = 0;
+            }
+            else {
+                column++;
+            }
+        }
+        return { line, column };
+    }
     async runLoop({ provider, messages, requirements = "", sessionId, runId, loopId, maxTurns = 50, maxStrikes = readMaxStrikes(), minCycles = readPositiveInt("PLURNK_MIN_CYCLES", DEFAULT_MIN_CYCLES), maxCyclePeriod = readPositiveInt("PLURNK_MAX_CYCLE_PERIOD", DEFAULT_MAX_CYCLE_PERIOD), origin = "model", signal, onDispatch, }) {
         const turnIds = [];
         const suddenDeathThreshold = maxTurns - maxStrikes;
@@ -611,7 +634,7 @@ class Engine {
         // (clamped to the scheme's count so FIND's strict <L> never 416s); off by default.
         if (seq === 1) {
             // #231 — a session's client-chosen manifestItems REPLACES the env default outright.
-            const { manifestItems: sessionMI, autoReadAgents } = await SessionSettings.read(this.#db, sessionId);
+            const { manifestItems: sessionMI } = await SessionSettings.read(this.#db, sessionId);
             const manifestItems = sessionMI !== null ? normalizeManifestItems(sessionMI) : readManifestItems();
             if (manifestItems !== null && runFirstLoop) { // #269 — catalog preview is run-once
                 // engine_scheme_catalog_summary is the scheme source: session-scoped, ordered,
@@ -656,31 +679,19 @@ class Engine {
                     });
                     nextActionIndex++;
                 }
-            }
-            // #268 — auto-READ the configured AGENTS file(s) into THIS first model turn. Env-driven
-            // (PLURNK_AGENTS_AUTO / PLURNK_AGENTS_FILES), overridable per-session by autoReadAgents; the
-            // matching files are auto-PICKed into membership at session setup (envelope). The model sees
-            // only the READ — a normal file:/// member READ, read-write so it edits the scratchpad back.
-            const { auto: agentsAuto, files: agentsFiles } = SessionSettings.resolveAgentsAutoload(autoReadAgents);
-            if (agentsAuto && runFirstLoop) { // #269 — run-once, the run's first loop
-                for (const file of agentsFiles) {
-                    const pathname = file.startsWith("/") ? file : `/${file}`;
-                    const member = await this.#db.crud_get_member_sig.get({ session_id: sessionId, scheme: null, pathname });
-                    if (member === undefined)
-                        continue; // absent / non-git session → not a member → skip
-                    const agentsRead = {
-                        op: "READ", suffix: "", signal: null, lineMarker: null,
-                        target: {
-                            kind: "url", raw: `file://${pathname}`, scheme: "file",
-                            username: null, password: null, hostname: null, port: null,
-                            pathname, params: {}, fragment: null,
-                        },
-                        body: null, position: { line: 1, column: 1 },
+                // §run-scheme — Manifest(run) = session-scope ∪ THIS run's run-scope. Foist the
+                // building run's OWN scratch (run:///**, uncapped — a run needs the full view to
+                // manage its private workspace) so it's catalogued in ITS perspective alone; other
+                // runs reach it only via explicit FIND(run://<name>/**). A run with no scratch foists nothing.
+                const selfRun = await this.#db.run_name_by_id.get({ run_id: runId });
+                const scratch = selfRun === undefined ? 0 : (await this.#db.engine_run_scratch_count.get({ session_id: sessionId, owner_prefix: `/${selfRun.name}/*` }))?.entries ?? 0;
+                if (scratch > 0) {
+                    const runFind = {
+                        op: "FIND", suffix: "", signal: null,
+                        target: { kind: "url", raw: "run:///**", scheme: "run", username: null, password: null, hostname: "", port: null, pathname: "/**", params: {}, fragment: null },
+                        body: null, lineMarker: null, position: { line: 1, column: 1 },
                     };
-                    await this.dispatch({
-                        statement: agentsRead, sessionId, runId, loopId, turnId,
-                        sequence: nextActionIndex, origin: "plurnk", onDispatch,
-                    });
+                    await this.dispatch({ statement: runFind, sessionId, runId, loopId, turnId, sequence: nextActionIndex, origin: "plurnk", onDispatch });
                     nextActionIndex++;
                 }
             }
@@ -740,17 +751,18 @@ class Engine {
             await this.#db.engine_close_turn.run({
                 id: turnId, status: 413, packet: JSON.stringify(hardPacket),
                 usage_prompt: 0, usage_completion: 0, usage_cached: 0, usage_cost_pico: 0,
-                finish_reason: "budget_hard_stop", model: provider.model,
+                finish_reason: "budget_hard_stop", model: provider.model, meta: "{}",
             });
             return { turnId, status: 413, statuses: [], fingerprint: "", budgetStruck: enforced.struck, budgetHardStop: true, steerStruck: false };
         }
         const modelMessages = this.#packetToWireMessages(requestPacket);
-        // maxTokens = remaining context window (loop policy, plurnk-providers#10).
-        // The 0.28.0 EOS-forcing root terminates the turn at the status SEND, but a
-        // grammar can't bound degeneration *inside* a statement body — this caps the
-        // decode at the free window so a runaway can't reach the context wall.
-        const genCeiling = _a.computeCeiling(provider.contextSize, this.#budgetCeiling); // provider.contextSize, the immutable identity, read by the budget — §provider-surface-identity
-        const maxTokens = genCeiling === null ? undefined : Math.max(1, genCeiling - requestPacket.tokens);
+        // No decode cap. Our budget governs the TRANSMISSION packet (the grinder folds
+        // the input under the ceiling); the model's decode — reasoning + emission — is
+        // out of band, owned by the provider's own context window. Deriving a maxTokens
+        // from our budget conflated the two and guillotined a reasoning model's
+        // out-of-band thinking as the packet filled (`ceiling - packet` → near-zero
+        // decode → finish=length mid-reasoning → no emission → strike spiral). The
+        // provider enforces its physical wall on its own.
         let response;
         // #249 — plugin attribution tags onto the per-turn generate() wire. Value is the
         // active-plugin set (placeholder); real per-turn grounding is deferred.
@@ -761,7 +773,7 @@ class Engine {
         // #249 — session-stable frontend id, forwarded as Plurnk-Client by the plurnk provider only.
         const { client } = await SessionSettings.read(this.#db, sessionId);
         try {
-            response = await provider.generate({ messages: modelMessages, runId: String(runId), signal, grammar: await this.#grammarConstraint(), maxTokens, attributions: attributions.length > 0 ? attributions : undefined, client: client ?? undefined }); // §provider-surface-generate §provider-guarantees-single-call §provider-guarantees-signal-wired §attribution-plurnk-namespace-reserved §client-telemetry
+            response = await provider.generate({ messages: modelMessages, runId: String(runId), signal, grammar: await this.#grammarConstraint(), attributions: attributions.length > 0 ? attributions : undefined, client: client ?? undefined }); // §provider-surface-generate §provider-guarantees-single-call §provider-guarantees-signal-wired §attribution-plurnk-namespace-reserved §client-telemetry
         }
         catch (err) {
             // Every provider error surfaces as telemetry (the client/model sees the cause). #256:
@@ -770,6 +782,9 @@ class Engine {
             // accepted: fall through as an empty no-op turn so the strike rail retries. Every other
             // kind (rate_limit, network_failure, unauthorized, …) is terminal — telemetry'd, then
             // propagated to end the loop (rather than only the opaque loop.run rejection).
+            // NOTE (providers 0.19.0 / #275): only the CONSTRAINED path still throws grammar_unenforced.
+            // In GBNF-filter mode the provider returns the bytes with a grammar_unenforced telemetry
+            // event instead — recovered on the success path below (response.telemetry), no empty turn.
             if (err instanceof ProviderError) {
                 this.#pushTelemetry(sessionId, loopId, { source: "provider", kind: err.kind, message: err.message });
                 if (err.kind !== "grammar_unenforced")
@@ -801,14 +816,28 @@ class Engine {
         // the snippet, the model sees "invalid xpath at 1:0" but can't
         // connect that to what IT wrote — and tends to regenerate the
         // same broken emission. See edit-todo demo for the canonical case.
-        for (const { message, line, column, source } of parseErrors ?? []) {
+        // Parse errors are LOG ITEMS now (§telemetry — one budget surface): each failed-to-parse
+        // emission records an actionless `error` row below, after the turn's dispatched ops are
+        // sequenced (see the parse-error log write past the dispatch loop). The errors section
+        // derives a pointer to it from log≥400, uniform with action_failure.
+        // providers#24 / #275: non-fatal provider telemetry on a SUCCESSFUL turn. In GBNF-filter
+        // mode the provider no longer THROWS grammar_unenforced — it returns the model's bytes
+        // (here, packetAssistant.content) and attaches the conflict as a telemetry event carrying
+        // the divergence code-point position. Mirror the parse_error path: forward each event, and
+        // when it locates a position, render the model its own emission around that line so it can
+        // self-correct — the exact recovery affordance parse errors already get, instead of the
+        // empty-turn cascade the old throw produced.
+        for (const event of response.telemetry ?? []) {
+            const located = typeof event.position === "number"
+                ? this.#offsetToLineColumn(packetAssistant.content, event.position)
+                : null;
             this.#pushTelemetry(sessionId, loopId, {
-                source: "grammar",
-                kind: "parse_error",
-                message,
-                position: { type: "content-offset", line, column },
-                snippet: this.#extractSnippet(packetAssistant.content, line, 2),
-                parserSource: source,
+                source: event.source,
+                kind: event.kind,
+                message: event.message ?? "",
+                ...(located !== null
+                    ? { position: { type: "content-offset", line: located.line, column: located.column }, snippet: this.#extractSnippet(packetAssistant.content, located.line, 2) }
+                    : {}),
             });
         }
         const opsCount = packetAssistant.ops.length;
@@ -822,19 +851,22 @@ class Engine {
         // §grinder-strike-coupling): the loop continues, the model sees the steering hint not the strike
         // count, and a non-resolver spins out to the engine's 500.
         let steerStruck = false;
-        // Premature terminate: a SEND[200] while the run still holds a live stream/spawn — the model
-        // declared done with work running. Downgrade the 200 to 102 so it dispatches as a continue (its
-        // body is preserved, not discarded) and steer; the stream's own conclusion or a KILL is the exit.
+        // Premature terminate: a SEND[200] while the run still holds a live thing — an open stream/spawn
+        // OR a non-terminal child run (§run-lifecycle: children and streams are the same kind of "live
+        // thing a run holds"). The model declared done with work running. Downgrade the 200 to 102 so it
+        // dispatches as a continue (its body is preserved, not discarded) and steer; the stream's/child's
+        // own conclusion (the wake edge) or a KILL is the exit.
         if (sendOp?.signal === 200) {
             const openSubs = await this.#db.find_open_subscriptions_for_run.all({ run_id: runId });
             const execHandler = this.#schemes.get("exec");
-            if (openSubs.length > 0 || execHandler?.hasActiveSpawns?.(runId) === true) {
+            const liveChild = await this.#db.engine_run_has_live_child.get({ run_id: runId });
+            if (openSubs.length > 0 || execHandler?.hasActiveSpawns?.(runId) === true || liveChild !== undefined) {
                 sendOp.signal = TURN_STATUS_IMPLICIT_CONTINUE; // 102 — downgraded, no longer a terminal
                 steerStruck = true;
                 this.#pushTelemetry(sessionId, loopId, {
                     source: "engine:rail",
                     kind: "premature_terminate",
-                    message: "Attempted termination with active streams. Terminate with 202 to hibernate until stream completion, KILL(path) with 200 again to clean up, or 499 to fail.",
+                    message: "Attempted termination with active streams or child runs. Terminate with 202 to hibernate until they complete, KILL(path) with 200 again to clean up, or 499 to fail.",
                 });
             }
         }
@@ -867,6 +899,9 @@ class Engine {
             usage_cost_pico: provider.costFor(usage), // §provider-surface-costfor
             finish_reason: finishReason,
             model,
+            // #252 — opaque provider→client metadata passthrough (e.g. balancePico the
+            // provider normalized). Stored verbatim, unenforced; the service never reads a field.
+            meta: JSON.stringify(response.meta ?? {}),
         });
         // Dispatch model ops starting at nextActionIndex (continues the
         // turn's running counter after any pre-model writes).
@@ -910,6 +945,27 @@ class Engine {
                 dropped: droppedCount,
             });
         }
+        // §telemetry — parse errors as LOG ITEMS: a failed-to-parse emission records an actionless
+        // `error` row (status 400, no target, snippet = the foldable body) at the turn's next free
+        // sequence (after the dispatched ops). The model folds/kills/recalls it like any log entry,
+        // and the errors section derives a pointer (status + coordinate) from log≥400 — one surface.
+        let errSeq = nextActionIndex + opsToDispatch.length;
+        for (const { message, line, column, source } of parseErrors ?? []) {
+            const snippet = this.#extractSnippet(packetAssistant.content, line, 2);
+            await this.#db.engine_insert_log_entry.get({
+                run_id: runId, loop_id: loopId, turn_id: turnId, sequence: errSeq++,
+                origin: "model", source: "grammar", op: "error", suffix: "", signal: null,
+                scheme: null, username: null, password: null, hostname: null, port: null,
+                pathname: null, params: null, fragment: null, lineMarker: null,
+                tx: "", mimetype_tx: "text/plain",
+                // `message`/`snippet` render as the foldable body (not a pointer `error` field), so the
+                // derived errors-section pointer stays minimal (status + coordinate), the bloated parser
+                // message lives in the log row, reclaimable on FOLD.
+                rx: JSON.stringify({ message, position: { type: "content-offset", line, column }, snippet, parserSource: source }),
+                mimetype_rx: "application/json",
+                status_rx: 400, tokens: 0, state: "resolved", outcome: null, attrs: "{}",
+            });
+        }
         // Zero ops is NOT an error to report — the model knows it emitted
         // nothing. Strike accounting (engine-internal) treats it as a
         // struck turn; the model just sees an empty packet next turn.
@@ -1032,15 +1088,23 @@ class Engine {
         // peer sections (unbundled). The budget section carries its {{tokensFree}}
         // placeholders here; they resolve below once the assembled total is known.
         const inject = await readPacketInject(); // #240 — operator section, per-turn, fail-hard on a broken path
+        const sessionRoot = (await this.#db.envelope_get_session.get({ id: sessionId }))?.project_root ?? null;
+        const systemPolicy = await readSystemPolicy(); // ~/.plurnk/AGENTS.md (or PLURNK_POLICY)
+        const projectPolicy = await readProjectPolicy(sessionRoot); // <projectRoot>/AGENTS.md (or PLURNK_PROJECT)
         const defaults = [
             { name: "definition", slot: "system", header: null, content: system_definition, tokens: 0 },
             { name: "tools", slot: "system", header: null, content: tools.join("\n"), tokens: 0 }, // titleless — the examples flow on from plurnk.md (definition) directly above
             { name: "schemes", slot: "system", header: "Plurnk System Schemes", content: this.#schemes.teach(), tokens: 0 },
             ...(inject !== null ? [{ name: "inject", slot: "system", header: "Plurnk Operator Notes", content: inject, tokens: 0 }] : []),
-            { name: "log", slot: "system", header: "Plurnk System Log", content: PacketWire.renderLog(log, countTokens), tokens: 0 },
+            // policy: the client's privileged rules — ~/.plurnk/AGENTS.md (system) then <root>/AGENTS.md (project) — below grammar/tools/schemes, above budget-the-law. AGENTS is POLICY here, never a curatable READable entry. Empty content ⇒ section omitted.
+            { name: "system-policy", slot: "system", header: "Plurnk System Policy", content: systemPolicy ?? "", tokens: 0 },
+            { name: "project-policy", slot: "system", header: "Project Policy", content: projectPolicy ?? "", tokens: 0 },
+            // budget at the BOTTOM of the SYSTEM slot — budget is LAW (the ceiling is a hard constraint the model must obey), so it belongs in the lean privileged zone, not as user-slot status.
+            { name: "budget", slot: "system", header: "Plurnk System Budget", content: budgetReadout, tokens: 0 },
             { name: "prompt", slot: "user", header: "Plurnk System User Prompt", content: prompt, tokens: 0 },
-            { name: "budget", slot: "user", header: "Plurnk System Budget", content: budgetReadout, tokens: 0 },
             { name: "errors", slot: "user", header: "Plurnk System Errors", content: PacketWire.renderErrors(telemetryErrors), tokens: 0 },
+            // log in the USER slot, low: short-term memory (data, not rules) at the action point, so the model consults its history instead of skimming it. git follows it as workspace status — not law, so it stays in user, never system.
+            { name: "log", slot: "user", header: "Plurnk System Log", content: PacketWire.renderLog(log, countTokens), tokens: 0 },
             { name: "git", slot: "user", header: "Plurnk System Git Status", content: PacketWire.renderGit(gitStatus), tokens: 0 },
             { name: "requirements", slot: "user", header: "Plurnk System Requirements", content: baseRequirements, tokens: 0 },
         ];
@@ -1894,8 +1958,9 @@ class Engine {
         const schemeName = this.#schemeNameOf(path);
         if (schemeName === null)
             return { status: 400, error: "KILL target must be a URL path with a scheme" };
-        if (schemeName === "log")
-            return { status: 405, error: "log:/// is append-only; KILL must bounce" };
+        // KILL on log:/// erases the log row(s) — the model's DB-storage curation lever
+        // (plurnk.md:36, :98), routed to Log.kill below via the killable.kill path. The old
+        // "append-only" 405 forbade what the grammar requires; FOLD only collapses the render.
         // Process-KILL: any scheme whose handler exposes kill() aborts a live stream — the
         // exec handler, registered as "exec" + under every runtime tag (sh/node), so a tag-
         // addressed stream (sh:///l/t/s) routes here, not to deleteEntry. §exec