@plurnk/plurnk-service 0.42.0 → 0.44.0

package/dist/core/Engine.js

@@ -4,6 +4,7 @@ import { Mimetypes, emptyRegistry } from "@plurnk/plurnk-mimetypes";
 import EntryCrud from "../schemes/_entry-crud.js";
 import EntryManifest from "../schemes/_entry-manifest.js";
 import GitMembership from "./git-membership.js";
+import { foldAuthorityIntoPath, renderAddress } from "./plurnk-uri.js";
 import GitState from "./git-state.js";
 import Fork from "./fork.js";
 import RunCap from "./run-cap.js";
@@ -538,10 +539,10 @@ class Engine {
             const promptRow = await this.#db.engine_get_loop_prompt.get({ loop_id: loopId });
             if (promptRow !== undefined && typeof promptRow.prompt === "string" && promptRow.prompt.length > 0) {
                 const promptPath = {
-                    kind: "url", raw: `plurnk:///prompt/${loopId}/${seq}`,
+                    kind: "url", raw: `plurnk://prompt/${loopId}/${seq}`,
                     scheme: "plurnk", username: null, password: null,
-                    hostname: null, port: null,
-                    pathname: `/prompt/${loopId}/${seq}`, params: {}, fragment: null,
+                    hostname: "prompt", port: null,
+                    pathname: `/${loopId}/${seq}`, params: {}, fragment: null,
                 };
                 const promptStmt = {
                     op: "EDIT", suffix: "", signal: null,
@@ -554,9 +555,9 @@ class Engine {
                     sequence: nextActionIndex, origin: "plurnk",
                     onDispatch: (id) => { promptLogId = id; onDispatch?.(id); },
                 });
-                // §prompt-fold (User Note 6): the prompt EDIT duplicates
-                // packet.user.prompt (its own section), so fold it — logged for
-                // forensics, collapsed in the model's log, re-OPENable.
+                // §prompt-fold (User Note 6): the prompt EDIT duplicates the
+                // prompt section, so fold it — logged for forensics, collapsed
+                // in the model's log, re-OPENable.
                 if (promptLogId !== undefined)
                     await this.#db.engine_fold_log_entry.run({ id: promptLogId });
                 nextActionIndex++;
@@ -659,7 +660,7 @@ class Engine {
         // grammar can't bound degeneration *inside* a statement body — this caps the
         // decode at the free window so a runaway can't reach the context wall.
         const genCeiling = _a.computeCeiling(provider.contextSize, this.#budgetCeiling); // provider.contextSize, the immutable identity, read by the budget — §provider-surface-identity
-        const maxTokens = genCeiling === null ? undefined : Math.max(1, genCeiling - requestPacket.system.tokens - requestPacket.user.tokens);
+        const maxTokens = genCeiling === null ? undefined : Math.max(1, genCeiling - requestPacket.tokens);
         const response = await provider.generate({ messages: modelMessages, runId: String(runId), signal, grammar: await this.#grammarConstraint(), maxTokens }); // §provider-surface-generate §provider-guarantees-single-call §provider-guarantees-signal-wired
         // Engine splits wire-level response: emission (content, reasoning,
         // parsed ops) → packet.assistant per Packet.json §assistant;
@@ -849,18 +850,19 @@ class Engine {
     // the stored packet and the wire payload share one source of truth.
     async #buildRequestPacket({ initialMessages, requirements, runId, loopId, currentTurnSeq, provider, gitStatus, telemetryErrors: presetTelemetry, }) {
         const byRole = (role) => initialMessages.filter((m) => m.role === role).map((m) => m.content).join("\n\n");
-        // plurnk.md (grammar/dialects) THEN the scheme catalogue: grammar 0.49+ is
-        // scheme-agnostic, so the service teaches what schemes exist + what they do
-        // at packet-time (grammar#239 item 7). SchemeRegistry.teach() assembles it.
-        const system_definition = `${byRole("system")}\n\n${this.#schemes.teach()}`;
-        // user.prompt sources from the loop's most recent prompt entry first
+        // plurnk.md (grammar/dialects only — grammar 0.49+ is scheme-agnostic). The
+        // scheme catalogue is now its OWN section (`schemes`, below tools), so the
+        // definition is just the operator's system prompt — tools sits right below it.
+        const system_definition = byRole("system");
+        // the prompt section sources from the loop's most recent prompt entry first
         // (plurnk:///prompt/<loop_id>/<N> for the highest N written to date).
         // This is what inject + the turn-1 foist write into. Falls back to
         // the runLoop caller's messages.user for tests that bypass the
         // foist mechanism entirely.
-        const latestPromptRow = await this.#db.drain_get_latest_prompt_body_for_loop.get({ pattern: `prompt/${loopId}/%` });
+        const latestPromptRow = await this.#db.drain_get_latest_prompt_body_for_loop.get({ pattern: `/prompt/${loopId}/%` });
+        const promptCap = Number.parseInt(process.env.PLURNK_PROMPT_PREVIEW_CHARS ?? "", 10);
         const prompt = (latestPromptRow !== undefined && typeof latestPromptRow.content === "string" && latestPromptRow.content.length > 0)
-            ? latestPromptRow.content
+            ? PacketWire.previewPrompt(latestPromptRow.content, renderAddress("plurnk", latestPromptRow.pathname), Number.isInteger(promptCap) ? promptCap : -1)
             : byRole("user");
         // Requirements is engine-sourced, NOT threaded from callers — that threading is
         // exactly how it went missing (callers read the sysprompt but never the
@@ -874,65 +876,83 @@ class Engine {
         const requirementsText = `Syntax: <<OPsuffix[signal]?(target)?<Line/Result>?:body?:OPsuffix\n\n${planDirective}${baseRequirements}`;
         const log = await this.#buildLog(runId);
         const telemetryErrors = presetTelemetry ?? await this.#buildTelemetryErrors(loopId, currentTurnSeq);
-        // Per-section render-cost subtotals via provider's tokenizer.
-        // Engine approximates each section by tokenizing its serialized
-        // form — wire-payload tokens may differ slightly because chat-
-        // template scaffolding adds bytes, but the subtotal tracks "what
-        // the model has to process" closely enough for budget diagnostics.
         const countTokens = (t) => provider.countTokens(t); // §provider-surface-counttokens
-        // Budget readout (SPEC.md §tokenomics). Two-pass: measure the wire-rendered
-        // index/log sections (budget-independent), install the readout with a
-        // tokensFree placeholder, measure the assembled total, resolve free,
-        // substitute. Subtotals come from the real render — meta and fences
-        // included — not a serialized approximation. ceiling is the provider's
-        // window × PLURNK_BUDGET_CEILING (null when no window is reported →
-        // headline omitted, section lines still shown).
+        const tools = this.#collectTools();
+        // Budget readout (SPEC.md §tokenomics). Two-pass: render the budget from
+        // the structured log's subtotals with a {{tokensFree}} placeholder, build
+        // the section list, measure the assembled total, resolve free, substitute.
+        // Subtotals come from the real log render — meta and fences included — not
+        // a serialized approximation. ceiling is the provider's window ×
+        // PLURNK_BUDGET_CEILING (null when no window is reported → headline
+        // omitted, section lines still shown). §tokenomics-render-weight-budget
         const ceiling = _a.computeCeiling(provider.contextSize, this.#budgetCeiling);
-        const scratch = {
-            system: { system_definition, log },
-            user: { prompt, telemetry: { budget: "", errors: telemetryErrors, git: gitStatus }, tools: this.#collectTools(), system_requirements: requirementsText },
-        };
-        const sections = PacketWire.measureBudgetSections(scratch, countTokens);
-        scratch.user.telemetry.budget = this.#renderBudget(sections, ceiling);
-        const total = countTokens(PacketWire.renderSystemContent(scratch.system)) + countTokens(PacketWire.renderUserContent(scratch.user));
+        const budgetReadout = this.#renderBudget(PacketWire.measureLogBudget(log, countTokens), ceiling);
+        // The default packet: an ordered list of sections, each addressable state
+        // (§packet-construction). `slot` is the prompt-cache boundary; the STATIC
+        // sections (definition, tools, schemes) lead the system slot so they form the
+        // cached prefix, with the dynamic log after. In the user slot, requirements renders
+        // last (the contract closest to the assistant turn); budget/errors/git are
+        // peer sections (unbundled). The budget section carries its {{tokensFree}}
+        // placeholders here; they resolve below once the assembled total is known.
+        const defaults = [
+            { name: "definition", slot: "system", header: null, content: system_definition, tokens: 0 },
+            { name: "tools", slot: "system", header: "Plurnk System Tools", content: tools.join("\n"), tokens: 0 },
+            { name: "schemes", slot: "system", header: "Plurnk System Schemes", content: this.#schemes.teach(), tokens: 0 },
+            { name: "log", slot: "system", header: "Plurnk System Log", content: PacketWire.renderLog(log), tokens: 0 },
+            { name: "prompt", slot: "user", header: "Plurnk System User Prompt", content: prompt, tokens: 0 },
+            { name: "budget", slot: "user", header: "Plurnk System Budget", content: budgetReadout, tokens: 0 },
+            { name: "errors", slot: "user", header: "Plurnk System Errors", content: PacketWire.renderErrors(telemetryErrors), tokens: 0 },
+            { name: "git", slot: "user", header: "Plurnk System Git Status", content: PacketWire.renderGit(gitStatus), tokens: 0 },
+            { name: "requirements", slot: "user", header: "Plurnk System Requirements", content: requirementsText, tokens: 0 },
+        ];
+        // Plugin packet control (§packet-construction): trusted schemes rewrite the
+        // default list — add, remove, reorder — in-process, before measurement.
+        const sections = await this.#schemes.transformSections(defaults);
+        // Pass 1: measure the assembled total with the placeholder budget in
+        // place, resolve free/percent, substitute into the budget section.
+        const total = countTokens(PacketWire.renderSlot(sections, "system")) + countTokens(PacketWire.renderSlot(sections, "user"));
         const tokensFree = ceiling === null ? null : Math.max(0, ceiling - total); // free floors at 0 on overshoot — §tokenomics-over-budget-floor
         const percent = ceiling === null ? null : Math.round((total / ceiling) * 100); // usage as % of the ceiling — §tokenomics-context-percent
-        const budget = tokensFree === null
-            ? scratch.user.telemetry.budget
-            : scratch.user.telemetry.budget
-                .replace(TOKEN_USAGE_PLACEHOLDER, String(total))
-                .replace(TOKEN_PERCENT_PLACEHOLDER, String(percent))
-                .replace(TOKENS_FREE_PLACEHOLDER, String(tokensFree));
-        const system = { tokens: 0, system_definition, log };
-        const user = { tokens: 0, prompt, telemetry: { budget, errors: telemetryErrors, git: gitStatus }, tools: scratch.user.tools, system_requirements: requirementsText };
-        system.tokens = countTokens(PacketWire.renderSystemContent(system));
-        user.tokens = countTokens(PacketWire.renderUserContent(user));
-        return { system, user };
+        if (tokensFree !== null) {
+            const budgetSec = sections.find((s) => s.name === "budget"); // a plugin may have removed it
+            if (budgetSec) {
+                budgetSec.content = budgetSec.content
+                    .replace(TOKEN_USAGE_PLACEHOLDER, String(total))
+                    .replace(TOKEN_PERCENT_PLACEHOLDER, percent === 0 && total > 0 ? "<1" : String(percent))
+                    .replace(TOKENS_FREE_PLACEHOLDER, String(tokensFree));
+            }
+        }
+        // Pass 2: per-section render-weight + the assembled packet total (post
+        // substitution — the placeholder/number length delta is negligible).
+        for (const s of sections)
+            s.tokens = countTokens(PacketWire.renderSection(s));
+        const packetTokens = countTokens(PacketWire.renderSlot(sections, "system")) + countTokens(PacketWire.renderSlot(sections, "user"));
+        return { tokens: packetTokens, sections, telemetryErrors };
     }
     // Budget readout body, rendered into the `# Plurnk System Budget` section.
     // Headline `ceiling/free` only when a ceiling exists; section lines for the
     // curatable index/log weight the model can FOLD back. tokensFree is a
     // placeholder here — buildSystem substitutes it after measuring the packet.
-    #renderBudget(sections, ceiling) {
+    #renderBudget(log, ceiling) {
         const lines = [];
         if (ceiling !== null)
             lines.push(`ceiling ${ceiling} · usage ${TOKEN_USAGE_PLACEHOLDER} (${TOKEN_PERCENT_PLACEHOLDER}%) · free ${TOKENS_FREE_PLACEHOLDER}`);
-        if (sections.log.entries > 0) {
-            lines.push(`Log entries: ${sections.log.entries} entries, ${sections.log.tokens} tokens`);
+        if (log.entries > 0) {
+            lines.push(`Log entries: ${log.entries} entries, ${log.tokens} tokens`);
             // Per-turn weight — the grinder's rollback unit, oldest first: the
             // model sees what's first to go (§tokenomics {§tokenomics-turn-totals}).
-            if (sections.log.byTurn.length > 0) {
+            if (log.byTurn.length > 0) {
                 lines.push("Turns:", "| turn | tokens |", "|---|--:|");
-                for (const t of sections.log.byTurn)
+                for (const t of log.byTurn)
                     lines.push(`| ${t.turn} | ${t.tokens} |`);
             }
             // The heaviest individual log items — the FOLD targets behind the weight
             // (§tokenomics {§tokenomics-largest-entries}). "items", not "entries": the readout
             // lists log:/// rows (log items), distinct from catalog entries (plurnk.md: "EDIT
             // is only for entries. Do not attempt to edit log items.").
-            if (sections.log.largest.length > 0) {
+            if (log.largest.length > 0) {
                 lines.push("Heaviest items:", "| item | tokens |", "|---|--:|");
-                for (const e of sections.log.largest)
+                for (const e of log.largest)
                     lines.push(`| ${e.path} | ${e.tokens} |`);
             }
         }
@@ -958,16 +978,16 @@ class Engine {
                 const entry = this.#executors.entry(tag);
                 if (entry?.example)
                     tools.push(`* ${entry.example}`);
-                // #note12 — link the executor's fuller doc (materialized at plurnk:///docs/<tag>);
+                // #note12 — link the executor's fuller doc (materialized at plurnk:///docs/<tag>.md);
                 // its token cost rides that manifest entry, so no inline recount here.
                 if (entry?.documentation)
-                    tools.push(`* docs for ${tag}: plurnk:///docs/${tag}`);
+                    tools.push(`* docs for ${tag}: plurnk://docs/${tag}.md`);
             }
         }
         return tools;
     }
     // #note12 — the daughter-provided reference docs (schemes' + execs' `documentation`),
-    // materialized at plurnk:///docs/<name> by loop_run (like operator docs) so the
+    // materialized at plurnk:///docs/<name>.md by loop_run (like operator docs) so the
     // catalogue's doc-links READ and the manifest carries each doc's token cost.
     docEntries() {
         const out = this.#schemes.docs();
@@ -988,7 +1008,7 @@ class Engine {
     // §grinder-overflow-only — fires only on actual overflow, never speculatively
     async #enforceBudget({ packet, provider, runId, loopId, turnId, sessionId, turnNumber, rebuild }) {
         const ceiling = _a.computeCeiling(provider.contextSize, this.#budgetCeiling);
-        const measure = (p) => p.system.tokens + p.user.tokens;
+        const measure = (p) => p.tokens;
         if (ceiling === null || measure(packet) <= ceiling)
             return { packet, fit: true, struck: false };
         const folded = new Map();
@@ -1000,7 +1020,7 @@ class Engine {
             note(le.scheme ?? "log");
         if (priorLogs.length > 0)
             await this.#db.engine_grinder_fold_prior_turn_logs.run({ loop_id: loopId, turn_id: turnId });
-        const errors = packet.user.telemetry.errors;
+        const errors = packet.telemetryErrors;
         let current = priorLogs.length > 0 ? await rebuild(errors) : packet;
         if (measure(current) <= ceiling) {
             this.#emitBudgetOverflow(sessionId, loopId, folded);
@@ -1037,9 +1057,9 @@ class Engine {
     #completePacket(requestPacket, assistant, assistantRaw, provider) {
         const assistantTokens = provider.countTokens(assistant.content);
         return {
-            tokens: requestPacket.system.tokens + requestPacket.user.tokens + assistantTokens,
-            system: requestPacket.system,
-            user: requestPacket.user,
+            tokens: requestPacket.tokens + assistantTokens,
+            sections: requestPacket.sections,
+            telemetryErrors: requestPacket.telemetryErrors,
             assistant,
             assistantRaw,
         };
@@ -1074,7 +1094,7 @@ class Engine {
         });
         return [...this.#drainTelemetry(loopId), ...actionFailures];
     }
-    // SPEC §packet packet.system.log — chronological action-entries for the loop.
+    // SPEC §packet the log section — chronological action-entries for the loop.
     // Snapshot is taken at packet build (pre-dispatch this turn), so it
     // reflects "what has happened before this turn." Each row carries a
     // log:///<loop_seq>/<turn_seq>/<sequence> coordinate the model can READ.
@@ -1404,8 +1424,8 @@ class Engine {
         return (row?.n ?? 0) > 0;
     }
     // Inject a prompt into the run's currently-executing loop. Writes a
-    // plurnk:///prompt/<loop_id>/<next-turn> entry whose body becomes
-    // packet.user.prompt at the next turn boundary. Last-wins: if two
+    // plurnk:///prompt/<loop_id>/<next-turn> entry whose body becomes the
+    // prompt section at the next turn boundary. Last-wins: if two
     // injects target the same next-turn slot, the second overwrites the
     // first.
     //
@@ -1425,7 +1445,7 @@ class Engine {
         const sessionRow = await this.#db.drain_get_run_session.get({ run_id: runId });
         if (sessionRow === undefined)
             throw new Error(`Engine.inject: run ${runId} not found`);
-        const pathname = `prompt/${loopId}/${turnSeq}`;
+        const pathname = `/prompt/${loopId}/${turnSeq}`; // canonical storage form (leading slash), matching the foist via #pathnameOf
         const ctx = {
             db: this.#db, sessionId: sessionRow.session_id, runId, loopId,
             turnId: 0, // no turn open at inject time; entries don't pin turnId
@@ -1955,9 +1975,14 @@ class Engine {
         if (path.kind === "local")
             return { scheme: null, username: null, password: null, hostname: null, port: null, pathname: decodePathParens(path.raw), params: null, fragment: null }; // #239 item 4
         const scheme = path.scheme === "file" ? null : path.scheme;
+        // plurnk uses its authority as a namespace — fold it into the canonical pathname so the
+        // log keys identically to the entry (/prompt/<loop>, /docs/x.md). A web host (http://) is
+        // NOT a namespace: keep it in hostname.
+        const foldNs = scheme === "plurnk";
         return {
             scheme, username: path.username, password: path.password,
-            hostname: path.hostname, port: path.port, pathname: decodePathParens(path.pathname), // #239 item 4
+            hostname: foldNs ? null : path.hostname, port: path.port,
+            pathname: decodePathParens(foldNs ? foldAuthorityIntoPath(path.hostname, path.pathname) : path.pathname), // #239 item 4
             params: JSON.stringify(path.params), fragment: path.fragment,
         };
     }