@visual-z/xeno 0.2.0 → 0.3.0

package/src/runtime/tool-mode-eval.ts

@@ -0,0 +1,146 @@
+/** Q.19: fixed workloads and admission overlays share production tools and the v2 auditor. */
+import { queryEvalVerifier } from "./query-eval-verifier.ts";
+import { createHash } from "node:crypto";
+import { join } from "node:path";
+import { seededRandom, type EvalJob } from "../core/experiment.ts";
+import { TOOL_MODE_TASKS, type ToolModeTask, type ToolModeMetrics } from "../core/tool-mode-protocol.ts";
+import { queryEvalContract } from "../core/query-eval-contract.ts";
+import { inputError } from "../core/tool-contract.ts";
+import type { LogEvent, ArtifactRef } from "../core/events.ts";
+import { readObservation } from "../store/observations.ts";
+import { TOOL_DEFINITIONS, modelToolSchema, type ToolContext } from "./tools.ts";
+import type { ToolSchema } from "./provider.ts";
+import type { ComparisonBackend } from "./harness-comparison.ts";
+import type { EvalFixture } from "./harness-eval.ts";
+
+const digest = (text: string) => createHash("sha256").update(text).digest("hex");
+export const TOOL_MODE_SYSTEM = "Solve the supplied data task using query over the complete source files. Preserve inputs and verifier. " +
+  "Use the declared query syntax. Inspect all failures and original evidence. " +
+  "Write only answer.txt, run bun run gate, and report the observed outcome. Do not delegate.";
+export interface ToolModeFixture {
+  answer: string;
+  queries: { path: string; json: Record<string, unknown>; sql: Record<string, unknown> }[];
+}
+export function toolModeFixture(seed: number, task: ToolModeTask, repeat: number): EvalFixture {
+  const random = seededRandom(seed + TOOL_MODE_TASKS.indexOf(task) * 1000 + repeat);
+  const count = task === "tool-batch" ? 8 : 1, rows = task === "tool-small" ? 1 : task === "tool-batch" ? 25 : 9000;
+  const files: Record<string, string> = {}, queries: ToolModeFixture["queries"] = [], expected: { path: string; rows: unknown[] }[] = [];
+  for (let i = 0; i < count; i++) {
+    const path = "data-" + i + ".jsonl", data = Array.from({ length: rows }, (_, row) => ({
+      team: "team" + row % 4, enabled: row % 3 !== 0, amount: 1 + Math.floor(random() * 97),
+    }));
+    files[path] = data.map(row => JSON.stringify(row)).join("\n") + "\n";
+    const base = { paths: [path], format: "jsonl" };
+    if (task === "tool-small") {
+      queries.push({ path, json: { ...base, select: ["amount"] }, sql: { ...base, sql: "SELECT amount FROM source" } });
+      expected.push({ path, rows: [{ amount: data[0]!.amount }] });
+    } else if (task === "tool-batch") {
+      queries.push({ path, json: { ...base, aggregate: [{ op: "sum", field: "amount", as: "total" }] }, sql: { ...base, sql: "SELECT SUM(amount) AS total FROM source" } });
+      expected.push({ path, rows: [{ total: data.reduce((n, row) => n + row.amount, 0) }] });
+    } else {
+      queries.push({ path, json: { ...base, where: [{ field: "enabled", op: "eq", value: true }], group_by: ["team"],
+        aggregate: [{ op: "sum", field: "amount", as: "total" }], order_by: [{ field: "team" }] },
+        sql: { ...base, sql: "SELECT team, SUM(amount) AS total FROM source WHERE enabled = TRUE GROUP BY team ORDER BY team" } });
+      // Independent fixture arithmetic; do not ask the query kernel to grade itself.
+      expected.push({ path, rows: Array.from({ length: 4 }, (_, n) => {
+        const team = "team" + n;
+        return { team, total: data.filter(row => row.enabled && row.team === team).reduce((sum, row) => sum + row.amount, 0) };
+      }) });
+    }
+  }
+  const answer = JSON.stringify(expected) + "\n", answerSha256 = digest(answer);
+  files["package.json"] = JSON.stringify({ scripts: { gate: "bun verify.ts" } }) + "\n";
+  const contract = queryEvalContract(task, queries.map(query => query.path), answerSha256);
+  files["verify.ts"] = queryEvalVerifier(contract);
+  return { task, repeat, files, digests: Object.fromEntries(Object.entries(files).map(([path, body]) => [path, digest(body)])),
+    answerSha256, prompts: [contract.instruction], toolMode: { answer, queries } };
+}
+export type ToolModeSchemas = Record<"json" | "sql", ToolSchema[]>;
+export function toolModeSchemas(): ToolModeSchemas {
+  const build = (syntax: "json" | "sql"): ToolSchema[] => {
+    const names = ["read", "query", "history", "write", "exec_command", "write_stdin", "note"];
+    return names.map(name => {
+      const definition = TOOL_DEFINITIONS.find(tool => tool.name === name)!;
+      if (name !== "query") return modelToolSchema(definition, names);
+      const schema = structuredClone(definition.input_schema) as ToolSchema["input_schema"] & { additionalProperties?: boolean };
+      const jsonKeys = ["where", "select", "group_by", "aggregate", "order_by"];
+      for (const key of syntax === "json" ? ["sql"] : jsonKeys) delete schema.properties[key];
+      if (syntax === "sql") schema.required.push("sql");
+      schema.additionalProperties = false;
+      return { name, description: definition.description + " This trial admits only " + syntax.toUpperCase() + " query syntax.", input_schema: schema };
+    });
+  };
+  return { json: build("json"), sql: build("sql") };
+}
+export const toolModeSchemaKey = (job: EvalJob): keyof ToolModeSchemas => job.policy.querySyntax === "sql" ? "sql" : "json";
+/** Schema visibility is not authority: the same frozen restriction runs before every admitted query. */
+export function toolModeHooks(schemas: readonly ToolSchema[]): NonNullable<ToolContext["extensionHooks"]> {
+  const query = schemas.find(schema => schema.name === "query")!;
+  return { async before(name, input) {
+    if (name !== "query") return undefined;
+    const error = inputError(query.input_schema, input);
+    const properties = (query.input_schema as { properties: Record<string, unknown> }).properties;
+    const extra = input && typeof input === "object" ? Object.keys(input).find(key => !Object.hasOwn(properties, key)) : undefined;
+    return error || extra ? "Frozen query syntax restriction: " + (error ?? "unsupported field " + extra) : undefined;
+  }, async event() {} };
+}
+/** Oracle driver tests the machinery; neither its answers nor call choices are model evidence. */
+export function toolModeBackend(input: EvalFixture): ComparisonBackend {
+  const fixture = input.toolMode!;
+  let step = 0;
+  return async payload => {
+    const declarations = payload["tools"] as { function: { name: string; parameters: { properties: Record<string, unknown> } } }[];
+    const syntax = declarations.find(tool => tool.function.name === "query")!.function.parameters.properties["sql"] ? "sql" : "json";
+    const calls: { name: string; input: unknown }[] = [], index = step++;
+    if (index === 0) {
+      for (const query of fixture.queries) calls.push({ name: "query", input: query[syntax] });
+    }
+    if (index === 1) {
+      const messages = payload["messages"] as { role: string; content?: unknown }[];
+      const observed = messages.filter(message => message.role === "tool").flatMap(message => {
+        const value = JSON.parse(String(message.content));
+        return [value];
+      }) as { sources: { path: string }[]; rows: unknown[] }[];
+      const answer = fixture.queries.map(query => {
+        const value = observed.find(value => value.sources?.length === 1 && value.sources[0]!.path === query.path);
+        if (!value) throw Error("mechanical driver lacks a successful source query receipt");
+        return { path: query.path, rows: value.rows };
+      });
+      calls.push({ name: "write", input: { path: "answer.txt", content: JSON.stringify(answer) + "\n" } });
+    }
+    if (index === 2) calls.push({ name: "exec_command", input: { cmd: "bun run gate", yield_time_ms: 1000 } });
+    return { id: "tool-mode-" + index, model: "eval-mechanical", choices: [{ index: 0, message: {
+      role: "assistant", content: calls.length ? null : "Mechanical protocol fixture finished.",
+      ...(calls.length ? { tool_calls: calls.map((call, i) => ({ id: "mode-" + index + "-" + i, type: "function",
+        function: { name: call.name, arguments: JSON.stringify(call.input) } })) } : {}),
+    }, finish_reason: calls.length ? "tool_calls" : "stop" }], usage: { prompt_tokens: 200, completion_tokens: 32, prompt_cache_hit_tokens: 0 } };
+  };
+}
+export async function toolModeMetrics(directory: string, input: EvalFixture, events: readonly LogEvent[], elapsedMs: number): Promise<ToolModeMetrics> {
+  if (!Number.isFinite(elapsedMs) || elapsedMs < 0) throw Error("missing tool mode elapsed observation");
+  const fixture = input.toolMode!, invocations = new Map(events.filter(event => event.t === "tool_invocation").map(event => [event.id, event]));
+  const sources = new Set<string>();
+  const original = async (ref: ArtifactRef): Promise<unknown> => {
+    const parts: Uint8Array[] = [];
+    for (let offset = 0;;) {
+      const page = await readObservation(join(directory, "observations"), ref.sha256, offset, 1024 * 1024);
+      parts.push(Buffer.from(page.base64, "base64"));
+      if (page.next === null) break;
+      offset = page.next;
+    }
+    return JSON.parse(Buffer.concat(parts).toString("utf8"));
+  };
+  for (const event of events) if (event.t === "tool_result" && event.ok && event.invocation_ref) {
+    const invocation = invocations.get(event.invocation_ref.event);
+    if (!invocation || invocation.call_id !== event.call_id || invocation.name !== "query") continue;
+    const value = event.data_ref ? await original(event.data_ref) : event.content_ref ? await original(event.content_ref) : event.data;
+    if (!value || typeof value !== "object" || !("sources" in value) || !Array.isArray(value.sources)) continue;
+    for (const source of value.sources as { path?: string; sha256?: string }[]) {
+      if (typeof source.path === "string" && fixture.queries.some(query => query.path === source.path) && input.digests[source.path] === source.sha256) sources.add(source.path);
+    }
+  }
+  const calls = [...invocations.values()], sourcePaths = [...sources].sort(), requiredSourcePaths = fixture.queries.map(query => query.path).sort();
+  return { elapsedMs, queryCalls: calls.filter(call => call.name === "query").length,
+    primitiveCalls: calls.length, sourcePaths, requiredSourcePaths,
+    sourceComplete: requiredSourcePaths.every(path => sources.has(path)) };
+}

package/src/runtime/tool-queue.ts

@@ -0,0 +1,77 @@
+/** Pi A / O.5: ordered admission, bounded readers and an exclusive write barrier. */
+import { schedulingWindow } from "../core/scheduling.ts";
+
+type Mode = "parallel" | "exclusive";
+type Job = { mode: Mode; prepare(): Promise<() => Promise<unknown>>; resolve(value: unknown): void; reject(error: unknown): void };
+
+/**
+ * A composing call owns an exclusive slot in its parent queue and gives its
+ * children a fresh queue. Re-entering the held parent slot would deadlock.
+ * Only exclusive tools may compose; concurrent readers therefore cannot each
+ * multiply the session's parallel limit with another queue.
+ */
+export class ToolQueue {
+  private readonly limit: number;
+  private readonly pending: Job[] = [];
+  private active = 0;
+  private exclusive = false;
+  private pumping = false;
+  private readonly idle = new Set<() => void>();
+  private closed = false;
+
+  constructor(limit: number) {
+    schedulingWindow([], 0, limit);
+    this.limit = limit;
+  }
+
+  submit<T>(mode: Mode, prepare: () => Promise<() => Promise<T>>): Promise<T> {
+    if (this.closed) return Promise.reject(new Error("tool scope is closed; no invocation started"));
+    const result = new Promise<T>((resolve, reject) => {
+      this.pending.push({ mode, prepare, resolve: value => resolve(value as T), reject });
+    });
+    this.kick();
+    return result;
+  }
+
+  private kick(): void { if (!this.pumping) { this.pumping = true; void this.pump(); } }
+
+  private async pump(): Promise<void> {
+    try {
+      while (this.pending.length > 0) {
+        const next = this.pending[0]!;
+        const occupied: Mode[] = Array.from({ length: this.active }, () => this.exclusive ? "exclusive" : "parallel");
+        if (schedulingWindow([...occupied, next.mode], 0, this.limit) <= this.active) break;
+        const job = this.pending.shift()!;
+        this.active++; this.exclusive = job.mode === "exclusive";
+        const finished = () => { this.active--; if (job.mode === "exclusive") this.exclusive = false; this.kick(); };
+        try {
+          const execute = await job.prepare();
+          void Promise.resolve().then(execute).then(job.resolve, job.reject).finally(finished);
+        } catch (error) { job.reject(error); this.active--; this.exclusive = false; }
+      }
+    } finally {
+      this.pumping = false;
+      if (this.active === 0 && this.pending.length === 0) { for (const resolve of this.idle) resolve(); this.idle.clear(); }
+    }
+  }
+
+  /** A generator can retain its slot across admission/display yields without executing early. */
+  acquire(mode: Mode): Promise<() => void> {
+    return new Promise((resolve, reject) => {
+      let release!: () => void;
+      const done = new Promise<void>(finish => { release = finish; });
+      void this.submit(mode, async () => async () => { resolve(release); await done; }).catch(reject);
+    });
+  }
+
+  whenIdle(): Promise<void> {
+    if (!this.pumping && this.active === 0 && this.pending.length === 0) return Promise.resolve();
+    return new Promise(resolve => this.idle.add(resolve));
+  }
+
+  /** Accepted jobs drain even when their caller forgot to await their promises. */
+  async close(): Promise<void> {
+    this.closed = true;
+    await this.whenIdle();
+  }
+}