@theokit/sdk 2.3.0 → 2.5.0

package/dist/eval.d.cts

@@ -31,6 +31,8 @@ export declare class Eval {
      */
     run(runOpts?: EvalRunOptions): Promise<EvalRun>;
 }
+export { captureArtifact } from "./internal/eval/code-runner.js";
 export { EvalAlreadyRunningError } from "./internal/eval/single-flight.js";
+export { JsonlParseError, loadJsonl } from "./internal/persistence/jsonl.js";
 export { Scorers } from "./scorers.js";
 export type * from "./types/eval.js";

package/dist/eval.d.ts

@@ -31,6 +31,8 @@ export declare class Eval {
      */
     run(runOpts?: EvalRunOptions): Promise<EvalRun>;
 }
+export { captureArtifact } from "./internal/eval/code-runner.js";
 export { EvalAlreadyRunningError } from "./internal/eval/single-flight.js";
+export { JsonlParseError, loadJsonl } from "./internal/persistence/jsonl.js";
 export { Scorers } from "./scorers.js";
 export type * from "./types/eval.js";

package/dist/eval.js

@@ -2,7 +2,7 @@ import { randomUUID, randomBytes, createHash } from 'crypto';
 import { readFile, unlink, mkdir, open, rename, statfs, stat, rm, readdir, appendFile, access } from 'fs/promises';
 import { join, dirname, resolve, sep, relative, isAbsolute } from 'path';
 import { z, toJSONSchema } from 'zod';
-import { mkdirSync, readdirSync, existsSync, realpathSync, lstatSync, readlinkSync, readFileSync } from 'fs';
+import { readFileSync, mkdirSync, appendFileSync, readdirSync, existsSync, realpathSync, lstatSync, readlinkSync } from 'fs';
 import { AsyncLocalStorage } from 'async_hooks';
 import { createRequire } from 'module';
 import { homedir } from 'os';
@@ -6496,6 +6496,8 @@ function parseSubagentMarkdown(raw, filename) {
   if (fields.model !== void 0) {
     definition.model = fields.model === "inherit" ? "inherit" : { id: fields.model };
   }
+  const tools = fields.tools?.split(/[\s,]+/).map((t) => t.trim()).filter((t) => t.length > 0);
+  if (tools !== void 0 && tools.length > 0) definition.tools = tools;
   return { name, definition };
 }
 function splitFrontmatter2(raw, filename) {
@@ -6659,21 +6661,24 @@ ${lines.join("\n")}
   }
 };
 
+// src/internal/runtime/skills/skills-block.ts
+function buildSkillsBlock(skills) {
+  if (skills.length === 0) return void 0;
+  const lines = skills.map(
+    (skill) => `  - ${escapeBlockBody(skill.name)}: ${escapeBlockBody(skill.description)}`
+  );
+  return `<skills>
+${lines.join("\n")}
+</skills>`;
+}
+
 // src/internal/runtime/system-prompt/sources/skills-provider.ts
 var SkillsPromptProvider = class {
   id = "skills";
   priority = 20;
   contribute(ctx) {
     if (ctx.skillsAutoInject === false) return Promise.resolve(void 0);
-    if (ctx.skills.length === 0) return Promise.resolve(void 0);
-    const lines = ctx.skills.map((skill) => {
-      const name = escapeBlockBody(skill.name);
-      const description = escapeBlockBody(skill.description);
-      return `  - ${name}: ${description}`;
-    });
-    return Promise.resolve(`<skills>
-${lines.join("\n")}
-</skills>`);
+    return Promise.resolve(buildSkillsBlock(ctx.skills));
   }
 };
 
@@ -7781,7 +7786,7 @@ async function loadPluginManifestFromMarkdown(pluginsRoot, folderName) {
   return metadata;
 }
 
-// src/internal/runtime/skills/skills-manager.ts
+// src/internal/runtime/skills/discover-skills.ts
 init_errors();
 
 // src/internal/runtime/skills/skill-frontmatter.ts
@@ -7853,6 +7858,61 @@ function hasContent(value) {
   return value !== void 0 && value.trim().length > 0;
 }
 
+// src/internal/runtime/skills/discover-skills.ts
+async function discoverSkills(dir, options) {
+  let entries;
+  try {
+    entries = await readWorkspaceDir(dir, "skills_read_error", "skills directory");
+  } catch {
+    return [];
+  }
+  const skills = [];
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+    let skillDir;
+    try {
+      skillDir = safePathJoin(dir, entry.name);
+      assertNoSymlinkEscape(skillDir, dir);
+    } catch {
+      continue;
+    }
+    const skillPath = join(skillDir, "SKILL.md");
+    let raw;
+    try {
+      raw = await readFile(skillPath, "utf8");
+    } catch {
+      continue;
+    }
+    const skill = tryParseSkill(raw, entry.name, skillPath, options);
+    if (skill !== void 0) skills.push(skill);
+  }
+  return skills;
+}
+function tryParseSkill(raw, fallbackName, source, options) {
+  try {
+    const frontmatter = parseSkillFrontmatter(raw, fallbackName);
+    const skill = {
+      name: frontmatter.name,
+      description: frontmatter.description,
+      source
+    };
+    if (frontmatter.category !== void 0) skill.category = frontmatter.category;
+    if (frontmatter.dependencies !== void 0) skill.dependencies = frontmatter.dependencies;
+    return skill;
+  } catch (cause) {
+    if (cause instanceof ConfigurationError) {
+      options?.onInvalidSkill?.({
+        name: fallbackName,
+        source,
+        code: cause.code ?? "unknown",
+        message: cause.message
+      });
+      return void 0;
+    }
+    throw cause;
+  }
+}
+
 // src/internal/runtime/skills/skills-manager.ts
 var SkillsManager = class {
   constructor(cwd, _enabled, settingSourcesIncludeProject) {
@@ -7870,56 +7930,20 @@ var SkillsManager = class {
     await this.refresh();
   }
   async refresh() {
-    this.skills = [];
     const skillsRoot = join(this.cwd, ".theokit", "skills");
-    const entries = await readWorkspaceDir(skillsRoot, "skills_read_error", "skills directory");
-    for (const entry of entries) {
-      if (!entry.isDirectory()) continue;
-      let skillDir;
-      try {
-        skillDir = safePathJoin(skillsRoot, entry.name);
-        assertNoSymlinkEscape(skillDir, skillsRoot);
-      } catch {
-        continue;
-      }
-      const skillPath = join(skillDir, "SKILL.md");
-      let raw;
-      try {
-        raw = await readFile(skillPath, "utf8");
-      } catch {
-        continue;
+    this.skills = await discoverSkills(skillsRoot, {
+      onInvalidSkill: (info) => {
+        process.stderr.write(
+          `[theokit-sdk] skill ${info.name} skipped (${info.code}): ${info.message}
+`
+        );
       }
-      const metadata = tryParseSkill(raw, entry.name, skillPath);
-      if (metadata !== void 0) this.skills.push(metadata);
-    }
+    });
   }
   list() {
     return Promise.resolve(this.skills);
   }
 };
-function tryParseSkill(raw, fallbackName, source) {
-  try {
-    const frontmatter = parseSkillFrontmatter(raw, fallbackName);
-    const metadata = {
-      name: frontmatter.name,
-      description: frontmatter.description,
-      source
-    };
-    if (frontmatter.category !== void 0) metadata.category = frontmatter.category;
-    if (frontmatter.dependencies !== void 0) metadata.dependencies = frontmatter.dependencies;
-    return metadata;
-  } catch (cause) {
-    if (cause instanceof ConfigurationError) {
-      const code = cause.code ?? "unknown";
-      process.stderr.write(
-        `[theokit-sdk] skill ${fallbackName} skipped (${code}): ${cause.message}
-`
-      );
-      return void 0;
-    }
-    throw cause;
-  }
-}
 
 // src/internal/runtime/local-agent/local-agent-bootstrap.ts
 function registerLocalAgent(args) {
@@ -8332,6 +8356,7 @@ async function initLoopContext(inputs) {
     finalStatus: "finished",
     usage: new UsageAccumulator(),
     nudgeAttempts: 0,
+    stopFeedbackAttempts: 0,
     ...memoryProviderHandle !== void 0 ? { memoryProviderHandle } : {},
     ...memorySystemPromptAdditions !== void 0 ? { memorySystemPromptAdditions } : {}
   };
@@ -8476,8 +8501,9 @@ function registerLoopError(ctx, cause) {
   if (ctx.error !== void 0) return;
   const rawMessage = cause?.message;
   const message = typeof rawMessage === "string" ? rawMessage : cause instanceof Error ? cause.message : String(cause);
+  const metaCode = cause?.metadata?.code;
   const rawCode = cause?.code;
-  const code = typeof rawCode === "string" ? rawCode : void 0;
+  const code = typeof metaCode === "string" ? metaCode : typeof rawCode === "string" ? rawCode : void 0;
   ctx.error = code !== void 0 ? { message, code, cause } : { message, cause };
 }
 async function runCollectorLoop(generator, inputs, ctx) {
@@ -9273,6 +9299,7 @@ function computeUsageCost(inputs, usage) {
 
 // src/internal/agent-loop/loop.ts
 var MAX_NUDGE_ATTEMPTS = 2;
+var MAX_STOP_FEEDBACK_ATTEMPTS = 2;
 async function runAgentLoop(inputs) {
   const sendSpan = inputs.telemetry?.startSpan("agent.send", {
     agentId: inputs.agentId,
@@ -9430,6 +9457,28 @@ function shouldNudgeAndContinue(ctx, llmOutput) {
   });
   return true;
 }
+async function reflectAfterStop(inputs, ctx) {
+  const result = await inputs.hooks.run({
+    event: "stop",
+    agentId: inputs.agentId,
+    runId: inputs.runId
+  });
+  if (result.blocked) return false;
+  if (ctx.stopFeedbackAttempts >= MAX_STOP_FEEDBACK_ATTEMPTS) return false;
+  const feedback = result.decisions.find(
+    (d) => d.decision === "feedback" && (d.feedback ?? "").length > 0
+  )?.feedback;
+  if (feedback === void 0) return false;
+  ctx.stopFeedbackAttempts += 1;
+  ctx.messages.push({ role: "user", content: [{ type: "text", text: feedback }] });
+  return true;
+}
+async function finishOrReflect(inputs, ctx, llmOutput) {
+  if (shouldNudgeAndContinue(ctx, llmOutput)) return "continue";
+  if (await reflectAfterStop(inputs, ctx)) return "continue";
+  ctx.finalStatus = "finished";
+  return "done";
+}
 async function runIteration(inputs, ctx) {
   const llmOutput = await streamLlmTurn(inputs, ctx);
   accumulateUsage(ctx.usage, llmOutput);
@@ -9463,9 +9512,7 @@ async function continueOrTerminate(inputs, ctx, llmOutput) {
     await emitAssistantTextStep(inputs, ctx, llmOutput.text);
   }
   if (llmOutput.stopReason !== "tool_use" || llmOutput.toolCalls.length === 0) {
-    if (shouldNudgeAndContinue(ctx, llmOutput)) return "continue";
-    ctx.finalStatus = "finished";
-    return "done";
+    return finishOrReflect(inputs, ctx, llmOutput);
   }
   ctx.messages.push(buildAssistantTurn(llmOutput.text, llmOutput.toolCalls));
   const toolResults = await dispatchTools(inputs, ctx.tools, llmOutput.toolCalls, ctx.events);
@@ -15488,6 +15535,69 @@ setAgentFacade({
   resume: (agentId, options) => Agent.resume(agentId, options),
   batch: (prompts, options) => Agent.batch(prompts, options)
 });
+var JsonlParseError = class extends Error {
+  constructor(message, line) {
+    super(message);
+    this.line = line;
+    this.name = "JsonlParseError";
+  }
+  line;
+};
+function isPlainObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function tryParseObjectLine(line) {
+  if (line.length === 0) return void 0;
+  let parsed;
+  try {
+    parsed = JSON.parse(line);
+  } catch {
+    return void 0;
+  }
+  return isPlainObject(parsed) ? parsed : void 0;
+}
+function loadJsonl(path, opts = {}) {
+  const text = readFileSync(path, "utf8");
+  const out = [];
+  let lineNumber = 0;
+  for (const rawLine of text.split("\n")) {
+    lineNumber += 1;
+    const line = rawLine.trim();
+    if (line.length === 0) continue;
+    let parsed;
+    try {
+      parsed = JSON.parse(line);
+    } catch {
+      throw new JsonlParseError(`line ${lineNumber}: invalid JSON`, lineNumber);
+    }
+    if (!isPlainObject(parsed)) {
+      throw new JsonlParseError(`line ${lineNumber}: not a JSON object`, lineNumber);
+    }
+    out.push(opts.map ? opts.map(parsed, lineNumber) : parsed);
+  }
+  return out;
+}
+function appendJsonl(path, record) {
+  mkdirSync(dirname(path), { recursive: true });
+  appendFileSync(path, `${JSON.stringify(record)}
+`);
+}
+function readJsonlIds(path, keyFn) {
+  const done = /* @__PURE__ */ new Set();
+  let text;
+  try {
+    text = readFileSync(path, "utf8");
+  } catch {
+    return done;
+  }
+  for (const rawLine of text.split("\n")) {
+    const parsed = tryParseObjectLine(rawLine.trim());
+    if (parsed === void 0) continue;
+    const key = keyFn(parsed);
+    if (typeof key === "string" && key.length > 0) done.add(key);
+  }
+  return done;
+}
 
 // src/internal/eval/runner.ts
 init_agent_factory_registry();
@@ -15679,6 +15789,50 @@ function normalizeScorers(input) {
     return { name: s.name, score: s.score };
   });
 }
+function probeRow(entry, index) {
+  return {
+    index,
+    input: entry.input,
+    output: "",
+    ...entry.expected !== void 0 ? { expected: entry.expected } : {},
+    scores: [],
+    meanScore: 0,
+    durationMs: 0,
+    ...entry.metadata !== void 0 ? { metadata: entry.metadata } : {}
+  };
+}
+function computeDoneKeys(persist) {
+  if (persist.resume !== true) return /* @__PURE__ */ new Set();
+  return readJsonlIds(
+    persist.path,
+    (parsed) => parsed.error === void 0 ? persist.key(parsed) : void 0
+  );
+}
+function appendRowSafely(path, row) {
+  try {
+    appendJsonl(path, row);
+  } catch (err) {
+    console.warn(
+      "[eval] persist append failed (ignored):",
+      err instanceof Error ? err.message : err
+    );
+  }
+}
+function makeRowSink(persist, classify) {
+  const doneKeys = persist !== void 0 ? computeDoneKeys(persist) : /* @__PURE__ */ new Set();
+  return {
+    isResumed(entry, index) {
+      if (persist === void 0 || doneKeys.size === 0) return false;
+      return doneKeys.has(persist.key(probeRow(entry, index)));
+    },
+    finalize(row) {
+      const outcome = classify?.(row);
+      const finalRow = outcome !== void 0 ? { ...row, outcome } : row;
+      if (persist !== void 0) appendRowSafely(persist.path, finalRow);
+      return finalRow;
+    }
+  };
+}
 async function applyScorer(scorer, output, expected) {
   let raw;
   try {
@@ -15724,7 +15878,15 @@ function makeAgentForBatch(spec, _entries) {
   }
   return spec;
 }
-async function runRowsManually(entries, spec, scorers, concurrency, signal, onRow) {
+async function runManualSlot(idx, entries, spec, scorers, sink, rows, onRow) {
+  const entry = entries[idx];
+  if (entry === void 0) return;
+  if (sink.isResumed(entry, idx)) return;
+  const row = sink.finalize(await runOneEntry(spec, entry, idx, scorers));
+  rows[idx] = row;
+  onRow(row, idx);
+}
+async function runRowsManually(entries, spec, scorers, concurrency, signal, onRow, sink) {
   const rows = new Array(entries.length);
   const state2 = { cursor: 0 };
   const worker = async () => {
@@ -15732,11 +15894,7 @@ async function runRowsManually(entries, spec, scorers, concurrency, signal, onRo
       if (signal?.aborted === true) return;
       const idx = state2.cursor;
       state2.cursor += 1;
-      const entry = entries[idx];
-      if (entry === void 0) continue;
-      const row = await runOneEntry(spec, entry, idx, scorers);
-      rows[idx] = row;
-      onRow(row, idx);
+      await runManualSlot(idx, entries, spec, scorers, sink, rows, onRow);
     }
   };
   const workers = Array.from({ length: Math.min(concurrency, entries.length) }, () => worker());
@@ -15786,23 +15944,32 @@ async function scoreBatchOutput(br, expected, scorers) {
   }
   return scoreEntries;
 }
-async function runRowsViaBatch(entries, agentOptions, scorers, concurrency, signal, onRow) {
-  const prompts = entries.map((e) => e.input);
+async function runRowsViaBatch(entries, agentOptions, scorers, concurrency, signal, onRow, sink) {
+  const pending = [];
+  for (let i = 0; i < entries.length; i += 1) {
+    const entry = entries[i];
+    if (entry === void 0) continue;
+    if (sink.isResumed(entry, i)) continue;
+    pending.push({ entry, index: i });
+  }
   const batchOpts = {
     ...agentOptions,
     concurrency,
     ...signal !== void 0 ? { signal } : {}
   };
-  const batchResults = await getAgentFacade().batch(prompts, batchOpts);
+  const batchResults = await getAgentFacade().batch(
+    pending.map((p) => p.entry.input),
+    batchOpts
+  );
   const rows = [];
   for (let i = 0; i < batchResults.length; i += 1) {
-    const entry = entries[i];
+    const slot = pending[i];
     const br = batchResults[i];
-    if (entry === void 0 || br === void 0) continue;
-    const scoreEntries = await scoreBatchOutput(br, entry.expected, scorers);
-    const row = rowFromBatchResult(entry, br, scoreEntries, i);
+    if (slot === void 0 || br === void 0) continue;
+    const scoreEntries = await scoreBatchOutput(br, slot.entry.expected, scorers);
+    const row = sink.finalize(rowFromBatchResult(slot.entry, br, scoreEntries, slot.index));
     rows.push(row);
-    onRow(row, i);
+    onRow(row, slot.index);
   }
   return rows;
 }
@@ -15827,6 +15994,7 @@ async function runEval(options, runOpts) {
     const onRow = (row, i) => {
       safeHook(() => hooks?.afterRow?.(row, i));
     };
+    const sink = makeRowSink(runOpts?.persist, runOpts?.classify);
     let rows;
     if (isAgentInstance(options.agent) || typeof options.agent === "function") {
       rows = await runRowsManually(
@@ -15835,11 +16003,12 @@ async function runEval(options, runOpts) {
         scorers,
         concurrency,
         signal,
-        onRow
+        onRow,
+        sink
       );
     } else {
       const batchOpts = makeAgentForBatch(options.agent, indexed);
-      rows = await runRowsViaBatch(indexed, batchOpts, scorers, concurrency, signal, onRow);
+      rows = await runRowsViaBatch(indexed, batchOpts, scorers, concurrency, signal, onRow, sink);
     }
     const aggregate = computeAggregate(rows);
     const endedAt = Date.now();
@@ -15865,6 +16034,25 @@ async function runEval(options, runOpts) {
   }
 }
 
+// src/sandbox/shell-escape.ts
+function shellEscapePosix(arg) {
+  return `'${arg.replace(/'/g, "'\\''")}'`;
+}
+
+// src/internal/eval/code-runner.ts
+var ARTIFACT_PATCH = ".theo-artifact.patch";
+async function captureArtifact(sandbox, repoDir) {
+  const dir = shellEscapePosix(repoDir);
+  const diffRes = await sandbox.execute(`git -C ${dir} diff`);
+  const diff = diffRes.stdout;
+  if (diff.length === 0) return { diff: "", applies: false };
+  await sandbox.uploadFile(`${repoDir}/${ARTIFACT_PATCH}`, diff);
+  const check = await sandbox.execute(
+    `git -C ${dir} apply --check --reverse ${shellEscapePosix(ARTIFACT_PATCH)}`
+  );
+  return { diff, applies: check.exitCode === 0 };
+}
+
 // src/internal/scorers/llm-judge.ts
 init_agent_factory_registry();
 function buildPrompt(subject, criteria, rubric, expected) {
@@ -16007,6 +16195,38 @@ var Scorers = {
       }
     };
   },
+  /**
+   * Verify-gate scorer (M6-2): runs the project's tests in the provisioned
+   * repo via `SandboxBackend.execute` and scores `1` iff the command exits `0`,
+   * else `0` with the exit code + truncated stderr in `reason`. Grades the
+   * artifact captured by `captureArtifact` (D2 — rides `execute`, never a
+   * direct `child_process`).
+   *
+   * SECURITY: `command` is REQUIRED and the caller's builder owns shell-safety
+   * of the (potentially untrusted, dataset-derived) test identifiers. There is
+   * NO default that runs bare test names — that would interpolate untrusted
+   * `failToPass`/`passToPass` straight into a shell. `repoDir` is shell-escaped
+   * by the SDK; the test list is the builder's responsibility to render safely.
+   *
+   * PORTABILITY: the command is wrapped as `cd <repoDir> && <cmd>`, which
+   * assumes a shell-backed `SandboxBackend` (LocalSandbox/Docker). A backend
+   * that rejects shell metacharacters in `execute` is unsupported for this scorer.
+   */
+  verifyGate(opts) {
+    const { sandbox, repoDir, failToPass, passToPass, command } = opts;
+    return {
+      name: "verify-gate",
+      score: async () => {
+        const cmd = command([...failToPass, ...passToPass]).trim();
+        if (cmd.length === 0) {
+          return { score: 0, reason: "verify_gate_empty_command" };
+        }
+        const r = await sandbox.execute(`cd ${shellEscapePosix(repoDir)} && ${cmd}`);
+        if (r.exitCode === 0) return { score: 1 };
+        return { score: 0, reason: `exit=${r.exitCode} ${r.stderr.slice(0, 200)}`.trim() };
+      }
+    };
+  },
   jsonShape(schema, opts = {}) {
     return {
       name: "json-shape",
@@ -16079,6 +16299,6 @@ var Eval = class _Eval {
   }
 };
 
-export { Eval, EvalAlreadyRunningError, Scorers };
+export { Eval, EvalAlreadyRunningError, JsonlParseError, Scorers, captureArtifact, loadJsonl };
 //# sourceMappingURL=eval.js.map
 //# sourceMappingURL=eval.js.map