@bjoernrochel/sevro 0.1.0-rc.16 → 0.1.0-rc.17

package/docs/installing.md

@@ -23,10 +23,10 @@ Local installation records the selected version in the project lockfile.
 
 ## Current release
 
-The current release is `0.1.0-rc.16`. Install this exact version:
+The current release is `0.1.0-rc.17`. Install this exact version:
 
 ```sh
-bun add --exact @bjoernrochel/sevro@0.1.0-rc.16
+bun add --exact @bjoernrochel/sevro@0.1.0-rc.17
 ```
 
 <!-- sevro-current-release:end -->

package/docs/native-controls.md

@@ -5,7 +5,8 @@ retain an observation with that ID. It exposes host facts for extension policy:
 
 - `method: native_control_calls`;
 - ordered `calls` containing only `ordinal`, `namespace`, and `name`;
-- `acceptedAgentCount`, a verified Codex acceptance count or `null` on Claude;
+- `acceptedAgentCount`, a verified direct native Codex acceptance count when
+  overall coverage is complete, otherwise `null` (also `null` on Claude);
 - `submittedExecCalls`, the Codex submitted-code count or `null` on Claude; and
 - `truncated`, which is `false` for a complete record and may be unknown (`null`)
   for an incomplete Codex record.
@@ -16,6 +17,18 @@ invocation attempt, not success. Extensions decide which calls are prohibited.
 An unknown acceptance count cannot establish agent acceptance.
 
 Codex projects labels and counts from its bound native-session observation.
+It additionally retains `directAcceptedAgentCount`, the number of positively
+correlated direct native spawn acceptances in that source, independently of exec
+inspection. This is an observed count, not a total across unknown launch paths.
+`coverage.nativeCalls` records native-session completeness;
+`coverage.execInspection` records whether all submitted exec sources were
+understood and matched to the direct submission inventory;
+`coverage.acceptedAgents: direct_native_calls` names the scope of both acceptance
+counts. Accepted launches through exec are not included in these direct counts.
+When native capture is incomplete, even a zero observed count cannot establish
+absence. Native labels remain usable as positive attempt evidence regardless
+of an unrelated exec inspection gap. A rejected attempt is never an acceptance.
+
 When submitted code is present, both observations also retain `execCalls`:
 one receipt per native `exec` submission, with its native `ordinal`, inspection
 `completeness`, and ordered `calls` containing `index`, `namespace`, and `name`.
@@ -27,25 +40,46 @@ programs without executing them. It accepts literal tool calls, constant result
 bindings, template interpolation and `+` composition of inspected expressions,
 direct `text`/`store` output, and literal `Promise.all`/`allSettled` batches.
 Batch results may use flat constant array bindings or result-only
-`forEach(text)` or `for ... of` output. Destructuring defaults, rest elements,
+`forEach(text)` or `for ... of` output, including blocks and conditional
+result-field selection. `JSON.stringify`, `String`, object spread/shorthand,
+and nested result fields are supported. Numeric output loops must use the
+bounded shape `for (let i = 0; i < results.length; i++)`, indexing the known
+batch with that loop variable. Output loops and conditional expressions cannot
+invoke tools. Destructuring defaults, rest elements,
 holes, nested bindings, duplicate names, and shadowed globals remain unsupported.
-It requires one
-unique, later native completion for the same call ID with an intact successful
-script envelope. Tool argument strings, comments and printed tool-looking text
+Tool argument strings, comments and printed tool-looking text
 do not become call labels. A rejected tool request returned by a completed
 script still appears in its submitted-call inventory. This inventory proves
 submission of those requests, not successful effects or acceptance; the existing
 acceptance count describes independently correlated direct native spawns.
 
 Dynamic lookup, aliases, shadowed globals, callbacks that invoke tools,
-conditional code, nested executors, malformed or truncated results, duplicate
-receipts, and missing completion keep inspection partial. Source is bounded at
+conditional tool execution, and nested executors keep source inspection partial.
+Each receipt's `inspectionReason` is `null` on success, or `unsupported_source`,
+`malformed_source`, `source_limit`, or `foreign_namespace`. Unsupported syntax
+and expression/call-budget exhaustion use `unsupported_source`.
+Source is bounded at
 64 KiB, traversed expressions at 2,048, and calls at 128 per submission; tool
 labels are bounded at 64 characters. Private source and output bytes are discarded.
 The portable observation becomes partial if any submission is uninspected or
-receipt counts/ordinals disagree with the native list. Incomplete counts remain
-`null`. Native-session completeness still describes session capture separately
-from each submission's inspection completeness.
+receipt counts/ordinals disagree with the native list. `acceptedAgentCount`
+remains `null` in that case, while native coverage, direct acceptance facts and
+the native `submittedExecCalls` count remain independently available.
+`submittedExecCalls` and `truncated` depend on native capture, not exec output.
+
+Receipt `completeness` describes source inspection, independently of execution
+or displayed output. `execution` is `completed`, `failed`, or `incomplete`;
+`output` is `complete`, `truncated`, or `unavailable`. Output evidence requires
+a unique later output bound to the same native call ID, with recognized text
+blocks. `complete` describes the displayed receipt, not the whole program's
+execution. Execution status additionally requires a successful or failed script
+header. Missing, ambiguous or
+malformed output yields `incomplete`/`unavailable`; a yielded or unrecognized
+header stays `incomplete`. Displayed truncation does not change source coverage.
+A fully inspected source with no launch call proves that absence even if its
+shell output was truncated or the exec cell failed. Source inspection cannot
+prove execution or success. Missing fields in historical receipts are unknown;
+historical artifacts retain their original results and meanings.
 
 Claude records every tool-use label from the bounded stream, with namespace
 `claude`, without filtering to Skill or Agent. It requires a completed successful
@@ -55,11 +89,12 @@ Tool-looking text in arguments or output does not become a native call.
 
 Consumers must verify the source, completeness, shape, ordering, and counts
 before using an empty call list to establish absence. Missing or partial
-evidence cannot establish a negative assertion. For Codex, consumers must
+source coverage cannot establish a negative assertion. For Codex, consumers must
 inspect every `execCalls` receipt in addition to the direct `calls` list,
 validate receipt ordinals against the direct `exec` submissions, and require
 all receipts to be complete with counts matching `submittedExecCalls`.
-They must examine submitted labels for prohibited controls even when
+They can use positively observed native attempt labels before requiring complete
+coverage for an absence pass. They must examine submitted labels for prohibited controls even when
 `acceptedAgentCount` is zero. Older consumers that require
 `submittedExecCalls === 0` remain conservative but cannot use this new evidence
 until they adopt receipt validation.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bjoernrochel/sevro",
-  "version": "0.1.0-rc.16",
+  "version": "0.1.0-rc.17",
   "license": "SEE LICENSE IN LICENSE",
   "description": "A standalone evaluation runner for agentic coding capabilities",
   "type": "module",

package/src/hosts/codex-exec-controls.ts

@@ -21,6 +21,14 @@ export interface ExecCallReceipt {
   ordinal: number;
   completeness: "complete" | "partial";
   calls: SubmittedCall[];
+  inspectionReason?:
+    | "unsupported_source"
+    | "malformed_source"
+    | "source_limit"
+    | "foreign_namespace"
+    | null;
+  execution?: "completed" | "failed" | "incomplete";
+  output?: "complete" | "truncated" | "unavailable";
 }
 
 function toolNamespace(name: string): SubmittedCall["namespace"] {
@@ -77,7 +85,15 @@ function member(node: ts.Expression, owner: string, names: string[]): boolean {
 function localName(node: ts.BindingName): string | undefined {
   const ts = typescript();
   if (!ts.isIdentifier(node)) return undefined;
-  return ["tools", "text", "store", "Promise", "load"].includes(node.text)
+  return [
+    "tools",
+    "text",
+    "store",
+    "Promise",
+    "load",
+    "JSON",
+    "String",
+  ].includes(node.text)
     ? undefined
     : node.text;
 }
@@ -85,6 +101,7 @@ class Submission {
   readonly calls: SubmittedCall[] = [];
   private readonly locals = new Set<string>();
   private readonly collections = new Set<string>();
+  private readonly indices = new Set<string>();
   private nodes = 0;
   expression(node: ts.Expression): boolean {
     const ts = typescript();
@@ -110,17 +127,42 @@ class Submission {
         return this.localProperty(node);
       case ts.isCallExpression(node):
         return this.call(node);
+      case ts.isConditionalExpression(node):
+        return [node.condition, node.whenTrue, node.whenFalse].every((part) =>
+          this.outputExpression(part),
+        );
+      case ts.isElementAccessExpression(node):
+        return (
+          this.collection(node.expression) &&
+          typescript().isIdentifier(node.argumentExpression) &&
+          this.indices.has(node.argumentExpression.text)
+        );
       default:
         return false;
     }
   }
   private property(node: ts.ObjectLiteralElementLike): boolean {
     const ts = typescript();
+    if (ts.isSpreadAssignment(node))
+      return this.outputExpression(node.expression);
+    if (ts.isShorthandPropertyAssignment(node)) return this.shorthand(node);
     if (!ts.isPropertyAssignment(node) || ts.isComputedPropertyName(node.name))
       return false;
     return this.expression(node.initializer);
   }
+  private shorthand(node: ts.ShorthandPropertyAssignment): boolean {
+    return !node.objectAssignmentInitializer && this.expression(node.name);
+  }
   private composition(node: ts.BinaryExpression): boolean {
+    if (
+      [
+        typescript().SyntaxKind.EqualsEqualsEqualsToken,
+        typescript().SyntaxKind.ExclamationEqualsEqualsToken,
+      ].includes(node.operatorToken.kind)
+    )
+      return (
+        this.outputExpression(node.left) && this.outputExpression(node.right)
+      );
     return (
       node.operatorToken.kind === typescript().SyntaxKind.PlusToken &&
       this.expression(node.left) &&
@@ -148,13 +190,12 @@ class Submission {
     if (ts.isVariableStatement(node))
       return this.declarations(node.declarationList);
     if (ts.isForOfStatement(node)) return this.outputLoop(node);
+    if (ts.isForStatement(node)) return this.numericOutputLoop(node);
     return ts.isExpressionStatement(node) && this.expression(node.expression);
   }
   private localProperty(node: ts.PropertyAccessExpression): boolean {
-    const ts = typescript();
     return (
-      ts.isIdentifier(node.expression) &&
-      this.locals.has(node.expression.text) &&
+      this.expression(node.expression) &&
       !["constructor", "__proto__", "prototype"].includes(node.name.text)
     );
   }
@@ -231,10 +272,43 @@ class Submission {
     if (!this.collection(node.expression)) return false;
     if (!ts.isVariableDeclarationList(node.initializer)) return false;
     const name = loopVariable(node.initializer);
+    if (!name || this.locals.has(name)) return false;
+    this.locals.add(name);
+    const valid = this.outputBody(node.statement);
+    this.locals.delete(name);
+    return valid;
+  }
+  private outputExpression(node: ts.Expression): boolean {
+    const count = this.calls.length;
+    return this.expression(node) && this.calls.length === count;
+  }
+  private numericOutputLoop(node: ts.ForStatement): boolean {
+    const name = numericLoopVariable(node);
+    if (!name || this.locals.has(name)) return false;
+    const condition = node.condition as ts.BinaryExpression;
+    const bound = condition.right as ts.PropertyAccessExpression;
+    if (!this.collection(bound.expression)) return false;
+    this.locals.add(name);
+    this.indices.add(name);
+    const valid = this.outputBody(node.statement);
+    this.locals.delete(name);
+    this.indices.delete(name);
+    return valid;
+  }
+  private outputBody(node: ts.Statement): boolean {
+    const ts = typescript();
+    if (ts.isBlock(node))
+      return node.statements.every((statement) => this.outputBody(statement));
+    if (ts.isIfStatement(node)) return this.outputConditional(node);
     return (
-      name !== undefined &&
-      !this.locals.has(name) &&
-      outputLoopBody(node.statement, name)
+      ts.isExpressionStatement(node) && this.outputExpression(node.expression)
+    );
+  }
+  private outputConditional(node: ts.IfStatement): boolean {
+    return (
+      this.outputExpression(node.expression) &&
+      this.outputBody(node.thenStatement) &&
+      (!node.elseStatement || this.outputBody(node.elseStatement))
     );
   }
   private collection(node: ts.Expression): boolean {
@@ -244,9 +318,12 @@ class Submission {
 }
 function helperCall(node: ts.Expression): boolean {
   const ts = typescript();
-  if (ts.isIdentifier(node) && ["text", "store"].includes(node.text))
+  if (ts.isIdentifier(node) && ["text", "store", "String"].includes(node.text))
     return true;
-  return member(node, "Promise", ["all", "allSettled"]);
+  return (
+    member(node, "Promise", ["all", "allSettled"]) ||
+    member(node, "JSON", ["stringify"])
+  );
 }
 function namedIdentifier(
   node: ts.Expression | undefined,
@@ -271,22 +348,69 @@ function loopVariable(list: ts.VariableDeclarationList): string | undefined {
     ? localName(declaration.name)
     : undefined;
 }
-function outputLoopBody(statement: ts.Statement, name: string): boolean {
+function numericLoopVariable(node: ts.ForStatement): string | undefined {
   const ts = typescript();
+  if (!node.initializer || !ts.isVariableDeclarationList(node.initializer))
+    return undefined;
+  const name = numericBinding(node.initializer);
+  return name && numericLoopBounds(node, name) ? name : undefined;
+}
+function numericBinding(list: ts.VariableDeclarationList): string | undefined {
   if (
-    !ts.isExpressionStatement(statement) ||
-    !ts.isCallExpression(statement.expression)
+    !(list.flags & typescript().NodeFlags.Let) ||
+    list.declarations.length !== 1
   )
-    return false;
-  const call = statement.expression;
+    return undefined;
+  const declaration = list.declarations[0];
+  return declaration && zeroInitializer(declaration.initializer)
+    ? localName(declaration.name)
+    : undefined;
+}
+function zeroInitializer(node: ts.Expression | undefined): boolean {
+  return (
+    node !== undefined &&
+    typescript().isNumericLiteral(node) &&
+    node.text === "0"
+  );
+}
+function numericLoopBounds(node: ts.ForStatement, name: string): boolean {
+  const ts = typescript();
+  const condition = node.condition;
+  if (!condition || !ts.isBinaryExpression(condition)) return false;
   return (
-    namedIdentifier(call.expression, "text") &&
-    call.arguments.length === 1 &&
-    namedIdentifier(call.arguments[0], name)
+    numericCondition(condition, name) &&
+    numericIncrement(node.incrementor, name)
   );
 }
-function inspectedSource(code: unknown): Submission | undefined {
-  if (typeof code !== "string" || code.length > 65536) return undefined;
+function numericCondition(node: ts.BinaryExpression, name: string): boolean {
+  return (
+    node.operatorToken.kind === typescript().SyntaxKind.LessThanToken &&
+    namedIdentifier(node.left, name) &&
+    lengthProperty(node.right)
+  );
+}
+function lengthProperty(node: ts.Expression): boolean {
+  return (
+    typescript().isPropertyAccessExpression(node) && node.name.text === "length"
+  );
+}
+function numericIncrement(
+  node: ts.Expression | undefined,
+  name: string,
+): boolean {
+  return (
+    node !== undefined &&
+    typescript().isPostfixUnaryExpression(node) &&
+    node.operator === typescript().SyntaxKind.PlusPlusToken &&
+    namedIdentifier(node.operand, name)
+  );
+}
+function inspectedSource(code: unknown): {
+  submission?: Submission;
+  reason: ExecCallReceipt["inspectionReason"];
+} {
+  if (typeof code !== "string") return { reason: "unsupported_source" };
+  if (code.length > 65536) return { reason: "source_limit" };
   const ts = typescript();
   const source = ts.createSourceFile(
     "submitted.js",
@@ -298,33 +422,26 @@ function inspectedSource(code: unknown): Submission | undefined {
   const diagnostics = (
     source as ts.SourceFile & { parseDiagnostics: readonly ts.Diagnostic[] }
   ).parseDiagnostics;
-  if (diagnostics.length !== 0) return undefined;
+  if (diagnostics.length !== 0) return { reason: "malformed_source" };
   const submission = new Submission();
   return source.statements.every((statement) => submission.statement(statement))
-    ? submission
-    : undefined;
+    ? { submission, reason: null }
+    : { reason: "unsupported_source" };
 }
-function completedOutput(output: unknown): boolean {
-  if (isUnknownArray(output)) return completedBlocks(output);
-  if (typeof output !== "string") return false;
+function outputBlocks(output: unknown): unknown[] {
+  if (isUnknownArray(output)) return output;
+  if (typeof output !== "string") return [];
   try {
     const blocks: unknown = JSON.parse(output);
-    if (!isUnknownArray(blocks)) return false;
-    return completedBlocks(blocks);
+    return isUnknownArray(blocks) ? blocks : [];
   } catch {
-    return false;
+    return [];
   }
 }
-function completedBlocks(blocks: unknown[]): boolean {
-  return blocks.every(completeTextBlock) && completedHeader(blocks[0]);
-}
 function completeTextBlock(block: unknown): boolean {
   if (!isRecord(block) || !["text", "input_text"].includes(String(block.type)))
     return false;
-  return (
-    typeof block.text === "string" &&
-    !block.text.includes("Warning: truncated output")
-  );
+  return typeof block.text === "string";
 }
 function completedHeader(block: unknown): boolean {
   return (
@@ -335,23 +452,35 @@ function completedHeader(block: unknown): boolean {
     )
   );
 }
-function completedSubmission(entry: Entry, entries: Entry[]): boolean {
-  if (!nativeExecNamespace(entry.payload.namespace)) return false;
+function submissionOutput(entry: Entry, entries: Entry[]): unknown[] {
   const id = entry.payload.call_id;
-  if (typeof id !== "string") return false;
+  if (typeof id !== "string") return [];
   const matches = entries.filter((other) => other.payload.call_id === id);
-  if (matches.length !== 2) return false;
+  if (matches.length !== 2) return [];
   const result = matches.find(
     (other) => other.payload.type === "custom_tool_call_output",
   );
-  return laterCompletion(result, entry.ordinal);
+  return result && result.ordinal > entry.ordinal
+    ? outputBlocks(result.payload.output)
+    : [];
 }
-function laterCompletion(result: Entry | undefined, ordinal: number): boolean {
-  return (
-    result !== undefined &&
-    result.ordinal > ordinal &&
-    completedOutput(result.payload.output)
+function outputEvidence(
+  blocks: unknown[],
+): Pick<ExecCallReceipt, "execution" | "output"> {
+  if (!blocks.length || !blocks.every(completeTextBlock))
+    return { execution: "incomplete", output: "unavailable" };
+  const header = blocks[0] as { text: string };
+  const execution = executionStatus(header);
+  const truncated = blocks.some((block) =>
+    (block as { text: string }).text.includes("Warning: truncated output"),
   );
+  return { execution, output: truncated ? "truncated" : "complete" };
+}
+function executionStatus(header: {
+  text: string;
+}): ExecCallReceipt["execution"] {
+  if (completedHeader(header)) return "completed";
+  return /^Script failed\b/.test(header.text) ? "failed" : "incomplete";
 }
 function nativeExecNamespace(namespace: unknown): boolean {
   return namespace === undefined || namespace === "functions";
@@ -363,14 +492,25 @@ export function codexExecCallReceipts(entries: Entry[]): ExecCallReceipt[] {
       ({ payload }) =>
         payload.type === "custom_tool_call" && payload.name === "exec",
     )
-    .map((entry) => {
-      const submission = inspectedSource(entry.payload.input);
-      const complete =
-        submission !== undefined && completedSubmission(entry, entries);
-      return {
-        ordinal: entry.ordinal,
-        completeness: complete ? "complete" : "partial",
-        calls: submission?.calls ?? [],
-      };
-    });
+    .map((entry) => ({
+      ordinal: entry.ordinal,
+      ...inspectionEvidence(entry),
+      ...outputEvidence(submissionOutput(entry, entries)),
+    }));
+}
+function inspectionEvidence(
+  entry: Entry,
+): Pick<ExecCallReceipt, "completeness" | "calls" | "inspectionReason"> {
+  if (!nativeExecNamespace(entry.payload.namespace))
+    return {
+      completeness: "partial",
+      calls: [],
+      inspectionReason: "foreign_namespace",
+    };
+  const inspected = inspectedSource(entry.payload.input);
+  return {
+    completeness: inspected.submission ? "complete" : "partial",
+    calls: inspected.submission?.calls ?? [],
+    inspectionReason: inspected.reason,
+  };
 }

package/src/hosts/native-controls.ts

@@ -14,6 +14,12 @@ export interface NativeControlObservation {
     method: "native_control_calls";
     calls: Call[];
     acceptedAgentCount: number | null;
+    directAcceptedAgentCount?: number;
+    coverage?: {
+      nativeCalls: NativeControlObservation["completeness"];
+      execInspection: "complete" | "partial";
+      acceptedAgents: "direct_native_calls";
+    };
     submittedExecCalls: number | null;
     execCalls?: ExecCallReceipt[];
     truncated: boolean | null;
@@ -47,12 +53,28 @@ export function codexNativeControls(
       acceptedAgentCount: complete
         ? observation.data.acceptedSpawns.length
         : null,
-      submittedExecCalls: complete ? observation.data.submittedExecCalls : null,
+      directAcceptedAgentCount: observation.data.acceptedSpawns.length,
+      coverage: {
+        nativeCalls: observation.completeness,
+        execInspection: inspectedExecCalls(observation)
+          ? "complete"
+          : "partial",
+        acceptedAgents: "direct_native_calls",
+      },
+      ...nativeCounts(observation),
       ...execEvidence(observation.data.execCalls),
-      truncated: complete ? false : null,
     },
   };
 }
+function nativeCounts(
+  observation: Awaited<ReturnType<typeof codexNativeCallObservation>>,
+) {
+  const complete = observation.completeness === "complete";
+  return {
+    submittedExecCalls: complete ? observation.data.submittedExecCalls : null,
+    truncated: complete ? false : null,
+  };
+}
 function controlCompleteness(
   source: NativeControlObservation["completeness"],
   complete: boolean,