@bjoernrochel/sevro 0.1.0-rc.19 → 0.1.0-rc.20

package/docs/installing.md

@@ -23,10 +23,10 @@ Local installation records the selected version in the project lockfile.
 
 ## Current release
 
-The current release is `0.1.0-rc.19`. Install this exact version:
+The current release is `0.1.0-rc.20`. Install this exact version:
 
 ```sh
-bun add --exact @bjoernrochel/sevro@0.1.0-rc.19
+bun add --exact @bjoernrochel/sevro@0.1.0-rc.20
 ```
 
 <!-- sevro-current-release:end -->

package/docs/native-hosts.md

@@ -90,6 +90,73 @@ Newly created or changed skill bodies are outside the prepared snapshot. A
 complete empty observation is limited to this scope and the supported local
 tool hook paths.
 
+Codex observations include `incompleteReasons`: an empty array for a complete
+collection, or all applicable reason codes for a partial collection. Older
+observations without this field have unknown reasons; their missing details
+cannot be reconstructed from balanced native call/return counts.
+
+| Reason                                           | Meaning                                                                                            |
+| ------------------------------------------------ | -------------------------------------------------------------------------------------------------- |
+| `host_incomplete`                                | Host execution did not complete.                                                                   |
+| `catalog_incomplete`                             | The prepared catalog could not be collected completely.                                            |
+| `receipts_unavailable`                           | The receipt file is missing or unreadable.                                                         |
+| `invalid_receipts`                               | A malformed receipt stopped processing; earlier valid receipts remain available.                   |
+| `receipt_limit`                                  | The receipt file exceeds the collection byte limit.                                                |
+| `observer_failure`                               | The hook recorded a failure or writer limit, including input, nesting or matching-budget failures. |
+| `session_identity_missing`                       | No root session identity was available.                                                            |
+| `session_start_missing` / `session_stop_missing` | The root session lacks the corresponding lifecycle hook.                                           |
+| `pending_tool_calls`                             | Pre-tool hooks remain without matching post-tool hooks.                                            |
+| `unmatched_post_tool_use`                        | Post-tool hooks were received without pending pre-tool hooks.                                      |
+| `duplicate_tool_calls`                           | A pre-tool identity was already pending or finished.                                               |
+| `partial_hook_events`                            | A hook receipt is partial independently of body ambiguity.                                         |
+| `ambiguous_skill_bodies`                         | Complete body exposure could not distinguish skill names.                                          |
+| `partial_skill_bodies`                           | Recognized fragments remain without complete body exposure.                                        |
+| `calls_truncated`                                | Relevant call evidence exceeded its public count or byte budget.                                   |
+
+The `collection` summary reports `eventCount`, `preToolUseCount`,
+`postToolUseCount`, `pendingToolCallCount`, `unmatchedPostToolUseCount`,
+`duplicateToolCallCount` and `partialEventCount`, plus `sessionIdentified`,
+`sessionStarted` and `sessionStopped` for the root session. Counts cover all
+successfully parsed receipts, including receipts omitted from `calls`; an
+unreadable or invalid tail remains unknown. Tool pairing spans the observed
+sessions and uses session and tool-use identities. A later balanced pair is
+still counted after an earlier anomaly.
+
+`calls` is a bounded selection of complete matches, fragment-only matches,
+ambiguous or partial receipts, and receipts associated with pending, orphaned
+or duplicate tool identities. Each entry's `fragmentSkills` names recognizable
+fragments without exposing their body offsets or digests. A fragment entry can
+remain in `calls` after later complete exposure resolves `partialSkills`.
+Neither `partial: false` on an individual receipt nor an empty `partialSkills`
+array proves aggregate completeness. `truncated` describes this selected call
+list, which is never the complete observer event ledger. A pending tool hook
+can follow a rejected launch or a missing hook; it does not establish either
+cause, and a tool error with a balanced hook pair does not itself imply an
+observer gap. These diagnostics preserve unknown activation verdicts and do
+not recover activation from native transcripts.
+
+Codex also reports `primarySkill` and `primarySkillCompleteness` separately
+from whole-collection completeness. A complete collection establishes its first
+observed skill when the first matching receipt credits exactly one skill, or
+`null` when none was observed. Multiple bodies credited by the first receipt
+leave primary selection unknown even with complete collection; their catalog
+order does not prove selection order. A partial collection can still
+establish the primary skill when its only gap is pending tool calls, exactly one
+skill was credited by the first matching receipt, and every call started through
+that receipt has a matching post hook. A later pending call cannot change that
+established first selection. An earlier or overlapping pending call leaves the
+primary unknown (`null` with `primarySkillCompleteness: partial`). Other
+collection gaps remain conservative. Consumers may use the established primary
+for first-selection assertions while retaining the partial collection status;
+this claim does not establish exclusions or a complete supporting sequence.
+When a partial collection establishes the primary, the same collector emits
+`sevro.host.skill-activation-claims` as complete evidence with scope
+`primary_and_positive_presence`, the qualified `primarySkill`, and `presentSkills`.
+It retains `collectionCompleteness: partial`. Consumers cite this receipt for
+those specific claims, preserving the protocol's requirement that passing
+results cite complete evidence. It cannot establish an exclusion or an
+unobserved supporting read and is absent when the primary is unknown.
+
 Claude's `skill_tool_result` method observes its existing structured `Skill`
 stream without enabling additional command hooks. It pairs each call with a
 successful result under the same session, parent actor and tool-use ID.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bjoernrochel/sevro",
-  "version": "0.1.0-rc.19",
+  "version": "0.1.0-rc.20",
   "license": "SEE LICENSE IN LICENSE",
   "description": "A standalone evaluation runner for agentic coding capabilities",
   "type": "module",

package/src/hosts/codex.ts

@@ -44,6 +44,7 @@ import { codexSkillCatalogRoots } from "./skill-hook-catalog";
 import {
   prepareSkillTracking,
   skillTrackingObservation,
+  skillActivationClaims,
   type SkillTracking,
 } from "./skill-hooks";
 import {
@@ -139,6 +140,7 @@ function codexCapabilities(
     "sevro.host.native-transcripts",
     "sevro.host.hooks",
     "sevro.host.skill-activations",
+    "sevro.host.skill-activation-claims",
     ...(entrypoint === "app-server" ? ["sevro.host.native-goal"] : []),
     "sevro.codex.plugin-marketplace",
     "sevro.codex.explicit-invocation",
@@ -732,6 +734,11 @@ function appServerObservation(result: AppServerResult | null): Observation[] {
     : [];
 }
 async function codexObservations(context: ExecutionContext, turns: TurnResult) {
+  const tracking = await skillTrackingObservation(
+    context.skillTracking,
+    turns.summary.threadId,
+    turns.summary.complete,
+  );
   const native = await codexNativeCallObservation(
     context.state.codexHome,
     turns.summary.threadId,
@@ -752,11 +759,8 @@ async function codexObservations(context: ExecutionContext, turns: TurnResult) {
     codexNativeControls(native),
     ...(turns.continuation ? [turns.continuation] : []),
     ...(explicit ? [explicit] : []),
-    await skillTrackingObservation(
-      context.skillTracking,
-      turns.summary.threadId,
-      turns.summary.complete,
-    ),
+    tracking,
+    ...skillActivationClaims(tracking),
   ];
 }
 

package/src/hosts/skill-hooks.ts

@@ -142,69 +142,107 @@ function receiptLabels(value: Record<string, unknown>): boolean {
 
 async function receipts(tracking: SkillTracking) {
   const root = tracking.configuration.receipts;
-  const result = { events: [] as SkillHookReceipt[], partial: false };
+  const result = { events: [] as SkillHookReceipt[], reasons: [] as string[] };
+  if (await Bun.file(join(root, "partial")).exists())
+    result.reasons.push("observer_failure");
   try {
     const path = join(root, "events.jsonl");
     if ((await stat(path)).size > MAX_EVENTS_BYTES)
-      throw new Error("oversized skill hook receipts");
-    result.partial = await Bun.file(join(root, "partial")).exists();
-    for (const line of (await readFile(path, "utf8")).trimEnd().split("\n")) {
-      const event: unknown = JSON.parse(line);
-      if (!isReceipt(event, tracking.configuration.skills))
-        throw new Error("invalid skill hook receipt");
-      result.events.push(event);
-    }
-  } catch {
-    result.partial = true;
+      return { ...result, reasons: [...result.reasons, "receipt_limit"] };
+    parseReceipts(
+      await readFile(path, "utf8"),
+      tracking.configuration.skills,
+      result.events,
+    );
+  } catch (error) {
+    result.reasons.push(
+      error instanceof SyntaxError
+        ? "invalid_receipts"
+        : "receipts_unavailable",
+    );
   }
   return result;
 }
 
+function parseReceipts(
+  text: string,
+  skills: HookSkill[],
+  events: SkillHookReceipt[],
+) {
+  for (const line of text.trimEnd().split("\n")) {
+    const event: unknown = JSON.parse(line);
+    if (!isReceipt(event, skills))
+      throw new SyntaxError("invalid skill hook receipt");
+    events.push(event);
+  }
+}
+
 function hookKey(event: SkillHookReceipt): string {
   return JSON.stringify([event.sessionId, event.toolUseId]);
 }
 
-function collectionComplete(
+function collectionSummary(
   events: SkillHookReceipt[],
   sessionId: string | null,
-): boolean {
-  if (!lifecycleAvailable(events, sessionId)) return false;
+) {
   const ledger = {
     pending: new Set<string>(),
     finished: new Set<string>(),
-    partial: false,
+    diagnosticKeys: new Set<string>(),
+    duplicateToolCallCount: 0,
+    unmatchedPostToolUseCount: 0,
   };
   for (const event of events) updatePending(event, ledger);
-  return (
-    ledger.pending.size === 0 &&
-    !ledger.partial &&
-    events.every((event) => !event.partial)
-  );
+  return {
+    diagnosticKeys: new Set([...ledger.diagnosticKeys, ...ledger.pending]),
+    eventCount: events.length,
+    preToolUseCount: events.filter((event) => event.event === "PreToolUse")
+      .length,
+    postToolUseCount: events.filter((event) => event.event === "PostToolUse")
+      .length,
+    pendingToolCallCount: ledger.pending.size,
+    duplicateToolCallCount: ledger.duplicateToolCallCount,
+    unmatchedPostToolUseCount: ledger.unmatchedPostToolUseCount,
+    partialEventCount: events.filter((event) => event.partial).length,
+    ...lifecycleSummary(events, sessionId),
+  };
 }
 
-function lifecycleAvailable(
+function lifecycleSummary(
   events: SkillHookReceipt[],
   sessionId: string | null,
-): boolean {
+) {
   const selected = events.filter((event) => event.sessionId === sessionId);
-  return (
-    sessionId !== null &&
-    selected.some((event) => event.event === "SessionStart") &&
-    selected.some((event) => event.event === "Stop")
-  );
+  return {
+    sessionIdentified: sessionId !== null,
+    sessionStarted: selected.some((event) => event.event === "SessionStart"),
+    sessionStopped: selected.some((event) => event.event === "Stop"),
+  };
 }
 
 function updatePending(
   event: SkillHookReceipt,
-  ledger: { pending: Set<string>; finished: Set<string>; partial: boolean },
+  ledger: {
+    pending: Set<string>;
+    finished: Set<string>;
+    diagnosticKeys: Set<string>;
+    duplicateToolCallCount: number;
+    unmatchedPostToolUseCount: number;
+  },
 ) {
   const key = hookKey(event);
   if (event.event === "PreToolUse") {
-    ledger.partial ||= duplicateCall(key, ledger);
+    if (duplicateCall(key, ledger)) {
+      ledger.duplicateToolCallCount++;
+      ledger.diagnosticKeys.add(key);
+    }
     ledger.pending.add(key);
   }
   if (event.event === "PostToolUse") {
-    ledger.partial ||= !ledger.pending.delete(key);
+    if (!ledger.pending.delete(key)) {
+      ledger.unmatchedPostToolUseCount++;
+      ledger.diagnosticKeys.add(key);
+    }
     ledger.finished.add(key);
   }
 }
@@ -299,7 +337,12 @@ function pagedEvents(events: SkillHookReceipt[], skills: HookSkill[]) {
 function publicReceipt(event: SkillHookReceipt) {
   const receipt = { ...event };
   delete receipt.bodyRanges;
-  return receipt;
+  return {
+    ...receipt,
+    fragmentSkills: [
+      ...new Set((event.bodyRanges ?? []).map((page) => page.invocation)),
+    ],
+  };
 }
 
 function incompleteSkills(events: SkillHookReceipt[]) {
@@ -317,10 +360,15 @@ function incompleteSkills(events: SkillHookReceipt[]) {
   return [...fragments].filter((invocation) => !settled.has(invocation));
 }
 
-function retainedCalls(events: SkillHookReceipt[]) {
-  const calls: SkillHookReceipt[] = [];
+function retainedCalls(
+  events: SkillHookReceipt[],
+  diagnosticKeys: Set<string>,
+) {
+  const calls: ReturnType<typeof publicReceipt>[] = [];
   let truncated = false;
-  for (const event of events.filter((event) => event.invocations.length > 0)) {
+  for (const event of events.filter((event) =>
+    relevantCall(event, diagnosticKeys),
+  )) {
     calls.push(publicReceipt(event));
     if (
       calls.length > MAX_RETAINED_CALLS ||
@@ -334,24 +382,113 @@ function retainedCalls(events: SkillHookReceipt[]) {
   return { calls, truncated };
 }
 
+function relevantCall(
+  event: SkillHookReceipt,
+  diagnosticKeys: Set<string>,
+): boolean {
+  return [
+    event.invocations.length > 0,
+    event.ambiguousInvocations.length > 0,
+    (event.bodyRanges ?? []).length > 0,
+    event.partial,
+    diagnosticKeys.has(hookKey(event)),
+  ].some(Boolean);
+}
+
+function primaryEvidence(events: SkillHookReceipt[], reasons: string[]) {
+  const index = events.findIndex((event) => event.invocations.length > 0);
+  const first = events[index];
+  const complete = primarySelectionComplete(events, index, reasons);
+  if (!complete)
+    return { primarySkill: null, primarySkillCompleteness: "partial" as const };
+  return {
+    primarySkill: first?.invocations[0] ?? null,
+    primarySkillCompleteness: "complete" as const,
+  };
+}
+
+function primarySelectionComplete(
+  events: SkillHookReceipt[],
+  index: number,
+  reasons: string[],
+) {
+  const matches = events[index]?.invocations.length ?? 0;
+  if (matches > 1) return false;
+  if (reasons.length === 0) return true;
+  return primaryBeforeGap(events, index, reasons);
+}
+
+function primaryBeforeGap(
+  events: SkillHookReceipt[],
+  index: number,
+  reasons: string[],
+) {
+  if (reasons.join(",") !== "pending_tool_calls") return false;
+  if (events[index]?.invocations.length !== 1) return false;
+  const prefix = collectionSummary(events.slice(0, index + 1), null);
+  return prefix.pendingToolCallCount === 0;
+}
+
+export function skillActivationClaims(
+  observation: Awaited<ReturnType<typeof skillTrackingObservation>>,
+) {
+  if (observation.completeness !== "partial") return [];
+  if (observation.data.primarySkillCompleteness !== "complete") return [];
+  return [
+    {
+      id: "sevro.host.skill-activation-claims",
+      completeness: "complete" as const,
+      data: {
+        method: "tool_result_body_match",
+        scope: "primary_and_positive_presence",
+        collectionCompleteness: "partial",
+        primarySkill: observation.data.primarySkill,
+        presentSkills: observation.data.observedSkills,
+      },
+    },
+  ];
+}
+
 export async function skillTrackingObservation(
   tracking: SkillTracking,
   sessionId: string | null,
   completed: boolean,
 ) {
   const collected = await receipts(tracking);
-  const { partial } = collected;
   const events = pagedEvents(collected.events, tracking.configuration.skills);
   const partialSkills = incompleteSkills(events);
-  const { calls, truncated } = retainedCalls(events);
-  const complete = [
-    completed,
-    !partial,
-    !truncated,
-    partialSkills.length === 0,
-    tracking.configuration.catalogComplete,
-    collectionComplete(events, sessionId),
-  ].every(Boolean);
+  const { diagnosticKeys, ...collection } = collectionSummary(
+    events,
+    sessionId,
+  );
+  const { calls, truncated } = retainedCalls(events, diagnosticKeys);
+  const ambiguousSkills = [
+    ...new Set(events.flatMap((event) => event.ambiguousInvocations)),
+  ];
+  const conditions: [string, boolean][] = [
+    ["host_incomplete", !completed],
+    ["catalog_incomplete", !tracking.configuration.catalogComplete],
+    ["session_identity_missing", !collection.sessionIdentified],
+    ["session_start_missing", !collection.sessionStarted],
+    ["session_stop_missing", !collection.sessionStopped],
+    ["pending_tool_calls", collection.pendingToolCallCount > 0],
+    ["unmatched_post_tool_use", collection.unmatchedPostToolUseCount > 0],
+    ["duplicate_tool_calls", collection.duplicateToolCallCount > 0],
+    [
+      "partial_hook_events",
+      events.some(
+        (event) => event.partial && event.ambiguousInvocations.length === 0,
+      ),
+    ],
+    ["ambiguous_skill_bodies", ambiguousSkills.length > 0],
+    ["partial_skill_bodies", partialSkills.length > 0],
+    ["calls_truncated", truncated],
+  ];
+  const incompleteReasons = [
+    ...collected.reasons,
+    ...conditions.filter(([, applies]) => applies).map(([reason]) => reason),
+  ];
+  const complete = incompleteReasons.length === 0;
   return {
     id: "sevro.host.skill-activations",
     completeness: complete ? ("complete" as const) : ("partial" as const),
@@ -361,12 +498,13 @@ export async function skillTrackingObservation(
       scope: "prepared_skill_bodies",
       catalogComplete: tracking.configuration.catalogComplete,
       observedSkills: [...new Set(calls.flatMap((call) => call.invocations))],
+      ...primaryEvidence(events, incompleteReasons),
       partialSkills,
-      ambiguousSkills: [
-        ...new Set(events.flatMap((event) => event.ambiguousInvocations)),
-      ],
+      ambiguousSkills,
       calls,
       truncated,
+      incompleteReasons,
+      collection,
     },
   };
 }