@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +170 -0
- package/LICENSE +63 -136
- package/README.md +83 -66
- package/docs/README.md +44 -0
- package/docs/acknowledgements.md +14 -0
- package/docs/architecture.md +146 -0
- package/docs/assets/README.md +27 -0
- package/docs/assets/sevro-logo.png +0 -0
- package/docs/assets/sevro-symbol.svg +1 -0
- package/docs/assets/sevro-wordmark.svg +1 -0
- package/docs/builtin-graders-v1.md +41 -13
- package/docs/codex-host.md +184 -52
- package/docs/contracts.md +19 -0
- package/docs/creating-cases.md +139 -0
- package/docs/development-cli.md +190 -30
- package/docs/documentation-quality.md +61 -0
- package/docs/evaluation-workflow.md +110 -0
- package/docs/evidence/code-mode-controls-71.md +64 -0
- package/docs/evidence/code-mode-programs-85.md +64 -0
- package/docs/evidence/codex-state-73/README.md +204 -0
- package/docs/evidence/codex-state-73/darrow-migration.patch +182 -0
- package/docs/evidence/codex-state-73/results.json +661 -0
- package/docs/evidence/local-http-74.md +104 -0
- package/docs/evidence/runtime-startup-30/README.md +185 -0
- package/docs/evidence/runtime-startup-30/after.json +319 -0
- package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +2 -0
- package/docs/evidence/runtime-startup-30/before.json +403 -0
- package/docs/evidence/runtime-startup-30/consumer-smoke.json +63 -0
- package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +3 -0
- package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +3 -0
- package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +20 -0
- package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +2 -0
- package/docs/evidence/skill-reads-70/results.json +823 -0
- package/docs/evidence.md +21 -0
- package/docs/extension-protocol-v1.md +50 -16
- package/docs/fixture-cleanup-validation.md +3 -0
- package/docs/getting-started.md +102 -0
- package/docs/guide-evaluation.md +89 -0
- package/docs/guide-verification.json +239 -0
- package/docs/identity-v1.md +35 -1
- package/docs/installing.md +88 -0
- package/docs/licenses/BUSL-1.1-0.1.0-rc.2.txt +159 -0
- package/docs/licensing.md +42 -0
- package/docs/native-controls.md +76 -5
- package/docs/native-hosts.md +228 -0
- package/docs/reading-results.md +141 -0
- package/docs/recipes.md +489 -0
- package/docs/releases.md +81 -40
- package/docs/report-v1.md +29 -1
- package/docs/results-v1.md +180 -0
- package/docs/run-settings-v1.md +147 -0
- package/docs/running-evaluations.md +31 -0
- package/docs/runtime-v1.md +614 -0
- package/docs/specs/repository-guide.md +58 -0
- package/docs/troubleshooting.md +30 -0
- package/docs/typescript-quality.md +234 -0
- package/docs/usability-study.md +114 -0
- package/examples/basic/host.ts +2 -2
- package/examples/recipes/candidate.ts +26 -0
- package/examples/recipes/cases/coding-change.json +21 -0
- package/examples/recipes/cases/host-tools.json +21 -0
- package/examples/recipes/cases/mise.json +21 -0
- package/examples/recipes/cases/node.json +21 -0
- package/examples/recipes/cases/python-uv.json +21 -0
- package/examples/recipes/cases/semantic.json +17 -0
- package/examples/recipes/coding-change/check.ts +23 -0
- package/examples/recipes/extension-artifacts.ts +65 -0
- package/examples/recipes/extension-methods.ts +174 -0
- package/examples/recipes/extension.ts +37 -0
- package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/README.md +3 -0
- package/examples/recipes/run-inputs.ts +188 -0
- package/examples/recipes/run-options.ts +131 -0
- package/examples/recipes/run.ts +63 -0
- package/examples/recipes/runtime/empty.json +1 -0
- package/examples/recipes/runtime/goals.json +1 -0
- package/examples/recipes/runtime/mise.json +6 -0
- package/examples/recipes/runtime/path.json +1 -0
- package/examples/recipes/runtime/plugins.json +4 -0
- package/examples/recipes/runtime/python-uv.json +10 -0
- package/examples/recipes/semantic-judge.ts +29 -0
- package/examples/templates/coding/adapter.ts +17 -0
- package/examples/templates/coding/case.json +15 -0
- package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
- package/examples/templates/repository-skill/extension-data.ts +64 -0
- package/examples/templates/repository-skill/extension.ts +31 -0
- package/examples/templates/response/adapter.ts +12 -0
- package/examples/templates/response/case.json +13 -0
- package/package.json +39 -6
- package/schemas/report-v1.schema.json +9 -0
- package/schemas/run-evidence-v1.schema.json +2 -1
- package/schemas/runtime-v1.schema.json +193 -0
- package/src/advisory.ts +10 -5
- package/src/candidate-artifacts.ts +208 -0
- package/src/candidate-codex-state.ts +185 -0
- package/src/case-validation.ts +105 -0
- package/src/cli/cancellation.ts +20 -0
- package/src/cli/command.ts +81 -0
- package/src/cli/completion.ts +15 -0
- package/src/cli/demo.ts +59 -0
- package/src/cli/evaluation.ts +55 -0
- package/src/cli/extension.ts +79 -0
- package/src/cli/help.ts +222 -0
- package/src/cli/hosts.ts +59 -0
- package/src/cli/init-settings.ts +30 -0
- package/src/cli/init.ts +111 -0
- package/src/cli/invocation.ts +627 -0
- package/src/cli/loading.ts +151 -0
- package/src/cli/preparation.ts +163 -0
- package/src/cli/presentation.ts +91 -0
- package/src/cli/progress.ts +6 -0
- package/src/cli/run-manifest.ts +133 -0
- package/src/cli/run-options.ts +101 -0
- package/src/cli/run-overrides.ts +83 -0
- package/src/cli/settings.ts +160 -0
- package/src/cli/summary.ts +43 -0
- package/src/cli.ts +13 -843
- package/src/contract-types.ts +94 -0
- package/src/engine.ts +19 -2373
- package/src/evaluation/advisory.ts +150 -0
- package/src/evaluation/check-plan.ts +510 -0
- package/src/evaluation/collection.ts +88 -0
- package/src/evaluation/context.ts +10 -0
- package/src/evaluation/grading-host.ts +39 -0
- package/src/evaluation/grading.ts +258 -0
- package/src/evaluation/invocation-plan.ts +548 -0
- package/src/evaluation/plan.ts +736 -0
- package/src/evaluation/progress.ts +19 -0
- package/src/evaluation/resources.ts +241 -0
- package/src/evaluation/run.ts +206 -0
- package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +3 -1
- package/src/evaluation/semantic.ts +312 -0
- package/src/evaluation/trial.ts +866 -0
- package/src/evaluation/types.ts +207 -0
- package/src/evidence/host-results.ts +92 -0
- package/src/evidence/retained-artifacts.ts +18 -0
- package/src/evidence/trial-writer.ts +203 -0
- package/src/evidence/usage.ts +17 -0
- package/src/extension-client.ts +228 -148
- package/src/extension-session.ts +326 -195
- package/src/fixtures/advisory.ts +338 -0
- package/src/fixtures/bin.ts +57 -0
- package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
- package/src/fixtures/create.ts +187 -0
- package/src/fixtures/generated.ts +300 -0
- package/src/fixtures/git-hooks.ts +24 -0
- package/src/fixtures/preparation.ts +270 -0
- package/src/{repository-fixture.ts → fixtures/repository.ts} +207 -85
- package/src/fixtures/setup.ts +224 -0
- package/src/fixtures/tools.ts +88 -0
- package/src/fixtures/workspace-state.ts +212 -0
- package/src/generated/cli-result.cjs +2 -0
- package/src/generated/cli-result.d.cts +3 -0
- package/src/generated/extension.cjs +2 -0
- package/src/generated/extension.d.cts +3 -0
- package/src/generated/report.cjs +2 -0
- package/src/generated/report.d.cts +3 -0
- package/src/generated/run-evidence.cjs +2 -0
- package/src/generated/run-evidence.d.cts +3 -0
- package/src/generated/runtime.cjs +2 -0
- package/src/generated/runtime.d.cts +3 -0
- package/src/graders/artifact.ts +56 -23
- package/src/graders/git-head.ts +79 -75
- package/src/graders/output.ts +222 -140
- package/src/graders/semantic.ts +102 -43
- package/src/graders/shell.ts +379 -207
- package/src/host-diagnostic-redaction.ts +52 -0
- package/src/host-diagnostics.ts +319 -0
- package/src/hosts/claude-continuation.ts +63 -18
- package/src/hosts/claude-credential.ts +43 -62
- package/src/hosts/claude-events.ts +86 -51
- package/src/hosts/claude-keychain.ts +59 -0
- package/src/hosts/claude-native-goal.ts +57 -33
- package/src/hosts/claude-process.ts +111 -0
- package/src/hosts/claude-repository-invocation.ts +208 -117
- package/src/hosts/claude-settings.ts +240 -21
- package/src/hosts/claude-skill-activations.ts +166 -0
- package/src/hosts/claude-state.ts +403 -0
- package/src/hosts/claude-tool-calls.ts +171 -106
- package/src/hosts/claude.ts +561 -451
- package/src/hosts/codex-app-server.ts +326 -178
- package/src/hosts/codex-configuration.ts +44 -18
- package/src/hosts/codex-events.ts +97 -60
- package/src/hosts/codex-exec-controls.ts +516 -0
- package/src/hosts/codex-invocation.ts +70 -0
- package/src/hosts/codex-marketplace.ts +283 -0
- package/src/hosts/codex-native-calls.ts +546 -500
- package/src/hosts/codex-process.ts +202 -0
- package/src/hosts/codex-profile.ts +257 -28
- package/src/hosts/codex-session-files.ts +106 -0
- package/src/hosts/codex-state.ts +492 -0
- package/src/hosts/codex.ts +783 -832
- package/src/hosts/isolation-common.ts +94 -0
- package/src/hosts/isolation-roots.ts +54 -27
- package/src/hosts/isolation.ts +10 -0
- package/src/hosts/linux-sandbox.ts +159 -0
- package/src/hosts/mac-sandbox.ts +187 -61
- package/src/hosts/native-controls.ts +152 -62
- package/src/hosts/process-lifecycle.ts +37 -0
- package/src/hosts/protected-worktrees.ts +51 -19
- package/src/hosts/skill-hook-catalog.ts +131 -0
- package/src/hosts/skill-hook-handler.ts +374 -0
- package/src/hosts/skill-hooks.ts +580 -0
- package/src/hosts/workspace-fingerprint.ts +55 -35
- package/src/identity.ts +102 -48
- package/src/instrumentation.ts +109 -48
- package/src/native-transcript-state.ts +131 -0
- package/src/native-transcript-view.ts +90 -0
- package/src/native-transcripts.ts +286 -0
- package/src/process-lifecycle.ts +150 -0
- package/src/provenance.ts +161 -81
- package/src/prune.ts +215 -0
- package/src/report.ts +172 -140
- package/src/resolved-case.ts +33 -0
- package/src/results-lock.ts +29 -0
- package/src/results.ts +133 -86
- package/src/run-owner.ts +154 -63
- package/src/runtime/config.ts +456 -0
- package/src/runtime/discovery-apple.ts +90 -0
- package/src/runtime/discovery-providers.ts +63 -0
- package/src/runtime/discovery.ts +70 -0
- package/src/runtime/hook-processes.ts +97 -0
- package/src/runtime/hooks.ts +476 -0
- package/src/runtime/paths.ts +313 -0
- package/src/runtime/query.ts +85 -0
- package/src/runtime/seeds.ts +134 -0
- package/src/runtime/shared-cache.ts +143 -0
- package/src/runtime/state.ts +143 -0
- package/src/schema-types.ts +198 -0
- package/src/schema.ts +15 -19
- package/src/transcript-retention.ts +146 -0
- package/src/value-guards.ts +20 -0
- package/src/advisory-fixture.ts +0 -282
- package/src/fixture-bin.ts +0 -87
- package/src/fixture-setup.ts +0 -163
- package/src/generated-fixture.ts +0 -220
- package/src/git-hooks.ts +0 -63
- package/src/hosts/claude-nested-skills.ts +0 -218
- package/src/hosts/codex-command-output.ts +0 -399
- package/src/hosts/codex-executor-command.ts +0 -195
- package/src/hosts/codex-skill-reads.ts +0 -391
- package/src/preparation.ts +0 -211
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { isRecord } from "../value-guards";
|
|
1
2
|
const MAX_EVENT_STREAM_BYTES = 8 * 1024 * 1024;
|
|
2
3
|
|
|
3
4
|
export class ClaudeEventError extends Error {
|
|
@@ -9,26 +10,25 @@ export class ClaudeEventError extends Error {
|
|
|
9
10
|
|
|
10
11
|
type Entry = Record<string, unknown>;
|
|
11
12
|
|
|
12
|
-
function record(value: unknown): value is Entry {
|
|
13
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
14
|
-
}
|
|
15
|
-
|
|
16
13
|
function nonnegative(value: unknown): value is number {
|
|
17
14
|
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
18
15
|
}
|
|
19
16
|
|
|
20
17
|
function validAssistant(value: Entry): boolean {
|
|
21
|
-
if (!
|
|
18
|
+
if (!isRecord(value.message) || !Array.isArray(value.message.content))
|
|
22
19
|
return false;
|
|
23
|
-
return value.message.content.every(
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
20
|
+
return value.message.content.every(validAssistantBlock);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function validAssistantBlock(block: unknown): boolean {
|
|
24
|
+
if (!isRecord(block)) return false;
|
|
25
|
+
if (block.type !== "tool_use" || block.name !== "Skill") return true;
|
|
26
|
+
return validSkillInput(block.input);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function validSkillInput(input: unknown): boolean {
|
|
30
|
+
if (!isRecord(input) || typeof input.skill !== "string") return false;
|
|
31
|
+
return !!input.skill.split(":").at(-1)?.trim();
|
|
32
32
|
}
|
|
33
33
|
|
|
34
34
|
function validResult(value: Entry): boolean {
|
|
@@ -41,7 +41,7 @@ function validResult(value: Entry): boolean {
|
|
|
41
41
|
}
|
|
42
42
|
|
|
43
43
|
function validEvent(value: unknown): value is Entry {
|
|
44
|
-
if (!
|
|
44
|
+
if (!isRecord(value)) return false;
|
|
45
45
|
if (value.type === "result") return validResult(value);
|
|
46
46
|
if (value.type === "assistant") return validAssistant(value);
|
|
47
47
|
return typeof value.type === "string";
|
|
@@ -53,22 +53,20 @@ interface Usage {
|
|
|
53
53
|
}
|
|
54
54
|
|
|
55
55
|
function usage(value: unknown): Usage | null {
|
|
56
|
-
if (!
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
return
|
|
69
|
-
|
|
70
|
-
outputTokens: output,
|
|
71
|
-
};
|
|
56
|
+
if (!isRecord(value)) return null;
|
|
57
|
+
const inputTokens = inputUsage(value);
|
|
58
|
+
if (inputTokens === null || !nonnegative(value.output_tokens)) return null;
|
|
59
|
+
return { inputTokens, outputTokens: value.output_tokens };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function inputUsage(value: Entry): number | null {
|
|
63
|
+
const counts = [
|
|
64
|
+
value.input_tokens,
|
|
65
|
+
value.cache_creation_input_tokens ?? 0,
|
|
66
|
+
value.cache_read_input_tokens ?? 0,
|
|
67
|
+
];
|
|
68
|
+
if (!counts.every(nonnegative)) return null;
|
|
69
|
+
return counts.reduce((total, count) => total + count);
|
|
72
70
|
}
|
|
73
71
|
|
|
74
72
|
export interface ClaudeEventSummary {
|
|
@@ -87,42 +85,79 @@ export function summarizeClaudeEvents(
|
|
|
87
85
|
): ClaudeEventSummary {
|
|
88
86
|
if (Buffer.byteLength(stream, "utf8") > MAX_EVENT_STREAM_BYTES)
|
|
89
87
|
throw new ClaudeEventError("Claude event stream exceeds 8 MiB");
|
|
88
|
+
const { results, malformed } = parseResultEvents(stream);
|
|
89
|
+
return resultSummary(results, malformed, exitCode);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function parseResultEvents(stream: string) {
|
|
90
93
|
const results: Entry[] = [];
|
|
91
94
|
let malformed = false;
|
|
92
95
|
for (const line of stream.split("\n")) {
|
|
93
96
|
if (!line.trim()) continue;
|
|
94
97
|
try {
|
|
95
|
-
const event
|
|
96
|
-
if (
|
|
97
|
-
else if (event.type === "result") results.push(event);
|
|
98
|
+
const event = parseValidEvent(line);
|
|
99
|
+
if (event.type === "result") results.push(event);
|
|
98
100
|
} catch {
|
|
99
101
|
malformed = true;
|
|
100
102
|
}
|
|
101
103
|
}
|
|
102
|
-
|
|
104
|
+
return { results, malformed };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function parseValidEvent(line: string): Entry {
|
|
108
|
+
const event: unknown = JSON.parse(line);
|
|
109
|
+
if (!validEvent(event)) throw new ClaudeEventError("Malformed Claude event");
|
|
110
|
+
return event;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function summarizeUsage(results: Entry[], malformed: boolean) {
|
|
103
114
|
const usages = results.map((result) => usage(result.usage));
|
|
115
|
+
const validUsages = usages.filter((value): value is Usage => value !== null);
|
|
104
116
|
const usageComplete =
|
|
105
|
-
!malformed && results.length > 0 && usages.
|
|
106
|
-
const costs = results.map((result) => result.total_cost_usd);
|
|
117
|
+
!malformed && results.length > 0 && validUsages.length === usages.length;
|
|
107
118
|
return {
|
|
108
|
-
complete:
|
|
109
|
-
!malformed &&
|
|
110
|
-
exitCode === 0 &&
|
|
111
|
-
results.length > 0 &&
|
|
112
|
-
results.every(
|
|
113
|
-
(result) => result.subtype === "success" && result.is_error === false,
|
|
114
|
-
),
|
|
115
|
-
finalMessage: typeof final?.result === "string" ? final.result : null,
|
|
116
119
|
inputTokens: usageComplete
|
|
117
|
-
?
|
|
120
|
+
? validUsages.reduce((total, value) => total + value.inputTokens, 0)
|
|
118
121
|
: null,
|
|
119
122
|
outputTokens: usageComplete
|
|
120
|
-
?
|
|
123
|
+
? validUsages.reduce((total, value) => total + value.outputTokens, 0)
|
|
121
124
|
: null,
|
|
122
|
-
costUsd:
|
|
123
|
-
!malformed && results.length && costs.every(nonnegative)
|
|
124
|
-
? costs.reduce((total, value) => total + (value as number), 0)
|
|
125
|
-
: null,
|
|
126
125
|
usageComplete,
|
|
127
126
|
};
|
|
128
127
|
}
|
|
128
|
+
|
|
129
|
+
function measuredCost(results: Entry[], malformed: boolean): number | null {
|
|
130
|
+
const costs = results.map((result) => result.total_cost_usd);
|
|
131
|
+
if (malformed || results.length === 0 || !costs.every(nonnegative))
|
|
132
|
+
return null;
|
|
133
|
+
return costs.reduce((total, value) => total + value, 0);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function resultSummary(
|
|
137
|
+
results: Entry[],
|
|
138
|
+
malformed: boolean,
|
|
139
|
+
exitCode: number,
|
|
140
|
+
): ClaudeEventSummary {
|
|
141
|
+
const final = results.at(-1);
|
|
142
|
+
return {
|
|
143
|
+
complete: completedResults(results, malformed, exitCode),
|
|
144
|
+
finalMessage: typeof final?.result === "string" ? final.result : null,
|
|
145
|
+
costUsd: measuredCost(results, malformed),
|
|
146
|
+
...summarizeUsage(results, malformed),
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function completedResults(
|
|
151
|
+
results: Entry[],
|
|
152
|
+
malformed: boolean,
|
|
153
|
+
exitCode: number,
|
|
154
|
+
): boolean {
|
|
155
|
+
return (
|
|
156
|
+
!malformed &&
|
|
157
|
+
exitCode === 0 &&
|
|
158
|
+
results.length > 0 &&
|
|
159
|
+
results.every(
|
|
160
|
+
(result) => result.subtype === "success" && result.is_error === false,
|
|
161
|
+
)
|
|
162
|
+
);
|
|
163
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
|
|
3
|
+
const MAX_CREDENTIAL_BYTES = 1024 * 1024;
|
|
4
|
+
|
|
5
|
+
function keychainProcess() {
|
|
6
|
+
if (process.platform !== "darwin" || !existsSync("/usr/bin/security"))
|
|
7
|
+
throw new Error("Claude keychain credential is unavailable");
|
|
8
|
+
const account = process.env.USER ?? process.env.LOGNAME;
|
|
9
|
+
return Bun.spawn(
|
|
10
|
+
[
|
|
11
|
+
"/usr/bin/security",
|
|
12
|
+
"find-generic-password",
|
|
13
|
+
"-s",
|
|
14
|
+
"Claude Code-credentials",
|
|
15
|
+
...(account ? ["-a", account] : []),
|
|
16
|
+
"-w",
|
|
17
|
+
],
|
|
18
|
+
{ stdout: "pipe", stderr: "ignore" },
|
|
19
|
+
);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
async function boundedCredentialStream(
|
|
23
|
+
reader: ReadableStreamDefaultReader<Uint8Array>,
|
|
24
|
+
): Promise<Buffer> {
|
|
25
|
+
const chunks: Uint8Array[] = [];
|
|
26
|
+
let size = 0;
|
|
27
|
+
for (;;) {
|
|
28
|
+
const { value, done } = await reader.read();
|
|
29
|
+
if (done) break;
|
|
30
|
+
size += value.byteLength;
|
|
31
|
+
if (size > MAX_CREDENTIAL_BYTES)
|
|
32
|
+
throw new Error("Claude credential exceeds the size limit");
|
|
33
|
+
chunks.push(value);
|
|
34
|
+
}
|
|
35
|
+
return Buffer.concat(chunks);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export async function keychainCredential(): Promise<Buffer> {
|
|
39
|
+
const proc = keychainProcess();
|
|
40
|
+
const timer = setTimeout(() => {
|
|
41
|
+
proc.kill("SIGKILL");
|
|
42
|
+
}, 10_000);
|
|
43
|
+
const reader = proc.stdout.getReader();
|
|
44
|
+
try {
|
|
45
|
+
const credential = await boundedCredentialStream(reader);
|
|
46
|
+
if ((await proc.exited) !== 0)
|
|
47
|
+
throw new Error("Claude keychain credential is unavailable");
|
|
48
|
+
return credential;
|
|
49
|
+
} catch (error) {
|
|
50
|
+
proc.kill("SIGKILL");
|
|
51
|
+
await proc.exited;
|
|
52
|
+
throw new Error("Claude keychain credential is unavailable", {
|
|
53
|
+
cause: error,
|
|
54
|
+
});
|
|
55
|
+
} finally {
|
|
56
|
+
clearTimeout(timer);
|
|
57
|
+
reader.releaseLock();
|
|
58
|
+
}
|
|
59
|
+
}
|
|
@@ -18,12 +18,7 @@ function goalAttachment(event: RecordValue) {
|
|
|
18
18
|
if (event.isSidechain === true || event.type !== "attachment") return [];
|
|
19
19
|
const attachment = record(event.attachment);
|
|
20
20
|
if (attachment.type !== "goal_status") return [];
|
|
21
|
-
const characters =
|
|
22
|
-
typeof attachment.condition === "string"
|
|
23
|
-
? [...attachment.condition].length
|
|
24
|
-
: 0;
|
|
25
|
-
if (!characters || characters > 4000 || typeof attachment.met !== "boolean")
|
|
26
|
-
throw new Error("Malformed native goal attachment");
|
|
21
|
+
const characters = goalCharacters(attachment);
|
|
27
22
|
return [
|
|
28
23
|
{
|
|
29
24
|
status: attachmentStatus(attachment),
|
|
@@ -33,6 +28,16 @@ function goalAttachment(event: RecordValue) {
|
|
|
33
28
|
];
|
|
34
29
|
}
|
|
35
30
|
|
|
31
|
+
function goalCharacters(attachment: RecordValue): number {
|
|
32
|
+
const characters =
|
|
33
|
+
typeof attachment.condition === "string"
|
|
34
|
+
? Array.from(attachment.condition).length
|
|
35
|
+
: 0;
|
|
36
|
+
if (!characters || characters > 4000 || typeof attachment.met !== "boolean")
|
|
37
|
+
throw new Error("Malformed native goal attachment");
|
|
38
|
+
return characters;
|
|
39
|
+
}
|
|
40
|
+
|
|
36
41
|
function attachmentStatus(attachment: RecordValue): string {
|
|
37
42
|
if (attachment.failed === true) return "blocked";
|
|
38
43
|
if (!attachment.met) return "active";
|
|
@@ -59,12 +64,7 @@ export function claudeNativeGoalEvidence(
|
|
|
59
64
|
sessionId: string,
|
|
60
65
|
) {
|
|
61
66
|
const events = records(transcript);
|
|
62
|
-
|
|
63
|
-
if (event.sessionId && event.sessionId !== sessionId)
|
|
64
|
-
throw new Error("Foreign native session record");
|
|
65
|
-
}
|
|
66
|
-
if (!events.some((event) => event.sessionId === sessionId))
|
|
67
|
-
throw new Error("Original native session not retained");
|
|
67
|
+
requireBoundSession(events, sessionId);
|
|
68
68
|
requireSuccessfulResult(stream, sessionId);
|
|
69
69
|
const goals = events.flatMap(goalAttachment);
|
|
70
70
|
return {
|
|
@@ -75,6 +75,15 @@ export function claudeNativeGoalEvidence(
|
|
|
75
75
|
};
|
|
76
76
|
}
|
|
77
77
|
|
|
78
|
+
function requireBoundSession(events: RecordValue[], sessionId: string): void {
|
|
79
|
+
for (const event of events) {
|
|
80
|
+
if (event.sessionId && event.sessionId !== sessionId)
|
|
81
|
+
throw new Error("Foreign native session record");
|
|
82
|
+
}
|
|
83
|
+
if (!events.some((event) => event.sessionId === sessionId))
|
|
84
|
+
throw new Error("Original native session not retained");
|
|
85
|
+
}
|
|
86
|
+
|
|
78
87
|
async function sessionPaths(
|
|
79
88
|
path: string,
|
|
80
89
|
sessionId: string,
|
|
@@ -88,7 +97,7 @@ async function sessionPaths(
|
|
|
88
97
|
throw new Error("Native transcript tree contains a symlink");
|
|
89
98
|
return (
|
|
90
99
|
await Promise.all(
|
|
91
|
-
entries.map((entry) =>
|
|
100
|
+
entries.map(async (entry) =>
|
|
92
101
|
entry.isDirectory()
|
|
93
102
|
? sessionPaths(join(path, entry.name), sessionId, bound)
|
|
94
103
|
: basename(entry.name) === `${sessionId}.jsonl`
|
|
@@ -109,26 +118,8 @@ export async function observeClaudeNativeGoal(
|
|
|
109
118
|
data: Record<string, unknown>;
|
|
110
119
|
}> {
|
|
111
120
|
try {
|
|
112
|
-
const
|
|
113
|
-
|
|
114
|
-
);
|
|
115
|
-
const ids = new Set(initializations.map((event) => event.session_id));
|
|
116
|
-
const id = [...ids][0];
|
|
117
|
-
if (
|
|
118
|
-
ids.size !== 1 ||
|
|
119
|
-
typeof id !== "string" ||
|
|
120
|
-
!/^[a-f0-9-]{36}$/i.test(id)
|
|
121
|
-
)
|
|
122
|
-
throw new Error("Missing or ambiguous original Claude session");
|
|
123
|
-
const paths = await sessionPaths(join(configRoot, "projects"), id);
|
|
124
|
-
if (paths.length !== 1)
|
|
125
|
-
throw new Error("Missing or ambiguous native transcript");
|
|
126
|
-
const entry = await lstat(paths[0]!);
|
|
127
|
-
if (!entry.isFile() || entry.size > 8 * 1024 * 1024)
|
|
128
|
-
throw new Error("Native transcript is unsafe or oversized");
|
|
129
|
-
const bytes = await readFile(paths[0]!);
|
|
130
|
-
if (bytes.length > 8 * 1024 * 1024)
|
|
131
|
-
throw new Error("Native transcript exceeds limit");
|
|
121
|
+
const id = originalSessionId(stream);
|
|
122
|
+
const bytes = await nativeTranscript(configRoot, id);
|
|
132
123
|
return {
|
|
133
124
|
id: "sevro.host.native-goal",
|
|
134
125
|
completeness: "complete",
|
|
@@ -152,3 +143,36 @@ export async function observeClaudeNativeGoal(
|
|
|
152
143
|
};
|
|
153
144
|
}
|
|
154
145
|
}
|
|
146
|
+
|
|
147
|
+
function originalSessionId(stream: string): string {
|
|
148
|
+
const initializations = records(stream).filter(
|
|
149
|
+
(event) => event.type === "system" && event.subtype === "init",
|
|
150
|
+
);
|
|
151
|
+
const ids = new Set(initializations.map((event) => event.session_id));
|
|
152
|
+
const [id] = ids;
|
|
153
|
+
if (ids.size !== 1 || typeof id !== "string" || !/^[a-f0-9-]{36}$/i.test(id))
|
|
154
|
+
throw new Error("Missing or ambiguous original Claude session");
|
|
155
|
+
return id;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
async function nativeTranscript(
|
|
159
|
+
configRoot: string,
|
|
160
|
+
id: string,
|
|
161
|
+
): Promise<Buffer> {
|
|
162
|
+
const paths = await sessionPaths(join(configRoot, "projects"), id);
|
|
163
|
+
const path = uniqueTranscriptPath(paths);
|
|
164
|
+
const entry = await lstat(path);
|
|
165
|
+
if (!entry.isFile() || entry.size > 8 * 1024 * 1024)
|
|
166
|
+
throw new Error("Native transcript is unsafe or oversized");
|
|
167
|
+
const bytes = await readFile(path);
|
|
168
|
+
if (bytes.length > 8 * 1024 * 1024)
|
|
169
|
+
throw new Error("Native transcript exceeds limit");
|
|
170
|
+
return bytes;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function uniqueTranscriptPath(paths: string[]): string {
|
|
174
|
+
const [path] = paths;
|
|
175
|
+
if (paths.length !== 1 || path === undefined)
|
|
176
|
+
throw new Error("Missing or ambiguous native transcript");
|
|
177
|
+
return path;
|
|
178
|
+
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import {
|
|
2
|
+
awaitHostProcess,
|
|
3
|
+
MAX_EVENT_BYTES,
|
|
4
|
+
HostProcessInterruption,
|
|
5
|
+
} from "./process-lifecycle";
|
|
6
|
+
import {
|
|
7
|
+
captureBoundedText,
|
|
8
|
+
terminateOwnedProcessGroup,
|
|
9
|
+
} from "../process-lifecycle";
|
|
10
|
+
import {
|
|
11
|
+
HostProcessDiagnostics,
|
|
12
|
+
type HostDiagnostics,
|
|
13
|
+
} from "../host-diagnostics";
|
|
14
|
+
|
|
15
|
+
const MAX_STDERR_BYTES = 64 * 1024;
|
|
16
|
+
|
|
17
|
+
function stop(proc: Bun.Subprocess): void {
|
|
18
|
+
if (process.platform !== "win32") {
|
|
19
|
+
terminateOwnedProcessGroup(proc, { fallbackError: "propagate" });
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
try {
|
|
23
|
+
proc.kill("SIGKILL");
|
|
24
|
+
} catch {
|
|
25
|
+
proc.kill("SIGKILL");
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
interface ClaudeProcessOptions {
|
|
30
|
+
diagnostics?: HostDiagnostics;
|
|
31
|
+
argv: string[];
|
|
32
|
+
cwd: string;
|
|
33
|
+
env: Record<string, string>;
|
|
34
|
+
timeoutMs: number;
|
|
35
|
+
signal?: AbortSignal;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function requireClaudeProcess(options: ClaudeProcessOptions): void {
|
|
39
|
+
if (options.signal?.aborted) throw new Error("Claude run cancelled");
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function spawnClaudeProcess(
|
|
43
|
+
options: ClaudeProcessOptions,
|
|
44
|
+
capture: HostProcessDiagnostics,
|
|
45
|
+
) {
|
|
46
|
+
try {
|
|
47
|
+
return Bun.spawn(options.argv, {
|
|
48
|
+
cwd: options.cwd,
|
|
49
|
+
env: options.env,
|
|
50
|
+
detached: process.platform !== "win32",
|
|
51
|
+
stdin: "ignore",
|
|
52
|
+
stdout: "pipe",
|
|
53
|
+
stderr: "pipe",
|
|
54
|
+
});
|
|
55
|
+
} catch (cause) {
|
|
56
|
+
capture.recordUnstarted();
|
|
57
|
+
throw cause;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function claudeCompletion(
|
|
62
|
+
proc: ReturnType<typeof spawnClaudeProcess>,
|
|
63
|
+
capture: HostProcessDiagnostics,
|
|
64
|
+
) {
|
|
65
|
+
const stdout = captureBoundedText(proc.stdout, {
|
|
66
|
+
limit: MAX_EVENT_BYTES,
|
|
67
|
+
overflow: "reject",
|
|
68
|
+
exceeded: () => new Error("Claude process output exceeds its limit"),
|
|
69
|
+
observe: (bytes) => {
|
|
70
|
+
capture.stdout.add(bytes);
|
|
71
|
+
},
|
|
72
|
+
});
|
|
73
|
+
const stderr = captureBoundedText(proc.stderr, {
|
|
74
|
+
limit: MAX_STDERR_BYTES,
|
|
75
|
+
overflow: "reject",
|
|
76
|
+
exceeded: () => new Error("Claude process output exceeds its limit"),
|
|
77
|
+
observe: (bytes) => {
|
|
78
|
+
capture.stderr.add(bytes);
|
|
79
|
+
},
|
|
80
|
+
});
|
|
81
|
+
const completed = Promise.all([stdout, stderr, proc.exited]).then(
|
|
82
|
+
([out, err, code]) => ({ out, err, code }),
|
|
83
|
+
);
|
|
84
|
+
return { completed, drains: [stdout, stderr] };
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function claudeInterruptionReason(cause: unknown) {
|
|
88
|
+
return cause instanceof HostProcessInterruption ? cause.reason : "error";
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export async function runClaudeProcess(
|
|
92
|
+
options: ClaudeProcessOptions,
|
|
93
|
+
): Promise<{ code: number; out: string; err: string }> {
|
|
94
|
+
requireClaudeProcess(options);
|
|
95
|
+
const capture = new HostProcessDiagnostics(options.diagnostics);
|
|
96
|
+
const proc = spawnClaudeProcess(options, capture);
|
|
97
|
+
const { completed, drains } = claudeCompletion(proc, capture);
|
|
98
|
+
try {
|
|
99
|
+
const result = await awaitHostProcess(proc, completed, {
|
|
100
|
+
...options,
|
|
101
|
+
host: "Claude",
|
|
102
|
+
stop,
|
|
103
|
+
});
|
|
104
|
+
await capture.record(proc);
|
|
105
|
+
return result;
|
|
106
|
+
} catch (cause) {
|
|
107
|
+
await Promise.allSettled(drains);
|
|
108
|
+
await capture.record(proc, claudeInterruptionReason(cause));
|
|
109
|
+
throw cause;
|
|
110
|
+
}
|
|
111
|
+
}
|