@fusionkit/ensemble 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent.d.ts +21 -0
- package/dist/agent.js +186 -0
- package/dist/artifacts.d.ts +21 -0
- package/dist/artifacts.js +36 -0
- package/dist/claude-code.d.ts +25 -0
- package/dist/claude-code.js +398 -0
- package/dist/codex.d.ts +69 -0
- package/dist/codex.js +467 -0
- package/dist/command.d.ts +15 -0
- package/dist/command.js +82 -0
- package/dist/dashboard.d.ts +62 -0
- package/dist/dashboard.js +788 -0
- package/dist/external-executor.d.ts +56 -0
- package/dist/external-executor.js +288 -0
- package/dist/harness.d.ts +337 -0
- package/dist/harness.js +1 -0
- package/dist/index.d.ts +30 -0
- package/dist/index.js +15 -0
- package/dist/isolation.d.ts +25 -0
- package/dist/isolation.js +509 -0
- package/dist/judge.d.ts +77 -0
- package/dist/judge.js +16 -0
- package/dist/mock.d.ts +20 -0
- package/dist/mock.js +56 -0
- package/dist/run.d.ts +5 -0
- package/dist/run.js +520 -0
- package/dist/synthesis.d.ts +25 -0
- package/dist/synthesis.js +221 -0
- package/dist/test/codex.test.d.ts +1 -0
- package/dist/test/codex.test.js +237 -0
- package/dist/test/dashboard.test.d.ts +1 -0
- package/dist/test/dashboard.test.js +214 -0
- package/dist/test/ensemble.test.d.ts +1 -0
- package/dist/test/ensemble.test.js +780 -0
- package/dist/test/external-executor.test.d.ts +1 -0
- package/dist/test/external-executor.test.js +273 -0
- package/dist/test/isolation.test.d.ts +1 -0
- package/dist/test/isolation.test.js +359 -0
- package/dist/test/tool-executor.test.d.ts +1 -0
- package/dist/test/tool-executor.test.js +113 -0
- package/dist/test/unified.test.d.ts +1 -0
- package/dist/test/unified.test.js +150 -0
- package/dist/tool-executor.d.ts +14 -0
- package/dist/tool-executor.js +156 -0
- package/dist/trace.d.ts +8 -0
- package/dist/trace.js +7 -0
- package/dist/unified.d.ts +101 -0
- package/dist/unified.js +422 -0
- package/dist/worktree.d.ts +25 -0
- package/dist/worktree.js +75 -0
- package/package.json +35 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export { createCommandHarness } from "./command.js";
|
|
2
|
+
export type { CommandHarnessOptions } from "./command.js";
|
|
3
|
+
export { claudeCodeHarness, claudeCodeHarnessCredentialSkipReason, createClaudeCodeHarness } from "./claude-code.js";
|
|
4
|
+
export type { ClaudeCodeHarnessEnv, ClaudeCodeHarnessOptions } from "./claude-code.js";
|
|
5
|
+
export { codexConfigToml, codexHarnessCredentialSkipReason, codexHarness, createCodexHarness } from "./codex.js";
|
|
6
|
+
export type { CodexAmbientProvider, CodexApprovalPolicy, CodexConfigTomlInput, CodexExecInput, CodexExecResult, CodexExecRunner, CodexHarnessEnv, CodexHarnessOptions, CodexOpenAiCompatibleProvider, CodexProvider, CodexResponsesProvider, CodexSandboxMode } from "./codex.js";
|
|
7
|
+
export { createArtifactStore } from "./artifacts.js";
|
|
8
|
+
export type { ArtifactStore } from "./artifacts.js";
|
|
9
|
+
export { createHarnessCapabilityMatrix, harnessDashboard, runHarnessSmokeDashboard } from "./dashboard.js";
|
|
10
|
+
export type { HarnessAdapterReadiness, HarnessAvailability, HarnessCapabilityMatrix, HarnessCapabilityMatrixRow, HarnessCapabilityTarget, HarnessLiveSmokeTarget, HarnessSmokeDashboard, HarnessSmokeDashboardOptions, HarnessSmokeOutcome, HarnessSmokePurpose, HarnessSmokeRecord } from "./dashboard.js";
|
|
11
|
+
export { createMockJudgeSynthesizer } from "./judge.js";
|
|
12
|
+
export type { JudgeCandidateEvidence, JudgeInput, JudgePatch, JudgeRepairInput, JudgeSynthesizer, JudgeSynthesisOutput, JudgeVerificationInput, MockJudgeSynthesizerOptions, SynthesisFailureSummary, SynthesisRepairAttempt, SynthesisVerificationResult } from "./judge.js";
|
|
13
|
+
export { ensemble, runEnsemble } from "./run.js";
|
|
14
|
+
export { createFusionKitJudgeSynthesizer, runFusionPanels, runUnifiedHarnessE2E } from "./unified.js";
|
|
15
|
+
export type { CursorHarnessRunnerInput, CursorHarnessRunnerResult, FusionPanelOptions, UnifiedHarnessE2EOptions, UnifiedHarnessE2EResult, UnifiedHarnessKind, UnifiedHarnessMatrixResult } from "./unified.js";
|
|
16
|
+
export { ambientTraceId, emitTrace, getTraceEmitter, newSpanId, newTraceId, TRACE_CANDIDATE_HEADER, TRACE_ID_HEADER, TRACE_PARENT_SPAN_HEADER, TRACE_SPAN_HEADER, TraceEmitter } from "./trace.js";
|
|
17
|
+
export type { EmitInput, FusionTraceComponent, FusionTraceEvent, FusionTraceEventType } from "./trace.js";
|
|
18
|
+
export { runJudgeSynthesis } from "./synthesis.js";
|
|
19
|
+
export type { RunSynthesisInput, SynthesisResult } from "./synthesis.js";
|
|
20
|
+
export { createMockHarness } from "./mock.js";
|
|
21
|
+
export type { MockCandidateFixture, MockHarnessOptions } from "./mock.js";
|
|
22
|
+
export { createToolExecutor, registerDemoTools, sideEffectsForTool } from "./tool-executor.js";
|
|
23
|
+
export type { ToolExecutor, ToolImplementation } from "./tool-executor.js";
|
|
24
|
+
export { executeFusionKitToolBatch, FusionKitToolExecutorClient, FusionKitToolExecutorClientError, FusionKitToolExecutorError, startFusionKitToolExecutorServer } from "./external-executor.js";
|
|
25
|
+
export { createCliContainerDriver, runCandidateCommandWithIsolation, secretAbsenceMetadata, secretValueHash } from "./isolation.js";
|
|
26
|
+
export type { FusionKitToolExecutionBatch, FusionKitToolExecutionRequest, FusionKitToolExecutionResponse, FusionKitToolExecutionResult, FusionKitToolExecutorServer, FusionKitToolExecutorServerOptions } from "./external-executor.js";
|
|
27
|
+
export type { CandidateCommandIsolationInput, CandidateCommandIsolationResult } from "./isolation.js";
|
|
28
|
+
export { cleanupCandidateWorktree, cleanupWorktreePlan, createWorktreePlan, defaultOutputRoot, diffCandidateWorktree, sealCandidateWorktree } from "./worktree.js";
|
|
29
|
+
export type { CandidateWorktree, WorktreePlan } from "./worktree.js";
|
|
30
|
+
export type { EnsembleCandidateSummary, EnsembleDescriptor, EnsembleJudge, EnsembleModel, EnsemblePolicy, EnsembleRunResult, EnsembleRuntime, CandidateContainerDriver, CandidateContainerDriverInput, CandidateContainerDriverResult, CandidateHardeningMetadata, CandidateIsolationConfig, CandidateIsolationKind, CandidateIsolationMountPolicy, CandidateIsolationNetworkPolicy, CandidateIsolationSecretPolicy, HarnessAdapter, HarnessArtifact, HarnessCapabilities, HarnessCandidateOutput, HarnessCollectInput, HarnessPrepareInput, HarnessRunInput, HarnessToolRecord, ReviewEvidence, EnsembleRunSummary, VerificationProfile } from "./harness.js";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export { createCommandHarness } from "./command.js";
|
|
2
|
+
export { claudeCodeHarness, claudeCodeHarnessCredentialSkipReason, createClaudeCodeHarness } from "./claude-code.js";
|
|
3
|
+
export { codexConfigToml, codexHarnessCredentialSkipReason, codexHarness, createCodexHarness } from "./codex.js";
|
|
4
|
+
export { createArtifactStore } from "./artifacts.js";
|
|
5
|
+
export { createHarnessCapabilityMatrix, harnessDashboard, runHarnessSmokeDashboard } from "./dashboard.js";
|
|
6
|
+
export { createMockJudgeSynthesizer } from "./judge.js";
|
|
7
|
+
export { ensemble, runEnsemble } from "./run.js";
|
|
8
|
+
export { createFusionKitJudgeSynthesizer, runFusionPanels, runUnifiedHarnessE2E } from "./unified.js";
|
|
9
|
+
export { ambientTraceId, emitTrace, getTraceEmitter, newSpanId, newTraceId, TRACE_CANDIDATE_HEADER, TRACE_ID_HEADER, TRACE_PARENT_SPAN_HEADER, TRACE_SPAN_HEADER, TraceEmitter } from "./trace.js";
|
|
10
|
+
export { runJudgeSynthesis } from "./synthesis.js";
|
|
11
|
+
export { createMockHarness } from "./mock.js";
|
|
12
|
+
export { createToolExecutor, registerDemoTools, sideEffectsForTool } from "./tool-executor.js";
|
|
13
|
+
export { executeFusionKitToolBatch, FusionKitToolExecutorClient, FusionKitToolExecutorClientError, FusionKitToolExecutorError, startFusionKitToolExecutorServer } from "./external-executor.js";
|
|
14
|
+
export { createCliContainerDriver, runCandidateCommandWithIsolation, secretAbsenceMetadata, secretValueHash } from "./isolation.js";
|
|
15
|
+
export { cleanupCandidateWorktree, cleanupWorktreePlan, createWorktreePlan, defaultOutputRoot, diffCandidateWorktree, sealCandidateWorktree } from "./worktree.js";
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { CandidateContainerDriver, CandidateHardeningMetadata, CandidateIsolationConfig, CandidateIsolationSecretPolicy } from "./harness.js";
|
|
2
|
+
export type CandidateCommandIsolationInput = {
|
|
3
|
+
command: string;
|
|
4
|
+
cwd: string;
|
|
5
|
+
timeoutMs?: number;
|
|
6
|
+
isolation?: CandidateIsolationConfig;
|
|
7
|
+
env?: Record<string, string | undefined>;
|
|
8
|
+
};
|
|
9
|
+
export type CandidateCommandIsolationResult = {
|
|
10
|
+
stdout: string;
|
|
11
|
+
stderr: string;
|
|
12
|
+
exitCode: number;
|
|
13
|
+
timedOut: boolean;
|
|
14
|
+
hardening: CandidateHardeningMetadata;
|
|
15
|
+
};
|
|
16
|
+
export declare function runCandidateCommandWithIsolation(input: CandidateCommandIsolationInput): Promise<CandidateCommandIsolationResult>;
|
|
17
|
+
export declare function createCliContainerDriver(engine?: "docker" | "podman"): CandidateContainerDriver;
|
|
18
|
+
export declare function secretAbsenceMetadata(input: {
|
|
19
|
+
cwd: string;
|
|
20
|
+
transcript: string;
|
|
21
|
+
secretPolicy?: CandidateIsolationSecretPolicy;
|
|
22
|
+
ignoredDirs?: readonly string[];
|
|
23
|
+
knownSecretValues?: readonly string[];
|
|
24
|
+
}): CandidateHardeningMetadata["secret_absence"];
|
|
25
|
+
export declare function secretValueHash(value: string): string;
|
|
@@ -0,0 +1,509 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
|
|
3
|
+
import { join, relative } from "node:path";
|
|
4
|
+
import { promisify } from "node:util";
|
|
5
|
+
import { hashCanonicalSha256 } from "@fusionkit/protocol";
|
|
6
|
+
const execFileAsync = promisify(execFile);
|
|
7
|
+
const DEFAULT_CONTAINER_IMAGE = "node:22";
|
|
8
|
+
const DEFAULT_CONTAINER_ENGINE = "docker";
|
|
9
|
+
const DEFAULT_CONTAINER_WORKDIR = "/workspace";
|
|
10
|
+
const DEFAULT_MICROVM_PROVIDER = "vercel-sandbox";
|
|
11
|
+
const DEFAULT_MICROVM_RUNTIME = "node24";
|
|
12
|
+
const UNKNOWN_RUNTIME_DIGEST = "unknown";
|
|
13
|
+
const DEFAULT_IGNORED_DIRS = [".git", "node_modules", ".warrant"];
|
|
14
|
+
const DEFAULT_MAX_SCAN_BYTES = 256 * 1024;
|
|
15
|
+
export async function runCandidateCommandWithIsolation(input) {
|
|
16
|
+
const isolation = normalizeIsolation(input.isolation);
|
|
17
|
+
switch (isolation.kind) {
|
|
18
|
+
case "process":
|
|
19
|
+
return runProcessCommand(input, isolation);
|
|
20
|
+
case "container":
|
|
21
|
+
return runContainerCommand(input, isolation);
|
|
22
|
+
case "microvm":
|
|
23
|
+
return runMicrovmCommand(input, isolation);
|
|
24
|
+
default:
|
|
25
|
+
return assertNever(isolation);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
export function createCliContainerDriver(engine = DEFAULT_CONTAINER_ENGINE) {
|
|
29
|
+
return {
|
|
30
|
+
id: engine,
|
|
31
|
+
supportsNetworkPolicy: false,
|
|
32
|
+
async execute(input) {
|
|
33
|
+
const args = [
|
|
34
|
+
"run",
|
|
35
|
+
"--rm",
|
|
36
|
+
"-v",
|
|
37
|
+
`${input.cwd}:${input.workdir}:rw`,
|
|
38
|
+
"-w",
|
|
39
|
+
input.workdir
|
|
40
|
+
];
|
|
41
|
+
for (const cachePath of input.mountPolicy.readOnlyCachePaths) {
|
|
42
|
+
args.push("-v", `${cachePath}:${cachePath}:ro`);
|
|
43
|
+
}
|
|
44
|
+
if (input.networkPolicy.defaultDeny && input.networkPolicy.allowHosts.length === 0) {
|
|
45
|
+
args.push("--network", "none");
|
|
46
|
+
}
|
|
47
|
+
args.push(input.image, "sh", "-lc", input.command);
|
|
48
|
+
const result = await runHostCommand(engine, args, input.cwd, input.timeoutMs);
|
|
49
|
+
return {
|
|
50
|
+
stdout: result.stdout,
|
|
51
|
+
stderr: result.stderr,
|
|
52
|
+
exitCode: result.exitCode,
|
|
53
|
+
timedOut: result.timedOut,
|
|
54
|
+
cleanup: {
|
|
55
|
+
attempted: true,
|
|
56
|
+
succeeded: !result.timedOut,
|
|
57
|
+
...(result.timedOut
|
|
58
|
+
? { error: "container cleanup not confirmed after timeout" }
|
|
59
|
+
: {})
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
export function secretAbsenceMetadata(input) {
|
|
66
|
+
const secretPolicy = normalizeSecretPolicy(input.secretPolicy);
|
|
67
|
+
const scanScope = ["transcript"];
|
|
68
|
+
const haystacks = [input.transcript];
|
|
69
|
+
for (const file of listScannableFiles(input.cwd, input.ignoredDirs ?? DEFAULT_IGNORED_DIRS)) {
|
|
70
|
+
scanScope.push(`file:${file.relativePath}`);
|
|
71
|
+
haystacks.push(file.content);
|
|
72
|
+
}
|
|
73
|
+
const leaks = countLeaks(haystacks, [
|
|
74
|
+
...secretPolicy.secretNames,
|
|
75
|
+
...secretPolicy.secretValueHashes,
|
|
76
|
+
...(input.knownSecretValues ?? [])
|
|
77
|
+
]);
|
|
78
|
+
return {
|
|
79
|
+
secret_names: secretPolicy.secretNames,
|
|
80
|
+
secret_value_hashes: secretPolicy.secretValueHashes,
|
|
81
|
+
injected_env_names: secretPolicy.injectedEnvNames,
|
|
82
|
+
scanned: true,
|
|
83
|
+
leaks_found: leaks > 0,
|
|
84
|
+
scan_scope: scanScope,
|
|
85
|
+
leak_count: leaks
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
async function runProcessCommand(input, isolation) {
|
|
89
|
+
const command = await runHostCommand("/bin/sh", ["-lc", input.command], input.cwd, input.timeoutMs, input.env);
|
|
90
|
+
const transcript = [command.stdout, command.stderr].filter(Boolean).join("\n");
|
|
91
|
+
return {
|
|
92
|
+
...command,
|
|
93
|
+
hardening: hardeningMetadata({
|
|
94
|
+
requestedIsolation: isolation.kind,
|
|
95
|
+
actualIsolation: "process",
|
|
96
|
+
isolation,
|
|
97
|
+
secretAbsence: secretAbsenceMetadata({
|
|
98
|
+
cwd: input.cwd,
|
|
99
|
+
transcript,
|
|
100
|
+
secretPolicy: isolation.secretPolicy,
|
|
101
|
+
ignoredDirs: isolation.mountPolicy.ignoredDirs
|
|
102
|
+
}),
|
|
103
|
+
networkEnforced: false,
|
|
104
|
+
cleanup: {
|
|
105
|
+
attempted: false,
|
|
106
|
+
succeeded: true
|
|
107
|
+
}
|
|
108
|
+
})
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
async function runContainerCommand(input, isolation) {
|
|
112
|
+
const driver = isolation.driver ?? createCliContainerDriver(isolation.engine ?? DEFAULT_CONTAINER_ENGINE);
|
|
113
|
+
const image = isolation.image ?? DEFAULT_CONTAINER_IMAGE;
|
|
114
|
+
if (isolation.networkPolicy.enforce &&
|
|
115
|
+
isolation.networkPolicy.defaultDeny &&
|
|
116
|
+
isolation.networkPolicy.allowHosts.length > 0 &&
|
|
117
|
+
!driver.supportsNetworkPolicy) {
|
|
118
|
+
const stderr = "container driver cannot enforce host allowlist network policy";
|
|
119
|
+
const secretAbsence = secretAbsenceMetadata({
|
|
120
|
+
cwd: input.cwd,
|
|
121
|
+
transcript: stderr,
|
|
122
|
+
secretPolicy: isolation.secretPolicy,
|
|
123
|
+
ignoredDirs: isolation.mountPolicy.ignoredDirs
|
|
124
|
+
});
|
|
125
|
+
return {
|
|
126
|
+
stdout: "",
|
|
127
|
+
stderr,
|
|
128
|
+
exitCode: 1,
|
|
129
|
+
timedOut: false,
|
|
130
|
+
hardening: hardeningMetadata({
|
|
131
|
+
requestedIsolation: "container",
|
|
132
|
+
actualIsolation: "container",
|
|
133
|
+
isolation,
|
|
134
|
+
driverId: driver.id,
|
|
135
|
+
image,
|
|
136
|
+
secretAbsence,
|
|
137
|
+
networkEnforced: true,
|
|
138
|
+
cleanup: {
|
|
139
|
+
attempted: true,
|
|
140
|
+
succeeded: false,
|
|
141
|
+
error: "network policy unsupported"
|
|
142
|
+
}
|
|
143
|
+
})
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
let result;
|
|
147
|
+
try {
|
|
148
|
+
result = await driver.execute({
|
|
149
|
+
command: input.command,
|
|
150
|
+
cwd: input.cwd,
|
|
151
|
+
timeoutMs: input.timeoutMs,
|
|
152
|
+
image,
|
|
153
|
+
workdir: isolation.mountPolicy.workdir,
|
|
154
|
+
mountPolicy: isolation.mountPolicy,
|
|
155
|
+
networkPolicy: isolation.networkPolicy
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
160
|
+
result = {
|
|
161
|
+
stdout: "",
|
|
162
|
+
stderr: message,
|
|
163
|
+
exitCode: 1,
|
|
164
|
+
timedOut: false,
|
|
165
|
+
cleanup: {
|
|
166
|
+
attempted: true,
|
|
167
|
+
succeeded: false,
|
|
168
|
+
error: message
|
|
169
|
+
}
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
const transcript = [result.stdout, result.stderr].filter(Boolean).join("\n");
|
|
173
|
+
return {
|
|
174
|
+
stdout: result.stdout,
|
|
175
|
+
stderr: result.stderr,
|
|
176
|
+
exitCode: result.exitCode,
|
|
177
|
+
timedOut: result.timedOut ?? false,
|
|
178
|
+
hardening: hardeningMetadata({
|
|
179
|
+
requestedIsolation: "container",
|
|
180
|
+
actualIsolation: "container",
|
|
181
|
+
isolation,
|
|
182
|
+
driverId: driver.id,
|
|
183
|
+
image,
|
|
184
|
+
secretAbsence: secretAbsenceMetadata({
|
|
185
|
+
cwd: input.cwd,
|
|
186
|
+
transcript,
|
|
187
|
+
secretPolicy: isolation.secretPolicy,
|
|
188
|
+
ignoredDirs: isolation.mountPolicy.ignoredDirs
|
|
189
|
+
}),
|
|
190
|
+
networkEnforced: isolation.networkPolicy.enforce &&
|
|
191
|
+
(driver.supportsNetworkPolicy ||
|
|
192
|
+
(isolation.networkPolicy.defaultDeny &&
|
|
193
|
+
isolation.networkPolicy.allowHosts.length === 0)),
|
|
194
|
+
cleanup: result.cleanup ?? {
|
|
195
|
+
attempted: true,
|
|
196
|
+
succeeded: true
|
|
197
|
+
}
|
|
198
|
+
})
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
async function runMicrovmCommand(input, isolation) {
|
|
202
|
+
const driver = isolation.driver;
|
|
203
|
+
if (driver === undefined) {
|
|
204
|
+
const stderr = "microVM isolation requires an execution driver";
|
|
205
|
+
return {
|
|
206
|
+
stdout: "",
|
|
207
|
+
stderr,
|
|
208
|
+
exitCode: 1,
|
|
209
|
+
timedOut: false,
|
|
210
|
+
hardening: hardeningMetadata({
|
|
211
|
+
requestedIsolation: "microvm",
|
|
212
|
+
actualIsolation: "process",
|
|
213
|
+
isolation,
|
|
214
|
+
runtimeFields: microvmRuntimeFields(isolation),
|
|
215
|
+
secretAbsence: secretAbsenceMetadata({
|
|
216
|
+
cwd: input.cwd,
|
|
217
|
+
transcript: stderr,
|
|
218
|
+
secretPolicy: isolation.secretPolicy,
|
|
219
|
+
ignoredDirs: isolation.mountPolicy.ignoredDirs
|
|
220
|
+
}),
|
|
221
|
+
networkEnforced: false,
|
|
222
|
+
cleanup: {
|
|
223
|
+
attempted: false,
|
|
224
|
+
succeeded: true
|
|
225
|
+
}
|
|
226
|
+
})
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
if (isolation.networkPolicy.enforce &&
|
|
230
|
+
isolation.networkPolicy.defaultDeny &&
|
|
231
|
+
isolation.networkPolicy.allowHosts.length > 0 &&
|
|
232
|
+
!driver.supportsNetworkPolicy) {
|
|
233
|
+
const stderr = "microVM driver cannot enforce host allowlist network policy";
|
|
234
|
+
return {
|
|
235
|
+
stdout: "",
|
|
236
|
+
stderr,
|
|
237
|
+
exitCode: 1,
|
|
238
|
+
timedOut: false,
|
|
239
|
+
hardening: hardeningMetadata({
|
|
240
|
+
requestedIsolation: "microvm",
|
|
241
|
+
actualIsolation: actualMicrovmIsolation(driver.provider),
|
|
242
|
+
isolation,
|
|
243
|
+
driverId: driver.id,
|
|
244
|
+
runtimeFields: microvmRuntimeFields(isolation, driver),
|
|
245
|
+
secretAbsence: secretAbsenceMetadata({
|
|
246
|
+
cwd: input.cwd,
|
|
247
|
+
transcript: stderr,
|
|
248
|
+
secretPolicy: isolation.secretPolicy,
|
|
249
|
+
ignoredDirs: isolation.mountPolicy.ignoredDirs
|
|
250
|
+
}),
|
|
251
|
+
networkEnforced: true,
|
|
252
|
+
cleanup: {
|
|
253
|
+
attempted: true,
|
|
254
|
+
succeeded: false,
|
|
255
|
+
error: "network policy unsupported"
|
|
256
|
+
}
|
|
257
|
+
})
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
let result;
|
|
261
|
+
try {
|
|
262
|
+
result = await driver.execute({
|
|
263
|
+
command: input.command,
|
|
264
|
+
cwd: input.cwd,
|
|
265
|
+
timeoutMs: input.timeoutMs,
|
|
266
|
+
provider: isolation.provider,
|
|
267
|
+
runtime: isolation.runtime,
|
|
268
|
+
snapshotId: isolation.snapshotId,
|
|
269
|
+
workdir: isolation.mountPolicy.workdir,
|
|
270
|
+
mountPolicy: isolation.mountPolicy,
|
|
271
|
+
networkPolicy: isolation.networkPolicy,
|
|
272
|
+
secretPolicy: isolation.secretPolicy
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
catch (error) {
|
|
276
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
277
|
+
result = {
|
|
278
|
+
stdout: "",
|
|
279
|
+
stderr: message,
|
|
280
|
+
exitCode: 1,
|
|
281
|
+
timedOut: false,
|
|
282
|
+
actualIsolation: actualMicrovmIsolation(driver.provider),
|
|
283
|
+
cleanup: {
|
|
284
|
+
attempted: true,
|
|
285
|
+
succeeded: false,
|
|
286
|
+
error: message
|
|
287
|
+
}
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
const transcript = [result.stdout, result.stderr].filter(Boolean).join("\n");
|
|
291
|
+
return {
|
|
292
|
+
stdout: result.stdout,
|
|
293
|
+
stderr: result.stderr,
|
|
294
|
+
exitCode: result.exitCode,
|
|
295
|
+
timedOut: result.timedOut ?? false,
|
|
296
|
+
hardening: hardeningMetadata({
|
|
297
|
+
requestedIsolation: "microvm",
|
|
298
|
+
actualIsolation: result.actualIsolation ?? actualMicrovmIsolation(driver.provider),
|
|
299
|
+
isolation,
|
|
300
|
+
driverId: driver.id,
|
|
301
|
+
runtimeFields: microvmRuntimeFields(isolation, driver, result.runtime),
|
|
302
|
+
secretAbsence: secretAbsenceMetadata({
|
|
303
|
+
cwd: input.cwd,
|
|
304
|
+
transcript,
|
|
305
|
+
secretPolicy: isolation.secretPolicy,
|
|
306
|
+
ignoredDirs: isolation.mountPolicy.ignoredDirs
|
|
307
|
+
}),
|
|
308
|
+
networkEnforced: isolation.networkPolicy.enforce && driver.supportsNetworkPolicy,
|
|
309
|
+
cleanup: result.cleanup ?? {
|
|
310
|
+
attempted: true,
|
|
311
|
+
succeeded: true
|
|
312
|
+
}
|
|
313
|
+
})
|
|
314
|
+
};
|
|
315
|
+
}
|
|
316
|
+
async function runHostCommand(command, args, cwd, timeoutMs, env) {
|
|
317
|
+
const mergedEnv = env === undefined
|
|
318
|
+
? undefined
|
|
319
|
+
: {
|
|
320
|
+
...process.env,
|
|
321
|
+
...Object.fromEntries(Object.entries(env).filter((entry) => entry[1] !== undefined))
|
|
322
|
+
};
|
|
323
|
+
try {
|
|
324
|
+
const result = await execFileAsync(command, args, {
|
|
325
|
+
cwd,
|
|
326
|
+
...(mergedEnv !== undefined ? { env: mergedEnv } : {}),
|
|
327
|
+
...(timeoutMs !== undefined ? { timeout: timeoutMs } : {})
|
|
328
|
+
});
|
|
329
|
+
return {
|
|
330
|
+
stdout: result.stdout,
|
|
331
|
+
stderr: result.stderr,
|
|
332
|
+
exitCode: 0,
|
|
333
|
+
timedOut: false
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
catch (error) {
|
|
337
|
+
const failed = error;
|
|
338
|
+
const timedOut = failed.killed === true || failed.signal === "SIGTERM";
|
|
339
|
+
return {
|
|
340
|
+
stdout: failed.stdout ?? "",
|
|
341
|
+
stderr: failed.stderr ?? failed.message ?? "",
|
|
342
|
+
exitCode: typeof failed.code === "number" ? failed.code : 1,
|
|
343
|
+
timedOut
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
function normalizeIsolation(isolation) {
|
|
348
|
+
const kind = isolation?.kind ?? "process";
|
|
349
|
+
const mountPolicy = normalizeMountPolicy(isolation?.mountPolicy);
|
|
350
|
+
const networkPolicy = normalizeNetworkPolicy(isolation?.networkPolicy);
|
|
351
|
+
const secretPolicy = normalizeSecretPolicy(isolation?.secretPolicy);
|
|
352
|
+
if (isolation?.kind === "container") {
|
|
353
|
+
return {
|
|
354
|
+
kind: "container",
|
|
355
|
+
image: isolation?.image ?? DEFAULT_CONTAINER_IMAGE,
|
|
356
|
+
engine: isolation?.engine ?? DEFAULT_CONTAINER_ENGINE,
|
|
357
|
+
driver: isolation?.driver,
|
|
358
|
+
mountPolicy,
|
|
359
|
+
networkPolicy,
|
|
360
|
+
secretPolicy
|
|
361
|
+
};
|
|
362
|
+
}
|
|
363
|
+
if (isolation?.kind === "microvm") {
|
|
364
|
+
return {
|
|
365
|
+
kind: "microvm",
|
|
366
|
+
provider: isolation.provider ?? DEFAULT_MICROVM_PROVIDER,
|
|
367
|
+
runtime: isolation.runtime ?? DEFAULT_MICROVM_RUNTIME,
|
|
368
|
+
snapshotId: isolation.snapshotId,
|
|
369
|
+
sandboxId: isolation.sandboxId,
|
|
370
|
+
imageDigest: isolation.imageDigest,
|
|
371
|
+
runtimeDigest: isolation.runtimeDigest ?? UNKNOWN_RUNTIME_DIGEST,
|
|
372
|
+
driver: isolation.driver,
|
|
373
|
+
mountPolicy,
|
|
374
|
+
networkPolicy,
|
|
375
|
+
secretPolicy
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
return {
|
|
379
|
+
kind: "process",
|
|
380
|
+
mountPolicy,
|
|
381
|
+
networkPolicy,
|
|
382
|
+
secretPolicy
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
function normalizeMountPolicy(policy) {
|
|
386
|
+
return {
|
|
387
|
+
workdir: policy?.workdir ?? DEFAULT_CONTAINER_WORKDIR,
|
|
388
|
+
worktreeWritable: policy?.worktreeWritable ?? true,
|
|
389
|
+
readOnlyCachePaths: [...(policy?.readOnlyCachePaths ?? [])],
|
|
390
|
+
ignoredDirs: [...(policy?.ignoredDirs ?? DEFAULT_IGNORED_DIRS)]
|
|
391
|
+
};
|
|
392
|
+
}
|
|
393
|
+
function normalizeNetworkPolicy(policy) {
|
|
394
|
+
return {
|
|
395
|
+
defaultDeny: policy?.defaultDeny ?? true,
|
|
396
|
+
allowHosts: [...(policy?.allowHosts ?? [])],
|
|
397
|
+
enforce: policy?.enforce ?? true
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
function normalizeSecretPolicy(policy) {
|
|
401
|
+
return {
|
|
402
|
+
secretNames: [...(policy?.secretNames ?? [])],
|
|
403
|
+
secretValueHashes: [...(policy?.secretValueHashes ?? [])],
|
|
404
|
+
injectedEnvNames: [...(policy?.injectedEnvNames ?? [])]
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
function hardeningMetadata(input) {
|
|
408
|
+
return {
|
|
409
|
+
requested_isolation: input.requestedIsolation,
|
|
410
|
+
actual_isolation: input.actualIsolation,
|
|
411
|
+
runtime: {
|
|
412
|
+
...(input.runtimeFields ?? {}),
|
|
413
|
+
...(input.image !== undefined ? { image: input.image } : {}),
|
|
414
|
+
...(input.driverId !== undefined ? { driver: input.driverId } : {}),
|
|
415
|
+
workdir: input.isolation.mountPolicy.workdir
|
|
416
|
+
},
|
|
417
|
+
mount_policy: {
|
|
418
|
+
worktree_writable: input.isolation.mountPolicy.worktreeWritable,
|
|
419
|
+
read_only_caches: input.isolation.mountPolicy.readOnlyCachePaths,
|
|
420
|
+
ignored_dirs: input.isolation.mountPolicy.ignoredDirs
|
|
421
|
+
},
|
|
422
|
+
network_policy: {
|
|
423
|
+
default_deny: input.isolation.networkPolicy.defaultDeny,
|
|
424
|
+
allow_hosts: input.isolation.networkPolicy.allowHosts,
|
|
425
|
+
enforced: input.networkEnforced
|
|
426
|
+
},
|
|
427
|
+
cleanup: {
|
|
428
|
+
attempted: input.cleanup.attempted,
|
|
429
|
+
succeeded: input.cleanup.succeeded,
|
|
430
|
+
status: cleanupStatus(input.cleanup),
|
|
431
|
+
...(input.cleanup.timedOut === true ? { timed_out: true } : {}),
|
|
432
|
+
...(input.cleanup.error !== undefined ? { error: input.cleanup.error } : {})
|
|
433
|
+
},
|
|
434
|
+
secret_absence: input.secretAbsence
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
function cleanupStatus(input) {
|
|
438
|
+
if (!input.attempted)
|
|
439
|
+
return "not_required";
|
|
440
|
+
if (input.succeeded)
|
|
441
|
+
return "succeeded";
|
|
442
|
+
if (input.timedOut === true)
|
|
443
|
+
return "timed_out";
|
|
444
|
+
return "failed";
|
|
445
|
+
}
|
|
446
|
+
function actualMicrovmIsolation(provider) {
|
|
447
|
+
return provider === "vercel-sandbox" ? "vercel-sandbox" : "microvm";
|
|
448
|
+
}
|
|
449
|
+
function microvmRuntimeFields(isolation, driver, runtime) {
|
|
450
|
+
return {
|
|
451
|
+
provider: runtime?.provider ?? driver?.provider ?? isolation.provider,
|
|
452
|
+
runtime: runtime?.runtime ?? isolation.runtime,
|
|
453
|
+
...(runtime?.snapshotId ?? isolation.snapshotId
|
|
454
|
+
? { snapshot_id: runtime?.snapshotId ?? isolation.snapshotId }
|
|
455
|
+
: {}),
|
|
456
|
+
...(runtime?.sandboxId ?? isolation.sandboxId
|
|
457
|
+
? { sandbox_id: runtime?.sandboxId ?? isolation.sandboxId }
|
|
458
|
+
: {}),
|
|
459
|
+
...(runtime?.imageDigest ?? isolation.imageDigest
|
|
460
|
+
? { image_digest: runtime?.imageDigest ?? isolation.imageDigest }
|
|
461
|
+
: {}),
|
|
462
|
+
runtime_digest: runtime?.runtimeDigest ?? isolation.runtimeDigest ?? UNKNOWN_RUNTIME_DIGEST
|
|
463
|
+
};
|
|
464
|
+
}
|
|
465
|
+
function assertNever(value) {
|
|
466
|
+
throw new Error(`unsupported candidate isolation kind: ${String(value)}`);
|
|
467
|
+
}
|
|
468
|
+
function listScannableFiles(root, ignoredDirs) {
|
|
469
|
+
if (!existsSync(root))
|
|
470
|
+
return [];
|
|
471
|
+
const ignored = new Set(ignoredDirs);
|
|
472
|
+
const files = [];
|
|
473
|
+
const walk = (dir) => {
|
|
474
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
475
|
+
if (ignored.has(entry.name))
|
|
476
|
+
continue;
|
|
477
|
+
const path = join(dir, entry.name);
|
|
478
|
+
if (entry.isDirectory()) {
|
|
479
|
+
walk(path);
|
|
480
|
+
continue;
|
|
481
|
+
}
|
|
482
|
+
if (!entry.isFile())
|
|
483
|
+
continue;
|
|
484
|
+
const size = statSync(path).size;
|
|
485
|
+
if (size > DEFAULT_MAX_SCAN_BYTES)
|
|
486
|
+
continue;
|
|
487
|
+
files.push({
|
|
488
|
+
relativePath: relative(root, path),
|
|
489
|
+
content: readFileSync(path, "utf8")
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
};
|
|
493
|
+
walk(root);
|
|
494
|
+
return files;
|
|
495
|
+
}
|
|
496
|
+
function countLeaks(haystacks, needles) {
|
|
497
|
+
const activeNeedles = needles.filter((needle) => needle.length > 0);
|
|
498
|
+
let count = 0;
|
|
499
|
+
for (const haystack of haystacks) {
|
|
500
|
+
for (const needle of activeNeedles) {
|
|
501
|
+
if (haystack.includes(needle))
|
|
502
|
+
count += 1;
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
return count;
|
|
506
|
+
}
|
|
507
|
+
export function secretValueHash(value) {
|
|
508
|
+
return hashCanonicalSha256(value);
|
|
509
|
+
}
|
package/dist/judge.d.ts
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import type { JudgeSynthesisDecision, ModelCallRecordV1, ModelFusionStatus } from "@fusionkit/protocol";
|
|
2
|
+
import type { EnsembleDescriptor, HarnessArtifact, HarnessCandidateOutput, HarnessToolRecord, HarnessTrajectory, ReviewEvidence } from "./harness.js";
|
|
3
|
+
export type JudgeCandidateEvidence = {
|
|
4
|
+
candidateId: string;
|
|
5
|
+
modelId: string;
|
|
6
|
+
model: string;
|
|
7
|
+
status: ModelFusionStatus;
|
|
8
|
+
artifacts: readonly HarnessArtifact[];
|
|
9
|
+
verification?: HarnessCandidateOutput["verification"];
|
|
10
|
+
trajectory?: HarnessTrajectory;
|
|
11
|
+
};
|
|
12
|
+
export type JudgeInput = {
|
|
13
|
+
descriptor: EnsembleDescriptor;
|
|
14
|
+
candidates: readonly JudgeCandidateEvidence[];
|
|
15
|
+
artifacts: readonly HarnessArtifact[];
|
|
16
|
+
toolRecords: readonly HarnessToolRecord[];
|
|
17
|
+
modelCallRecords: readonly ModelCallRecordV1[];
|
|
18
|
+
reviewEvidence?: ReviewEvidence;
|
|
19
|
+
};
|
|
20
|
+
export type JudgePatch = {
|
|
21
|
+
content: string;
|
|
22
|
+
sourceCandidateIds?: string[];
|
|
23
|
+
author?: "judge" | "candidate";
|
|
24
|
+
};
|
|
25
|
+
export type JudgeSynthesisOutput = {
|
|
26
|
+
decision: JudgeSynthesisDecision;
|
|
27
|
+
finalOutput: string;
|
|
28
|
+
rationale?: string;
|
|
29
|
+
selectedCandidateId?: string;
|
|
30
|
+
judgeModelCallId?: string;
|
|
31
|
+
score?: number;
|
|
32
|
+
patch?: JudgePatch;
|
|
33
|
+
contributions?: Array<{
|
|
34
|
+
candidateId: string;
|
|
35
|
+
reason: string;
|
|
36
|
+
}>;
|
|
37
|
+
rejections?: Array<{
|
|
38
|
+
candidateId: string;
|
|
39
|
+
reason: string;
|
|
40
|
+
}>;
|
|
41
|
+
};
|
|
42
|
+
export type JudgeRepairInput = JudgeInput & {
|
|
43
|
+
failureEvidence: SynthesisVerificationResult;
|
|
44
|
+
priorOutput: JudgeSynthesisOutput;
|
|
45
|
+
};
|
|
46
|
+
export type SynthesisVerificationResult = {
|
|
47
|
+
status: ModelFusionStatus;
|
|
48
|
+
evidence: string[];
|
|
49
|
+
exitCode?: number;
|
|
50
|
+
};
|
|
51
|
+
export type SynthesisFailureSummary = {
|
|
52
|
+
reason: string;
|
|
53
|
+
verification?: SynthesisVerificationResult;
|
|
54
|
+
repair?: SynthesisVerificationResult;
|
|
55
|
+
};
|
|
56
|
+
export type SynthesisRepairAttempt = {
|
|
57
|
+
round: number;
|
|
58
|
+
verification: SynthesisVerificationResult;
|
|
59
|
+
status: ModelFusionStatus;
|
|
60
|
+
};
|
|
61
|
+
export type JudgeVerificationInput = {
|
|
62
|
+
descriptor: EnsembleDescriptor;
|
|
63
|
+
worktreePath: string;
|
|
64
|
+
output: JudgeSynthesisOutput;
|
|
65
|
+
repairRound: number;
|
|
66
|
+
};
|
|
67
|
+
export type JudgeSynthesizer = {
|
|
68
|
+
synthesize(input: JudgeInput): Promise<JudgeSynthesisOutput> | JudgeSynthesisOutput;
|
|
69
|
+
repair?(input: JudgeRepairInput): Promise<JudgeSynthesisOutput> | JudgeSynthesisOutput;
|
|
70
|
+
verify?(input: JudgeVerificationInput): Promise<SynthesisVerificationResult> | SynthesisVerificationResult;
|
|
71
|
+
};
|
|
72
|
+
export type MockJudgeSynthesizerOptions = {
|
|
73
|
+
output: JudgeSynthesisOutput;
|
|
74
|
+
repairOutput?: JudgeSynthesisOutput;
|
|
75
|
+
verificationResults?: SynthesisVerificationResult[];
|
|
76
|
+
};
|
|
77
|
+
export declare function createMockJudgeSynthesizer(options: MockJudgeSynthesizerOptions): JudgeSynthesizer;
|
package/dist/judge.js
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export function createMockJudgeSynthesizer(options) {
|
|
2
|
+
let verificationIndex = 0;
|
|
3
|
+
return {
|
|
4
|
+
synthesize: () => options.output,
|
|
5
|
+
repair: options.repairOutput ? () => options.repairOutput : undefined,
|
|
6
|
+
verify: () => {
|
|
7
|
+
const result = options.verificationResults?.[verificationIndex];
|
|
8
|
+
verificationIndex += 1;
|
|
9
|
+
return (result ?? {
|
|
10
|
+
status: "succeeded",
|
|
11
|
+
evidence: ["mock judge verification passed"],
|
|
12
|
+
exitCode: 0
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
};
|
|
16
|
+
}
|