@fusionkit/cli 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -2
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +1 -0
- package/dist/commands/ensemble-gateway.js +0 -2
- package/dist/commands/ensemble-records.d.ts +2 -1
- package/dist/commands/ensemble-records.js +3 -1
- package/dist/commands/ensemble.js +3 -4
- package/dist/commands/fusion.js +0 -5
- package/dist/commands/local.js +3 -3
- package/dist/cursor-acp.d.ts +3 -5
- package/dist/cursor-acp.js +12 -11
- package/dist/dashboard.d.ts +65 -0
- package/dist/dashboard.js +587 -0
- package/dist/fusion/env.d.ts +108 -0
- package/dist/fusion/env.js +98 -0
- package/dist/fusion/observability.d.ts +39 -0
- package/dist/fusion/observability.js +227 -0
- package/dist/fusion/preflight.d.ts +12 -0
- package/dist/fusion/preflight.js +42 -0
- package/dist/fusion/stack.d.ts +62 -0
- package/dist/fusion/stack.js +295 -0
- package/dist/fusion-config.d.ts +0 -1
- package/dist/fusion-config.js +0 -6
- package/dist/fusion-init.js +2 -11
- package/dist/fusion-quickstart.d.ts +11 -222
- package/dist/fusion-quickstart.js +57 -759
- package/dist/gateway.d.ts +0 -2
- package/dist/gateway.js +0 -2
- package/dist/local.d.ts +10 -17
- package/dist/local.js +50 -116
- package/dist/shared/options.d.ts +2 -1
- package/dist/shared/options.js +13 -19
- package/dist/shared/proc.d.ts +4 -70
- package/dist/shared/proc.js +3 -228
- package/dist/test/cli.test.js +8 -3
- package/dist/test/dashboard.test.d.ts +1 -0
- package/dist/test/dashboard.test.js +214 -0
- package/dist/test/gateway-e2e.test.js +13 -10
- package/dist/test/local.test.js +4 -4
- package/dist/tools.d.ts +2 -0
- package/dist/tools.js +25 -0
- package/package.json +14 -9
- package/scope/.next/BUILD_ID +1 -1
- package/scope/.next/app-build-manifest.json +16 -16
- package/scope/.next/app-path-routes-manifest.json +4 -4
- package/scope/.next/build-manifest.json +2 -2
- package/scope/.next/prerender-manifest.json +9 -9
- package/scope/.next/server/app/_not-found/page_client-reference-manifest.js +1 -1
- package/scope/.next/server/app/_not-found.html +1 -1
- package/scope/.next/server/app/_not-found.rsc +1 -1
- package/scope/.next/server/app/api/environments/route_client-reference-manifest.js +1 -1
- package/scope/.next/server/app/api/ingest/route_client-reference-manifest.js +1 -1
- package/scope/.next/server/app/api/models/route_client-reference-manifest.js +1 -1
- package/scope/.next/server/app/api/replay/route_client-reference-manifest.js +1 -1
- package/scope/.next/server/app/api/sessions/[traceId]/route_client-reference-manifest.js +1 -1
- package/scope/.next/server/app/api/sessions/route_client-reference-manifest.js +1 -1
- package/scope/.next/server/app/api/stream/route_client-reference-manifest.js +1 -1
- package/scope/.next/server/app/environments/page_client-reference-manifest.js +1 -1
- package/scope/.next/server/app/environments.html +1 -1
- package/scope/.next/server/app/environments.rsc +1 -1
- package/scope/.next/server/app/index.html +1 -1
- package/scope/.next/server/app/index.rsc +1 -1
- package/scope/.next/server/app/models/page_client-reference-manifest.js +1 -1
- package/scope/.next/server/app/models.html +1 -1
- package/scope/.next/server/app/models.rsc +1 -1
- package/scope/.next/server/app/page_client-reference-manifest.js +1 -1
- package/scope/.next/server/app/sessions/[traceId]/page_client-reference-manifest.js +1 -1
- package/scope/.next/server/app-paths-manifest.json +4 -4
- package/scope/.next/server/functions-config-manifest.json +2 -2
- package/scope/.next/server/pages/404.html +1 -1
- package/scope/.next/server/pages/500.html +1 -1
- package/scope/.next/server/server-reference-manifest.json +1 -1
- /package/scope/.next/static/{vxqImMqlOwssVTua5Facf → x7wPUCpgS31-5ZHJkcKsU}/_buildManifest.js +0 -0
- /package/scope/.next/static/{vxqImMqlOwssVTua5Facf → x7wPUCpgS31-5ZHJkcKsU}/_ssgManifest.js +0 -0
|
@@ -0,0 +1,587 @@
|
|
|
1
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { join, resolve } from "node:path";
|
|
3
|
+
import { assertHarnessRunResultV1, MODEL_FUSION_SCHEMA_BUNDLE_HASH } from "@fusionkit/protocol";
|
|
4
|
+
import { gitText } from "@fusionkit/workspace";
|
|
5
|
+
import { createCommandHarness, createMockHarness, runEnsemble } from "@fusionkit/ensemble";
|
|
6
|
+
import { envFlagEnabled, readEnv } from "@fusionkit/tools";
|
|
7
|
+
import { toolRegistry } from "./tools.js";
|
|
8
|
+
const PRODUCER_GIT_SHA = "0".repeat(40);
|
|
9
|
+
const PRODUCER = "handoffkit-ensemble";
|
|
10
|
+
const PRODUCER_VERSION = "0.1.0";
|
|
11
|
+
const ZERO_GIT_SHA = "0".repeat(40);
|
|
12
|
+
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
13
|
+
const DEFAULT_PROMPT = "Run the CI-safe harness smoke task and report concise evidence.";
|
|
14
|
+
const DEFAULT_COMMAND_SUCCESS = "printf command-ok";
|
|
15
|
+
const DEFAULT_COMMAND_FAILURE = "exit 7";
|
|
16
|
+
const DEFAULT_OUTPUT_DIR = ".warrant/ensemble-dashboard";
|
|
17
|
+
const ALL_LIVE_SMOKE_ENV = "FUSIONKIT_ENSEMBLE_LIVE_SMOKE";
|
|
18
|
+
/** Dashboard capability overlays for the generic (non-tool) harnesses. */
|
|
19
|
+
const COMMAND_CAPABILITIES = {
|
|
20
|
+
model_override: "supported",
|
|
21
|
+
transcript_capture: "supported",
|
|
22
|
+
diff_capture: "unsupported",
|
|
23
|
+
tool_loop_capture: "supported",
|
|
24
|
+
patch_apply_visibility: "unsupported",
|
|
25
|
+
route_model_observation: "unsupported",
|
|
26
|
+
verification_hint: "supported",
|
|
27
|
+
replay_support: "supported"
|
|
28
|
+
};
|
|
29
|
+
const MOCK_CAPABILITIES = {
|
|
30
|
+
model_override: "supported",
|
|
31
|
+
transcript_capture: "supported",
|
|
32
|
+
diff_capture: "supported",
|
|
33
|
+
tool_loop_capture: "supported",
|
|
34
|
+
patch_apply_visibility: "supported",
|
|
35
|
+
route_model_observation: "degraded",
|
|
36
|
+
verification_hint: "supported",
|
|
37
|
+
replay_support: "supported"
|
|
38
|
+
};
|
|
39
|
+
function metadata(schema, createdAt) {
|
|
40
|
+
return {
|
|
41
|
+
schema,
|
|
42
|
+
schema_version: "v1",
|
|
43
|
+
schema_bundle_hash: MODEL_FUSION_SCHEMA_BUNDLE_HASH,
|
|
44
|
+
producer: PRODUCER,
|
|
45
|
+
producer_version: PRODUCER_VERSION,
|
|
46
|
+
producer_git_sha: PRODUCER_GIT_SHA,
|
|
47
|
+
created_at: createdAt
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
function dashboardTools(options) {
|
|
51
|
+
return options.tools ?? toolRegistry.dashboardTools();
|
|
52
|
+
}
|
|
53
|
+
function safeFileName(value) {
|
|
54
|
+
return value.replace(/[^A-Za-z0-9_.:-]/g, "_");
|
|
55
|
+
}
|
|
56
|
+
function escapeMarkdownCell(value) {
|
|
57
|
+
return value.replace(/\|/g, "\\|").replace(/\n/g, " ");
|
|
58
|
+
}
|
|
59
|
+
function liveSmokeEnvEnabled(env, tool) {
|
|
60
|
+
return (tool.liveSmoke !== undefined &&
|
|
61
|
+
(envFlagEnabled(env, ALL_LIVE_SMOKE_ENV) || envFlagEnabled(env, tool.liveSmoke.envName)));
|
|
62
|
+
}
|
|
63
|
+
function requestedLiveSmokeTools(options) {
|
|
64
|
+
const liveCapable = options.tools.filter((tool) => tool.liveSmoke !== undefined);
|
|
65
|
+
const selected = options.liveSmoke?.length
|
|
66
|
+
? liveCapable.filter((tool) => options.liveSmoke?.includes(tool.id))
|
|
67
|
+
: liveCapable;
|
|
68
|
+
return selected.filter((tool) => liveSmokeEnvEnabled(options.env, tool));
|
|
69
|
+
}
|
|
70
|
+
function descriptorForCapabilities(harness) {
|
|
71
|
+
return {
|
|
72
|
+
id: "capability_matrix",
|
|
73
|
+
harness,
|
|
74
|
+
models: [{ id: "capability", model: "capability-model" }],
|
|
75
|
+
runtime: { id: "local" },
|
|
76
|
+
judge: { id: "none" },
|
|
77
|
+
policy: {
|
|
78
|
+
id: "capability-policy",
|
|
79
|
+
allowedTools: ["read_file"],
|
|
80
|
+
sideEffects: "read_only",
|
|
81
|
+
timeoutMs: DEFAULT_TIMEOUT_MS
|
|
82
|
+
},
|
|
83
|
+
prompt: DEFAULT_PROMPT,
|
|
84
|
+
sourceRepo: "handoffkit",
|
|
85
|
+
baseGitSha: ZERO_GIT_SHA
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
function adapterCapabilities(harness) {
|
|
89
|
+
return harness.capabilities(descriptorForCapabilities(harness));
|
|
90
|
+
}
|
|
91
|
+
function mergeCapabilities(adapter, overlay) {
|
|
92
|
+
return { ...adapter, ...overlay };
|
|
93
|
+
}
|
|
94
|
+
function matrixRow(input) {
|
|
95
|
+
return input;
|
|
96
|
+
}
|
|
97
|
+
export function createHarnessCapabilityMatrix(options = {}) {
|
|
98
|
+
const env = options.env ?? {};
|
|
99
|
+
const toolRows = dashboardTools(options).map((tool) => matrixRow({
|
|
100
|
+
harnessId: tool.id,
|
|
101
|
+
harnessKind: tool.harnessKind,
|
|
102
|
+
displayName: tool.displayName,
|
|
103
|
+
availability: tool.availability,
|
|
104
|
+
capabilities: mergeCapabilities(adapterCapabilities(tool.makeMatrixHarness(env)), tool.capabilities),
|
|
105
|
+
notes: tool.notes
|
|
106
|
+
}));
|
|
107
|
+
const rows = [
|
|
108
|
+
...toolRows,
|
|
109
|
+
matrixRow({
|
|
110
|
+
harnessId: "command",
|
|
111
|
+
harnessKind: "generic",
|
|
112
|
+
displayName: "Command",
|
|
113
|
+
availability: "available",
|
|
114
|
+
capabilities: mergeCapabilities(adapterCapabilities(createCommandHarness({
|
|
115
|
+
command: options.commandSuccess ?? DEFAULT_COMMAND_SUCCESS,
|
|
116
|
+
cwd: options.repo
|
|
117
|
+
})), COMMAND_CAPABILITIES),
|
|
118
|
+
notes: ["Runs local shell commands through the command harness."]
|
|
119
|
+
}),
|
|
120
|
+
matrixRow({
|
|
121
|
+
harnessId: "mock",
|
|
122
|
+
harnessKind: "generic",
|
|
123
|
+
displayName: "Mock",
|
|
124
|
+
availability: "available",
|
|
125
|
+
capabilities: mergeCapabilities(adapterCapabilities(createMockHarness()), MOCK_CAPABILITIES),
|
|
126
|
+
notes: ["Pure synthetic fixture harness for CI."]
|
|
127
|
+
})
|
|
128
|
+
];
|
|
129
|
+
const capabilities = [...new Set(rows.flatMap((row) => Object.keys(row.capabilities)))].sort();
|
|
130
|
+
return { capabilities, rows };
|
|
131
|
+
}
|
|
132
|
+
function currentGitSha(repo) {
|
|
133
|
+
try {
|
|
134
|
+
const sha = gitText(repo, ["rev-parse", "HEAD"]).trim();
|
|
135
|
+
return sha.length > 0 ? sha : ZERO_GIT_SHA;
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
return ZERO_GIT_SHA;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
function smokeDescriptor(input) {
|
|
142
|
+
return {
|
|
143
|
+
id: input.id,
|
|
144
|
+
harness: input.harness,
|
|
145
|
+
models: [input.model],
|
|
146
|
+
runtime: { id: "local" },
|
|
147
|
+
judge: { id: "none" },
|
|
148
|
+
policy: {
|
|
149
|
+
id: `${input.id}_policy`,
|
|
150
|
+
allowedTools: input.allowedTools,
|
|
151
|
+
sideEffects: input.sideEffects,
|
|
152
|
+
timeoutMs: input.timeoutMs
|
|
153
|
+
},
|
|
154
|
+
prompt: input.prompt,
|
|
155
|
+
sourceRepo: input.repo,
|
|
156
|
+
baseGitSha: input.baseGitSha,
|
|
157
|
+
outputRoot: join(input.outputRoot, "runs", input.id)
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
function liveSmokePreflightFailureResult(input) {
|
|
161
|
+
const result = {
|
|
162
|
+
...metadata("harness-run-result.v1", input.createdAt),
|
|
163
|
+
result_id: `ensemble_result_${input.taskId}`,
|
|
164
|
+
request_id: `ensemble_req_${input.taskId}`,
|
|
165
|
+
harness_kind: input.harnessKind,
|
|
166
|
+
status: "failed",
|
|
167
|
+
candidate_ids: [],
|
|
168
|
+
output_summary: `Explicit live smoke failed before launch: ${input.reason}`,
|
|
169
|
+
capabilities: input.capabilities,
|
|
170
|
+
started_at: input.createdAt,
|
|
171
|
+
finished_at: input.createdAt,
|
|
172
|
+
errors: [
|
|
173
|
+
{
|
|
174
|
+
kind: "capability_missing",
|
|
175
|
+
message: input.reason,
|
|
176
|
+
retryable: false
|
|
177
|
+
}
|
|
178
|
+
],
|
|
179
|
+
metadata: {
|
|
180
|
+
dashboard_outcome: "failure",
|
|
181
|
+
harness_id: input.taskId,
|
|
182
|
+
live_smoke: true,
|
|
183
|
+
preflight: "credentials"
|
|
184
|
+
}
|
|
185
|
+
};
|
|
186
|
+
assertHarnessRunResultV1(result);
|
|
187
|
+
return result;
|
|
188
|
+
}
|
|
189
|
+
function failSkippedLiveSmokeResult(result, taskId) {
|
|
190
|
+
if (result.status !== "skipped")
|
|
191
|
+
return result;
|
|
192
|
+
const metadata = {
|
|
193
|
+
...(result.metadata ?? {}),
|
|
194
|
+
dashboard_outcome: "failure",
|
|
195
|
+
explicit_live_smoke: true,
|
|
196
|
+
original_status: "skipped"
|
|
197
|
+
};
|
|
198
|
+
const promoted = {
|
|
199
|
+
...result,
|
|
200
|
+
status: "failed",
|
|
201
|
+
output_summary: `Explicit live smoke ${taskId} failed because the adapter returned skipped. ` +
|
|
202
|
+
(result.output_summary ?? ""),
|
|
203
|
+
errors: [
|
|
204
|
+
...(result.errors ?? []),
|
|
205
|
+
{
|
|
206
|
+
kind: "capability_missing",
|
|
207
|
+
message: "Explicit live smoke was requested but the adapter returned skipped.",
|
|
208
|
+
retryable: false
|
|
209
|
+
}
|
|
210
|
+
],
|
|
211
|
+
metadata
|
|
212
|
+
};
|
|
213
|
+
assertHarnessRunResultV1(promoted);
|
|
214
|
+
return promoted;
|
|
215
|
+
}
|
|
216
|
+
function writeRunResult(outputRoot, taskId, result) {
|
|
217
|
+
const dir = join(outputRoot, "harness-run-results");
|
|
218
|
+
mkdirSync(dir, { recursive: true });
|
|
219
|
+
const path = join(dir, `${safeFileName(taskId)}.json`);
|
|
220
|
+
assertHarnessRunResultV1(result);
|
|
221
|
+
writeFileSync(path, JSON.stringify(result, null, 2) + "\n");
|
|
222
|
+
return path;
|
|
223
|
+
}
|
|
224
|
+
async function runSmokeTask(input) {
|
|
225
|
+
if (input.run.preflightFailureReason !== undefined) {
|
|
226
|
+
const result = liveSmokePreflightFailureResult({
|
|
227
|
+
createdAt: input.createdAt,
|
|
228
|
+
taskId: input.run.taskId,
|
|
229
|
+
harnessKind: input.run.harnessKind,
|
|
230
|
+
capabilities: mergeCapabilities(adapterCapabilities(input.run.harness), input.run.capabilities),
|
|
231
|
+
reason: input.run.preflightFailureReason
|
|
232
|
+
});
|
|
233
|
+
const resultPath = writeRunResult(input.outputRoot, input.run.taskId, result);
|
|
234
|
+
return {
|
|
235
|
+
taskId: input.run.taskId,
|
|
236
|
+
harnessId: input.run.harnessId,
|
|
237
|
+
purpose: input.run.purpose,
|
|
238
|
+
outcome: input.run.outcome,
|
|
239
|
+
result,
|
|
240
|
+
resultPath
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
const descriptor = smokeDescriptor({
|
|
244
|
+
id: input.run.taskId,
|
|
245
|
+
harness: input.run.harness,
|
|
246
|
+
model: input.run.model,
|
|
247
|
+
repo: input.repo,
|
|
248
|
+
baseGitSha: input.baseGitSha,
|
|
249
|
+
outputRoot: input.outputRoot,
|
|
250
|
+
sideEffects: input.run.sideEffects,
|
|
251
|
+
allowedTools: input.run.allowedTools,
|
|
252
|
+
timeoutMs: input.timeoutMs,
|
|
253
|
+
prompt: input.run.prompt ?? DEFAULT_PROMPT
|
|
254
|
+
});
|
|
255
|
+
const result = await runEnsemble(descriptor);
|
|
256
|
+
const harnessRunResult = input.run.purpose === "live"
|
|
257
|
+
? failSkippedLiveSmokeResult(result.harnessRunResult, input.run.taskId)
|
|
258
|
+
: result.harnessRunResult;
|
|
259
|
+
const resultPath = writeRunResult(input.outputRoot, input.run.taskId, harnessRunResult);
|
|
260
|
+
return {
|
|
261
|
+
taskId: input.run.taskId,
|
|
262
|
+
harnessId: input.run.harnessId,
|
|
263
|
+
purpose: input.run.purpose,
|
|
264
|
+
outcome: input.run.outcome,
|
|
265
|
+
result: harnessRunResult,
|
|
266
|
+
resultPath
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
function genericSmokeRuns(options) {
|
|
270
|
+
return [
|
|
271
|
+
{
|
|
272
|
+
taskId: "mock-success",
|
|
273
|
+
harnessId: "mock",
|
|
274
|
+
harnessKind: "generic",
|
|
275
|
+
purpose: "contract",
|
|
276
|
+
outcome: "success",
|
|
277
|
+
harness: createMockHarness(),
|
|
278
|
+
model: { id: "mock", model: "synthetic-mock" },
|
|
279
|
+
sideEffects: "read_only",
|
|
280
|
+
allowedTools: ["read_file"],
|
|
281
|
+
capabilities: MOCK_CAPABILITIES
|
|
282
|
+
},
|
|
283
|
+
{
|
|
284
|
+
taskId: "command-success",
|
|
285
|
+
harnessId: "command",
|
|
286
|
+
harnessKind: "generic",
|
|
287
|
+
purpose: "contract",
|
|
288
|
+
outcome: "success",
|
|
289
|
+
harness: createCommandHarness({ command: options.commandSuccess }),
|
|
290
|
+
model: { id: "command", model: "local-shell" },
|
|
291
|
+
sideEffects: "tool_execution",
|
|
292
|
+
allowedTools: ["shell_command"],
|
|
293
|
+
capabilities: COMMAND_CAPABILITIES
|
|
294
|
+
},
|
|
295
|
+
{
|
|
296
|
+
taskId: "command-failure",
|
|
297
|
+
harnessId: "command",
|
|
298
|
+
harnessKind: "generic",
|
|
299
|
+
purpose: "contract",
|
|
300
|
+
outcome: "failure",
|
|
301
|
+
harness: createCommandHarness({ command: options.commandFailure }),
|
|
302
|
+
model: { id: "command", model: "local-shell" },
|
|
303
|
+
sideEffects: "tool_execution",
|
|
304
|
+
allowedTools: ["shell_command"],
|
|
305
|
+
capabilities: COMMAND_CAPABILITIES
|
|
306
|
+
}
|
|
307
|
+
];
|
|
308
|
+
}
|
|
309
|
+
function credentialSkipSmokeRuns(tools) {
|
|
310
|
+
return tools.map((tool) => ({
|
|
311
|
+
taskId: tool.smoke.taskId,
|
|
312
|
+
harnessId: tool.id,
|
|
313
|
+
harnessKind: tool.harnessKind,
|
|
314
|
+
purpose: "credential-skip",
|
|
315
|
+
outcome: "skipped",
|
|
316
|
+
harness: tool.smoke.makeHarness(),
|
|
317
|
+
model: tool.smoke.model,
|
|
318
|
+
sideEffects: tool.smoke.sideEffects,
|
|
319
|
+
allowedTools: tool.smoke.allowedTools,
|
|
320
|
+
capabilities: tool.capabilities
|
|
321
|
+
}));
|
|
322
|
+
}
|
|
323
|
+
function liveSmokeRuns(options) {
|
|
324
|
+
const runs = [];
|
|
325
|
+
for (const tool of options.tools) {
|
|
326
|
+
const live = tool.liveSmoke;
|
|
327
|
+
if (live === undefined)
|
|
328
|
+
continue;
|
|
329
|
+
const harness = options.harnesses?.[tool.id] ?? live.makeHarness(options.env);
|
|
330
|
+
runs.push({
|
|
331
|
+
taskId: live.taskId,
|
|
332
|
+
harnessId: tool.id,
|
|
333
|
+
harnessKind: tool.harnessKind,
|
|
334
|
+
purpose: "live",
|
|
335
|
+
outcome: "success",
|
|
336
|
+
harness,
|
|
337
|
+
model: { id: tool.smoke.model.id, model: readEnv(options.env, live.modelEnvName) ?? live.defaultModel },
|
|
338
|
+
sideEffects: "read_only",
|
|
339
|
+
allowedTools: ["read_file"],
|
|
340
|
+
capabilities: tool.capabilities,
|
|
341
|
+
prompt: live.prompt,
|
|
342
|
+
preflightFailureReason: tool.credentialSkipReason(options.env)
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
return runs;
|
|
346
|
+
}
|
|
347
|
+
function capabilityCell(capabilities, capability) {
|
|
348
|
+
return capabilities[capability] ?? "unknown";
|
|
349
|
+
}
|
|
350
|
+
function renderCapabilityMatrix(matrix) {
|
|
351
|
+
const header = [
|
|
352
|
+
"Harness",
|
|
353
|
+
"Kind",
|
|
354
|
+
"Availability",
|
|
355
|
+
...matrix.capabilities,
|
|
356
|
+
"Notes"
|
|
357
|
+
];
|
|
358
|
+
const lines = [
|
|
359
|
+
"## Capability Matrix",
|
|
360
|
+
"",
|
|
361
|
+
`| ${header.map(escapeMarkdownCell).join(" | ")} |`,
|
|
362
|
+
`| ${header.map(() => "---").join(" | ")} |`
|
|
363
|
+
];
|
|
364
|
+
for (const row of matrix.rows) {
|
|
365
|
+
const cells = [
|
|
366
|
+
row.displayName,
|
|
367
|
+
row.harnessKind,
|
|
368
|
+
row.availability,
|
|
369
|
+
...matrix.capabilities.map((capability) => capabilityCell(row.capabilities, capability)),
|
|
370
|
+
row.notes.join(" ")
|
|
371
|
+
];
|
|
372
|
+
lines.push(`| ${cells.map(escapeMarkdownCell).join(" | ")} |`);
|
|
373
|
+
}
|
|
374
|
+
return lines;
|
|
375
|
+
}
|
|
376
|
+
function relativePath(path, from) {
|
|
377
|
+
return path.startsWith(from) ? path.slice(from.length + 1) : path;
|
|
378
|
+
}
|
|
379
|
+
function safeArtifactRefs(artifacts) {
|
|
380
|
+
if (artifacts === undefined || artifacts.length === 0)
|
|
381
|
+
return [];
|
|
382
|
+
const refs = [];
|
|
383
|
+
let rawWithheld = 0;
|
|
384
|
+
for (const artifact of artifacts) {
|
|
385
|
+
if (artifact.redaction_status === "raw") {
|
|
386
|
+
rawWithheld++;
|
|
387
|
+
continue;
|
|
388
|
+
}
|
|
389
|
+
refs.push(`${artifact.kind}:${artifact.artifact_id}:${artifact.hash}`);
|
|
390
|
+
if (refs.length >= 5)
|
|
391
|
+
break;
|
|
392
|
+
}
|
|
393
|
+
if (rawWithheld > 0)
|
|
394
|
+
refs.push(`${rawWithheld} raw artifact ref(s) withheld`);
|
|
395
|
+
return refs;
|
|
396
|
+
}
|
|
397
|
+
function liveSmokeState(record) {
|
|
398
|
+
if (record === undefined)
|
|
399
|
+
return "live smoke not requested";
|
|
400
|
+
switch (record.result.status) {
|
|
401
|
+
case "succeeded":
|
|
402
|
+
return "live smoke passed";
|
|
403
|
+
case "failed":
|
|
404
|
+
case "canceled":
|
|
405
|
+
case "requires_action":
|
|
406
|
+
case "unsupported":
|
|
407
|
+
case "pending":
|
|
408
|
+
case "running":
|
|
409
|
+
return "live smoke failed";
|
|
410
|
+
case "skipped":
|
|
411
|
+
return "live smoke skipped";
|
|
412
|
+
default: {
|
|
413
|
+
const exhausted = record.result.status;
|
|
414
|
+
throw new Error(`unsupported live smoke status: ${String(exhausted)}`);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
function createAdapterReadiness(input) {
|
|
419
|
+
return input.matrix.rows.map((row) => {
|
|
420
|
+
const liveRecord = input.records.find((record) => record.harnessId === row.harnessId && record.purpose === "live");
|
|
421
|
+
const credentialRecord = input.records.find((record) => record.harnessId === row.harnessId && record.purpose === "credential-skip");
|
|
422
|
+
const credentialState = input.credentialStateFor(row.harnessId);
|
|
423
|
+
const evidence = [
|
|
424
|
+
...(credentialRecord !== undefined && credentialState.includes("missing")
|
|
425
|
+
? [`credential skip: ${relativePath(credentialRecord.resultPath, input.outputRoot)}`]
|
|
426
|
+
: []),
|
|
427
|
+
...(liveRecord !== undefined
|
|
428
|
+
? [
|
|
429
|
+
`live result: ${relativePath(liveRecord.resultPath, input.outputRoot)}`,
|
|
430
|
+
`status=${liveRecord.result.status}`
|
|
431
|
+
]
|
|
432
|
+
: [])
|
|
433
|
+
];
|
|
434
|
+
return {
|
|
435
|
+
harnessId: row.harnessId,
|
|
436
|
+
displayName: row.displayName,
|
|
437
|
+
contractReadiness: "contract/mock ready",
|
|
438
|
+
credentialState,
|
|
439
|
+
liveSmoke: liveSmokeState(liveRecord),
|
|
440
|
+
evidence,
|
|
441
|
+
artifactRefs: liveRecord !== undefined ? safeArtifactRefs(liveRecord.result.artifacts) : []
|
|
442
|
+
};
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
function renderAdapterReadiness(readiness) {
|
|
446
|
+
const lines = [
|
|
447
|
+
"## Adapter Readiness",
|
|
448
|
+
"",
|
|
449
|
+
"| Adapter | Contract/Mock Readiness | Credentials | Live Smoke | Last Evidence | Safe Artifact Refs |",
|
|
450
|
+
"| --- | --- | --- | --- | --- | --- |"
|
|
451
|
+
];
|
|
452
|
+
for (const row of readiness) {
|
|
453
|
+
const cells = [
|
|
454
|
+
row.displayName,
|
|
455
|
+
row.contractReadiness,
|
|
456
|
+
row.credentialState,
|
|
457
|
+
row.liveSmoke,
|
|
458
|
+
row.evidence.length > 0 ? row.evidence.join("; ") : "-",
|
|
459
|
+
row.artifactRefs.length > 0 ? row.artifactRefs.join("; ") : "-"
|
|
460
|
+
];
|
|
461
|
+
lines.push(`| ${cells.map(escapeMarkdownCell).join(" | ")} |`);
|
|
462
|
+
}
|
|
463
|
+
return lines;
|
|
464
|
+
}
|
|
465
|
+
function renderSmokeRecords(records, outputRoot, displayNameFor) {
|
|
466
|
+
const lines = [
|
|
467
|
+
"## Smoke Records",
|
|
468
|
+
"",
|
|
469
|
+
"| Task | Harness | Purpose | Expected | Result Status | Harness Kind | Result Record | Summary |",
|
|
470
|
+
"| --- | --- | --- | --- | --- | --- | --- | --- |"
|
|
471
|
+
];
|
|
472
|
+
for (const record of records) {
|
|
473
|
+
const cells = [
|
|
474
|
+
record.taskId,
|
|
475
|
+
displayNameFor(record.harnessId),
|
|
476
|
+
record.purpose,
|
|
477
|
+
record.outcome,
|
|
478
|
+
record.result.status,
|
|
479
|
+
record.result.harness_kind,
|
|
480
|
+
relativePath(record.resultPath, outputRoot),
|
|
481
|
+
record.result.output_summary ?? ""
|
|
482
|
+
];
|
|
483
|
+
lines.push(`| ${cells.map(escapeMarkdownCell).join(" | ")} |`);
|
|
484
|
+
}
|
|
485
|
+
return lines;
|
|
486
|
+
}
|
|
487
|
+
function renderDashboard(input) {
|
|
488
|
+
const counts = new Map();
|
|
489
|
+
for (const record of input.records) {
|
|
490
|
+
counts.set(record.result.status, (counts.get(record.result.status) ?? 0) + 1);
|
|
491
|
+
}
|
|
492
|
+
const countText = [...counts.entries()]
|
|
493
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
494
|
+
.map(([status, count]) => `${status}:${count}`)
|
|
495
|
+
.join(", ");
|
|
496
|
+
return [
|
|
497
|
+
"# HandoffKit Harness Smoke Dashboard",
|
|
498
|
+
"",
|
|
499
|
+
`Generated: ${input.createdAt}`,
|
|
500
|
+
`Output root: ${input.outputRoot}`,
|
|
501
|
+
`Run-result status counts: ${countText}`,
|
|
502
|
+
"",
|
|
503
|
+
...renderCapabilityMatrix(input.matrix),
|
|
504
|
+
"",
|
|
505
|
+
...renderAdapterReadiness(input.readiness),
|
|
506
|
+
"",
|
|
507
|
+
...renderSmokeRecords(input.records, input.outputRoot, input.displayNameFor),
|
|
508
|
+
""
|
|
509
|
+
].join("\n");
|
|
510
|
+
}
|
|
511
|
+
export async function runHarnessSmokeDashboard(options = {}) {
|
|
512
|
+
const repo = resolve(options.repo ?? process.cwd());
|
|
513
|
+
const outputRoot = resolve(options.outputRoot ?? join(repo, DEFAULT_OUTPUT_DIR));
|
|
514
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
515
|
+
const createdAt = options.createdAt ?? new Date().toISOString();
|
|
516
|
+
const commandSuccess = options.commandSuccess ?? DEFAULT_COMMAND_SUCCESS;
|
|
517
|
+
const commandFailure = options.commandFailure ?? DEFAULT_COMMAND_FAILURE;
|
|
518
|
+
const env = options.env ?? process.env;
|
|
519
|
+
const tools = dashboardTools(options);
|
|
520
|
+
mkdirSync(outputRoot, { recursive: true });
|
|
521
|
+
const matrix = createHarnessCapabilityMatrix({
|
|
522
|
+
...options,
|
|
523
|
+
repo,
|
|
524
|
+
commandSuccess,
|
|
525
|
+
commandFailure,
|
|
526
|
+
env
|
|
527
|
+
});
|
|
528
|
+
const toolsById = new Map(tools.map((tool) => [tool.id, tool]));
|
|
529
|
+
const displayNameFor = (harnessId) => toolsById.get(harnessId)?.displayName ??
|
|
530
|
+
matrix.rows.find((row) => row.harnessId === harnessId)?.displayName ??
|
|
531
|
+
harnessId;
|
|
532
|
+
const credentialStateFor = (harnessId) => {
|
|
533
|
+
const tool = toolsById.get(harnessId);
|
|
534
|
+
if (tool === undefined)
|
|
535
|
+
return "not required";
|
|
536
|
+
return tool.credentialSkipReason(env) === undefined
|
|
537
|
+
? "credentials available"
|
|
538
|
+
: "credentials missing/skipped";
|
|
539
|
+
};
|
|
540
|
+
const baseGitSha = currentGitSha(repo);
|
|
541
|
+
const records = [];
|
|
542
|
+
const runs = [
|
|
543
|
+
...genericSmokeRuns({ commandSuccess, commandFailure }),
|
|
544
|
+
...credentialSkipSmokeRuns(tools),
|
|
545
|
+
...liveSmokeRuns({
|
|
546
|
+
env,
|
|
547
|
+
tools: requestedLiveSmokeTools({ env, tools, liveSmoke: options.liveSmoke }),
|
|
548
|
+
harnesses: options.liveSmokeHarnesses
|
|
549
|
+
})
|
|
550
|
+
];
|
|
551
|
+
for (const run of runs) {
|
|
552
|
+
records.push(await runSmokeTask({
|
|
553
|
+
run,
|
|
554
|
+
repo,
|
|
555
|
+
baseGitSha,
|
|
556
|
+
outputRoot,
|
|
557
|
+
timeoutMs,
|
|
558
|
+
createdAt
|
|
559
|
+
}));
|
|
560
|
+
}
|
|
561
|
+
const readiness = createAdapterReadiness({
|
|
562
|
+
matrix,
|
|
563
|
+
records,
|
|
564
|
+
credentialStateFor,
|
|
565
|
+
outputRoot
|
|
566
|
+
});
|
|
567
|
+
const dashboardPath = join(outputRoot, "dashboard.md");
|
|
568
|
+
writeFileSync(dashboardPath, renderDashboard({
|
|
569
|
+
matrix,
|
|
570
|
+
readiness,
|
|
571
|
+
records,
|
|
572
|
+
createdAt,
|
|
573
|
+
outputRoot,
|
|
574
|
+
displayNameFor
|
|
575
|
+
}));
|
|
576
|
+
return {
|
|
577
|
+
outputRoot,
|
|
578
|
+
dashboardPath,
|
|
579
|
+
matrix,
|
|
580
|
+
records,
|
|
581
|
+
readiness
|
|
582
|
+
};
|
|
583
|
+
}
|
|
584
|
+
export const harnessDashboard = {
|
|
585
|
+
capabilities: createHarnessCapabilityMatrix,
|
|
586
|
+
run: runHarnessSmokeDashboard
|
|
587
|
+
};
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/** A launchable tool id from the registry, or the `serve` pseudo-tool. */
|
|
2
|
+
export type FusionTool = string;
|
|
3
|
+
export type PanelProvider = "mlx" | "openai" | "anthropic" | "google" | "openai-compatible";
|
|
4
|
+
/**
|
|
5
|
+
* One panel model. `mlx` models run locally via the in-repo provisioner; cloud
|
|
6
|
+
* providers (openai/anthropic/google/openai-compatible) are fronted as
|
|
7
|
+
* OpenAI-compatible endpoints by FusionKit's `serve-endpoint` command, run via
|
|
8
|
+
* `uvx fusionkit` (no checkout required).
|
|
9
|
+
*/
|
|
10
|
+
export type PanelModelSpec = {
|
|
11
|
+
id: string;
|
|
12
|
+
model: string;
|
|
13
|
+
provider?: PanelProvider;
|
|
14
|
+
baseUrl?: string;
|
|
15
|
+
keyEnv?: string;
|
|
16
|
+
};
|
|
17
|
+
export type RunFusionOptions = {
|
|
18
|
+
models?: PanelModelSpec[];
|
|
19
|
+
endpoints?: Record<string, string>;
|
|
20
|
+
fusionkitDir?: string;
|
|
21
|
+
repo?: string;
|
|
22
|
+
judgeModel?: string;
|
|
23
|
+
synthesisUrl?: string;
|
|
24
|
+
authToken?: string;
|
|
25
|
+
port?: number;
|
|
26
|
+
timeoutMs?: number;
|
|
27
|
+
/** Use the local MLX panel trio (Apple Silicon) instead of the cloud panel. */
|
|
28
|
+
local?: boolean;
|
|
29
|
+
/** Boot the local scope dashboard and stream trace events into it. */
|
|
30
|
+
observe?: boolean;
|
|
31
|
+
/** Skip the interactive cost/scope confirmation for the cloud panel. */
|
|
32
|
+
yes?: boolean;
|
|
33
|
+
/** Route services through portless (stable named URLs + singletons). Default on. */
|
|
34
|
+
portless?: boolean;
|
|
35
|
+
log?: (line: string) => void;
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* Structured boot progress. When a reporter is supplied the stack emits these
|
|
39
|
+
* events instead of the plain `fusion: ...` log lines, so a live TUI (or any
|
|
40
|
+
* other consumer) can render per-stage status. Without one, callers keep getting
|
|
41
|
+
* the existing line logs.
|
|
42
|
+
*/
|
|
43
|
+
export type StackEvent = {
|
|
44
|
+
kind: "server.start";
|
|
45
|
+
id: string;
|
|
46
|
+
label: string;
|
|
47
|
+
} | {
|
|
48
|
+
kind: "server.ready";
|
|
49
|
+
id: string;
|
|
50
|
+
detail: string;
|
|
51
|
+
} | {
|
|
52
|
+
kind: "server.fail";
|
|
53
|
+
id: string;
|
|
54
|
+
detail: string;
|
|
55
|
+
} | {
|
|
56
|
+
kind: "synth.start";
|
|
57
|
+
} | {
|
|
58
|
+
kind: "synth.ready";
|
|
59
|
+
detail: string;
|
|
60
|
+
} | {
|
|
61
|
+
kind: "gateway.start";
|
|
62
|
+
} | {
|
|
63
|
+
kind: "gateway.ready";
|
|
64
|
+
detail: string;
|
|
65
|
+
} | {
|
|
66
|
+
kind: "dashboard.start";
|
|
67
|
+
} | {
|
|
68
|
+
kind: "dashboard.ready";
|
|
69
|
+
detail: string;
|
|
70
|
+
} | {
|
|
71
|
+
kind: "dashboard.fail";
|
|
72
|
+
detail: string;
|
|
73
|
+
};
|
|
74
|
+
export type StackReporter = (event: StackEvent) => void;
|
|
75
|
+
/**
|
|
76
|
+
* The PyPI version of the `fusionkit` Python distribution that provides the
|
|
77
|
+
* synthesizer (`fusionkit serve`) and the single-model OpenAI shim
|
|
78
|
+
* (`fusionkit serve-endpoint`). Pinned so `uvx` resolves a reproducible build.
|
|
79
|
+
*/
|
|
80
|
+
export declare const FUSIONKIT_PYPI_VERSION = "0.2.0";
|
|
81
|
+
/**
|
|
82
|
+
* Default cloud panel — works cross-platform with only `OPENAI_API_KEY` and
|
|
83
|
+
* `ANTHROPIC_API_KEY` set. The judge defaults to the first entry.
|
|
84
|
+
*/
|
|
85
|
+
export declare const DEFAULT_CLOUD_PANEL: readonly PanelModelSpec[];
|
|
86
|
+
/** The locally cached MLX trio (Apple Silicon only) used behind `--local`. */
|
|
87
|
+
export declare const DEFAULT_TRIO: readonly PanelModelSpec[];
|
|
88
|
+
/**
|
|
89
|
+
* How to invoke the `fusionkit` Python CLI: from PyPI via `uvx` by default
|
|
90
|
+
* (no checkout), or from a local checkout via `uv run` when `fusionkitDir` is
|
|
91
|
+
* given (a dev override). Returns the command plus the argv prefix that
|
|
92
|
+
* precedes the subcommand (`serve`, `serve-endpoint`, ...).
|
|
93
|
+
*/
|
|
94
|
+
export declare function fusionkitPyCommand(fusionkitDir?: string): {
|
|
95
|
+
command: string;
|
|
96
|
+
prefix: string[];
|
|
97
|
+
cwd?: string;
|
|
98
|
+
};
|
|
99
|
+
/**
|
|
100
|
+
* Parse a `.env` file (KEY=VALUE lines, `#` comments, optional `export`,
|
|
101
|
+
* single/double quotes) and fill any keys not already present in `env`.
|
|
102
|
+
* Existing env values win, so an explicitly exported key is never overridden.
|
|
103
|
+
*/
|
|
104
|
+
export declare function loadEnvFileInto(path: string, env: Record<string, string | undefined>): void;
|
|
105
|
+
/** Default env var holding the API key for each cloud provider. */
|
|
106
|
+
export declare function defaultKeyEnv(provider: PanelProvider): string | undefined;
|
|
107
|
+
/** The git repository root containing `dir`, or undefined if it is not in a repo. */
|
|
108
|
+
export declare function gitToplevel(dir: string): string | undefined;
|