@workbench-ai/workbench 0.0.64 → 0.0.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark-fingerprint.js +2 -2
- package/dist/command-model.d.ts +1 -1
- package/dist/command-model.d.ts.map +1 -1
- package/dist/command-model.js +106 -35
- package/dist/dev-open/client.js +109 -109
- package/dist/dev-open-server.d.ts +2 -37
- package/dist/dev-open-server.d.ts.map +1 -1
- package/dist/dev-open-server.js +39 -322
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +333 -193
- package/dist/local-archive.d.ts +4 -4
- package/dist/local-archive.d.ts.map +1 -1
- package/dist/local-inspection.d.ts +9 -0
- package/dist/local-inspection.d.ts.map +1 -0
- package/dist/local-inspection.js +317 -0
- package/dist/project-source.d.ts +6 -6
- package/dist/project-source.js +6 -6
- package/package.json +4 -4
package/dist/local-archive.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type CandidateRecord, type EvaluationScorecard, type
|
|
1
|
+
import { type CandidateRecord, type EvaluationScorecard, type RemoteWorkbenchJob, type RunSummary, type RuntimeEvent, type SurfaceSnapshotFile, type WorkbenchRuntimeBundle, type WorkbenchRuntimeBundleStats, type WorkbenchRuntimeImportResult, type WorkbenchExecutionTrace, type WorkbenchTraceSession } from "@workbench-ai/workbench-core";
|
|
2
2
|
export interface LocalArchiveSnapshot {
|
|
3
3
|
activeId: string | null;
|
|
4
4
|
candidates: CandidateRecord[];
|
|
@@ -14,7 +14,7 @@ export interface LocalArchiveIndex {
|
|
|
14
14
|
runs: RunSummary[];
|
|
15
15
|
events: RuntimeEvent[];
|
|
16
16
|
}
|
|
17
|
-
export type LocalArchivedJob =
|
|
17
|
+
export type LocalArchivedJob = RemoteWorkbenchJob & {
|
|
18
18
|
trace?: WorkbenchExecutionTrace;
|
|
19
19
|
traceSessions?: WorkbenchTraceSession[];
|
|
20
20
|
};
|
|
@@ -22,13 +22,13 @@ export declare function localRuntimeDir(workspace: string): string;
|
|
|
22
22
|
export declare function loadLocalArchive(workspace: string): Promise<LocalArchiveSnapshot>;
|
|
23
23
|
export declare function loadLocalArchiveIndex(workspace: string): Promise<LocalArchiveIndex>;
|
|
24
24
|
export declare function saveLocalArchive(workspace: string, snapshot: LocalArchiveSnapshot): Promise<void>;
|
|
25
|
-
export declare function saveLocalJobs(workspace: string, jobs: readonly
|
|
25
|
+
export declare function saveLocalJobs(workspace: string, jobs: readonly RemoteWorkbenchJob[]): Promise<void>;
|
|
26
26
|
export declare function exportLocalRuntimeBundle(workspace: string, options?: {
|
|
27
27
|
currentBenchmarkFingerprint?: string;
|
|
28
28
|
}): Promise<WorkbenchRuntimeBundle>;
|
|
29
29
|
export declare function importLocalRuntimeBundle(workspace: string, bundle: WorkbenchRuntimeBundle, currentBenchmarkFingerprint: string): Promise<WorkbenchRuntimeImportResult>;
|
|
30
30
|
export declare function runtimeBundleStats(bundle: WorkbenchRuntimeBundle): WorkbenchRuntimeBundleStats;
|
|
31
|
-
export declare function sanitizeRuntimeJobForExchange(job:
|
|
31
|
+
export declare function sanitizeRuntimeJobForExchange(job: RemoteWorkbenchJob): RemoteWorkbenchJob;
|
|
32
32
|
export declare function readLocalExecutionFiles(workspace: string, jobId: string): Promise<SurfaceSnapshotFile[]>;
|
|
33
33
|
export declare function readLocalCandidateRecord(workspace: string, candidateId: string): Promise<CandidateRecord>;
|
|
34
34
|
export declare function readLocalCandidateFilesForId(workspace: string, candidateId: string): Promise<SurfaceSnapshotFile[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local-archive.d.ts","sourceRoot":"","sources":["../src/local-archive.ts"],"names":[],"mappings":"AAGA,OAAO,EAYL,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,KAAK,UAAU,EACf,KAAK,YAAY,EAEjB,KAAK,mBAAmB,EACxB,KAAK,sBAAsB,EAC3B,KAAK,2BAA2B,EAChC,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAC5B,KAAK,qBAAqB,EAC3B,MAAM,8BAA8B,CAAC;AAOtC,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,mBAAmB,EAAE,CAAC,CAAC;IACtD,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,MAAM,MAAM,gBAAgB,GAAG,kBAAkB,GAAG;IAClD,KAAK,CAAC,EAAE,uBAAuB,CAAC;IAChC,aAAa,CAAC,EAAE,qBAAqB,EAAE,CAAC;CACzC,CAAC;AASF,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAevF;AAED,wBAAsB,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAkBzF;AAED,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,oBAAoB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,SAAS,kBAAkB,EAAE,GAClC,OAAO,CAAC,IAAI,CAAC,CAKf;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IAAE,2BAA2B,CAAC,EAAE,MAAM,CAAA;CAAO,GACrD,OAAO,CAAC,sBAAsB,CAAC,CAyBjC;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,sBAAsB,EAC9B,2BAA2B,EAAE,MAAM,GAClC,OAAO,CAAC,4BAA4B,CAAC,CAoHvC;AAED,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,sBAAsB,GAC7B,2BAA2B,CAE7B;AAED,wBAAgB,6BAA6B,CAC3C,GAAG,EAAE,kBAAkB,GACtB,kBAAkB,CAEpB;
|
|
1
|
+
{"version":3,"file":"local-archive.d.ts","sourceRoot":"","sources":["../src/local-archive.ts"],"names":[],"mappings":"AAGA,OAAO,EAYL,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,KAAK,UAAU,EACf,KAAK,YAAY,EAEjB,KAAK,mBAAmB,EACxB,KAAK,sBAAsB,EAC3B,KAAK,2BAA2B,EAChC,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAC5B,KAAK,qBAAqB,EAC3B,MAAM,8BAA8B,CAAC;AAOtC,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,mBAAmB,EAAE,CAAC,CAAC;IACtD,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,MAAM,MAAM,gBAAgB,GAAG,kBAAkB,GAAG;IAClD,KAAK,CAAC,EAAE,uBAAuB,CAAC;IAChC,aAAa,CAAC,EAAE,qBAAqB,EAAE,CAAC;CACzC,CAAC;AASF,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAevF;AAED,wBAAsB,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAkBzF;AAED,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,oBAAoB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,SAAS,kBAAkB,EAAE,GAClC,OAAO,CAAC,IAAI,CAAC,CAKf;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE;IAAE,2BAA2B,CAAC,EAAE,MAAM,CAAA;CAAO,GACrD,OAAO,CAAC,sBAAsB,CAAC,CAyBjC;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,sBAAsB,EAC9B,2BAA2B,EAAE,MAAM,GAClC,OAAO,CAAC,4BAA4B,CAAC,CAoHvC;AAED,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,sBAAsB,GAC7B,2BAA2B,CAE7B;AAED,wBAAgB,6BAA6B,CAC3C,GAAG,EAAE,kBAAkB,GACtB,kBAAkB,CAEpB;AA+DD,wBAAsB,uBAAuB,CAC3C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAOhC;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,eAAe,CAAC,CAU1B;AAED,wBAAsB,4BAA4B,CAChD,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAKhC;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,mBAAmB,CAAC,CAU9B;AAED,wBAAsB,kBAAkB,CACtC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,UAAU,CAAC,CAUrB;AAED,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAM7B;AAED,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAE7B;AAED,wBAAsB,iBAAiB,CACrC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAElC;AAED,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,oBAAoB,EAC9B,SAAS,EAAE,eAAe,EAC1B,KAAK,EAAE,SAAS,mBAAmB,EAAE,GACpC,oBAAoB,CAYtB;AAED,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,oBAAoB,EAC9B,UAAU,EAAE,mBAAmB,GAC9B,oBAAoB,CAQtB;AAED,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,oBAAoB,EAC9B,GAAG,EAAE,UAAU,EACf,MAAM,EAAE,SAAS,YAAY,EAAE,GAC9B,oBAAoB,CAYtB;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,oBAAoB,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,oBAAoB,CAK5G;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,GAAG,eAAe,CAMvG;AAED,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,GAAG,mBAAmB,EAAE,CAGlH;AAkkBD,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,KAAK,EAAE,SAAS,mBAAmB,EAAE,GACpC,OAAO,CAAC,MAAM,EAAE,CAAC,CAOnB;AAED,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,SAAS,mBAAmB,EAAE,EACrC,QAAQ,EAAE,MAAM,GACf,mBAAmB,GAAG,IAAI,CAG5B"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type WorkbenchInspection } from "@workbench-ai/workbench-core";
|
|
2
|
+
import { type LocalProjectSource } from "./project-source.js";
|
|
3
|
+
export interface LocalWorkbenchInspectionOptions {
|
|
4
|
+
workspace: string;
|
|
5
|
+
readProjectSource?: () => Promise<LocalProjectSource>;
|
|
6
|
+
}
|
|
7
|
+
export declare function createLocalWorkbenchInspection(options: LocalWorkbenchInspectionOptions): WorkbenchInspection;
|
|
8
|
+
export declare function createLocalProjectSourceReader(workspace: string): () => Promise<LocalProjectSource>;
|
|
9
|
+
//# sourceMappingURL=local-inspection.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local-inspection.d.ts","sourceRoot":"","sources":["../src/local-inspection.ts"],"names":[],"mappings":"AAEA,OAAO,EAaL,KAAK,mBAAmB,EAGzB,MAAM,8BAA8B,CAAC;AAetC,OAAO,EAGL,KAAK,kBAAkB,EAExB,MAAM,qBAAqB,CAAC;AAgB7B,MAAM,WAAW,+BAA+B;IAC9C,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,MAAM,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACvD;AAED,wBAAgB,8BAA8B,CAC5C,OAAO,EAAE,+BAA+B,GACvC,mBAAmB,CAqCrB;AAED,wBAAgB,8BAA8B,CAC5C,SAAS,EAAE,MAAM,GAChB,MAAM,OAAO,CAAC,kBAAkB,CAAC,CAiBnC"}
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { WorkbenchInspectionError, candidateRecordWithoutDerivedFields, candidateSummaryFromRecord, createWorkbenchInspection, loadAuthoredWorkbenchSourceDocument, traceSessionLabel, } from "@workbench-ai/workbench-core";
|
|
3
|
+
import { localBenchmarkFingerprint } from "./benchmark-fingerprint.js";
|
|
4
|
+
import { loadLocalArchiveIndex, readLocalCandidateFilesForId, readLocalCandidateRecord, readLocalEvaluationRecord, readLocalExecutionFiles, readLocalJobInRun, readLocalRunJobs, readLocalRunRecord, } from "./local-archive.js";
|
|
5
|
+
import { readLocalAuthoredProjectSource, readLocalProjectSource, WORKBENCH_BENCHMARK_FILE, } from "./project-source.js";
|
|
6
|
+
const PROJECT_SOURCE_CACHE_TTL_MS = 1000;
|
|
7
|
+
export function createLocalWorkbenchInspection(options) {
|
|
8
|
+
const context = {
|
|
9
|
+
workspace: path.resolve(options.workspace),
|
|
10
|
+
readProjectSource: options.readProjectSource ??
|
|
11
|
+
createLocalProjectSourceReader(options.workspace),
|
|
12
|
+
};
|
|
13
|
+
const backend = {
|
|
14
|
+
projectId: "local",
|
|
15
|
+
snapshot: () => localBenchmarkSnapshot(context),
|
|
16
|
+
spec: (input) => localSpecDocument(context, input.fingerprint),
|
|
17
|
+
sourceFiles: (input) => localBenchmarkMountedFiles(context, input.fingerprint),
|
|
18
|
+
candidate: (input) => readCandidateForInspection(context.workspace, input.id),
|
|
19
|
+
candidateFiles: async (input) => {
|
|
20
|
+
const candidate = await readCandidateForInspection(context.workspace, input.id);
|
|
21
|
+
return {
|
|
22
|
+
files: await readCandidateFilesForInspection(context.workspace, input.id),
|
|
23
|
+
changedPaths: candidate.fileChanges,
|
|
24
|
+
};
|
|
25
|
+
},
|
|
26
|
+
evaluation: (input) => readEvaluationForInspection(context.workspace, input.id),
|
|
27
|
+
run: async (input) => {
|
|
28
|
+
const run = await readRunForInspection(context.workspace, input.id);
|
|
29
|
+
return {
|
|
30
|
+
run,
|
|
31
|
+
...(input.includeJobs
|
|
32
|
+
? { jobs: await readLocalRunJobs(context.workspace, input.id) }
|
|
33
|
+
: {}),
|
|
34
|
+
};
|
|
35
|
+
},
|
|
36
|
+
jobInRun: (input) => readExecutionJobForRun(context.workspace, input.runId, input.jobId),
|
|
37
|
+
executionFiles: (input) => readExecutionFilesForRun(context.workspace, input.runId, input.jobId),
|
|
38
|
+
traceForJob: readLocalAggregateTrace,
|
|
39
|
+
traceSessionsForJob: readLocalTraceSessions,
|
|
40
|
+
};
|
|
41
|
+
return createWorkbenchInspection(backend);
|
|
42
|
+
}
|
|
43
|
+
export function createLocalProjectSourceReader(workspace) {
|
|
44
|
+
const resolvedWorkspace = path.resolve(workspace);
|
|
45
|
+
let cached = null;
|
|
46
|
+
return () => {
|
|
47
|
+
const now = Date.now();
|
|
48
|
+
if (cached && now - cached.loadedAt < PROJECT_SOURCE_CACHE_TTL_MS) {
|
|
49
|
+
return cached.promise;
|
|
50
|
+
}
|
|
51
|
+
const promise = readLocalProjectSource(resolvedWorkspace);
|
|
52
|
+
cached = { loadedAt: now, promise };
|
|
53
|
+
promise.catch(() => {
|
|
54
|
+
if (cached?.promise === promise) {
|
|
55
|
+
cached = null;
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
return promise;
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
async function localBenchmarkSnapshot(context) {
|
|
62
|
+
const { workspace } = context;
|
|
63
|
+
const snapshot = await loadLocalArchiveIndex(workspace);
|
|
64
|
+
const candidates = snapshot.candidates.filter(isInspectableCandidateRecord);
|
|
65
|
+
const summaries = candidates.map(candidateSummaryFromRecord);
|
|
66
|
+
const activeId = snapshot.activeId && candidates.some((candidate) => candidate.id === snapshot.activeId)
|
|
67
|
+
? snapshot.activeId
|
|
68
|
+
: null;
|
|
69
|
+
const currentBenchmarkFingerprint = await readCurrentBenchmarkFingerprint(context);
|
|
70
|
+
return {
|
|
71
|
+
workspaceRoot: path.resolve(workspace),
|
|
72
|
+
activeId,
|
|
73
|
+
currentBenchmarkFingerprint,
|
|
74
|
+
summaries,
|
|
75
|
+
evaluations: snapshot.evaluations.map(evaluationSummary),
|
|
76
|
+
runs: snapshot.runs.map(publicLocalRunSummary),
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
async function localSpecDocument(context, benchmarkFingerprint) {
|
|
80
|
+
const { workspace } = context;
|
|
81
|
+
const projectSource = await context.readProjectSource().catch(() => null);
|
|
82
|
+
const authoredSource = projectSource
|
|
83
|
+
? null
|
|
84
|
+
: await readLocalAuthoredProjectSource(workspace).catch(() => null);
|
|
85
|
+
const requestedFingerprint = normalizeOptionalFingerprint(benchmarkFingerprint);
|
|
86
|
+
const currentFingerprint = projectSource
|
|
87
|
+
? localBenchmarkFingerprint(projectSource)
|
|
88
|
+
: null;
|
|
89
|
+
if (requestedFingerprint &&
|
|
90
|
+
currentFingerprint &&
|
|
91
|
+
requestedFingerprint !== currentFingerprint) {
|
|
92
|
+
const snapshot = await loadLocalArchiveIndex(workspace);
|
|
93
|
+
const document = localHistoricalBenchmarkDocument(snapshot, requestedFingerprint);
|
|
94
|
+
if (document) {
|
|
95
|
+
return document;
|
|
96
|
+
}
|
|
97
|
+
throw new WorkbenchInspectionError(`Benchmark version not found: ${requestedFingerprint}`, { status: 404 });
|
|
98
|
+
}
|
|
99
|
+
const sourceYaml = projectSource?.specSource ?? authoredSource?.specSource ?? "";
|
|
100
|
+
const cases = projectSource
|
|
101
|
+
? caseSummaryFilesFromEngineCases(projectSource.engineCases, projectSource.engineResolveFiles)
|
|
102
|
+
: [];
|
|
103
|
+
return loadAuthoredWorkbenchSourceDocument({
|
|
104
|
+
sourceYaml,
|
|
105
|
+
path: WORKBENCH_BENCHMARK_FILE,
|
|
106
|
+
sourceFiles: projectSource?.sourceFiles ?? authoredSource?.sourceFiles,
|
|
107
|
+
cases,
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
async function localBenchmarkMountedFiles(context, benchmarkFingerprint) {
|
|
111
|
+
const requestedFingerprint = normalizeOptionalFingerprint(benchmarkFingerprint);
|
|
112
|
+
const { workspace } = context;
|
|
113
|
+
const projectSource = await context.readProjectSource();
|
|
114
|
+
const currentFingerprint = localBenchmarkFingerprint(projectSource);
|
|
115
|
+
if (requestedFingerprint &&
|
|
116
|
+
currentFingerprint &&
|
|
117
|
+
requestedFingerprint !== currentFingerprint) {
|
|
118
|
+
const snapshot = await loadLocalArchiveIndex(workspace);
|
|
119
|
+
return localHistoricalBenchmarkFiles(snapshot, requestedFingerprint);
|
|
120
|
+
}
|
|
121
|
+
return inspectableEngineCaseFiles(projectSource.engineCases);
|
|
122
|
+
}
|
|
123
|
+
function publicLocalRunSummary(run) {
|
|
124
|
+
const { executionFingerprint: _executionFingerprint, ...summary } = run;
|
|
125
|
+
return summary;
|
|
126
|
+
}
|
|
127
|
+
async function readCurrentBenchmarkFingerprint(context) {
|
|
128
|
+
return await context.readProjectSource()
|
|
129
|
+
.then(localBenchmarkFingerprint)
|
|
130
|
+
.catch(() => null);
|
|
131
|
+
}
|
|
132
|
+
function caseSummaryFilesFromEngineCases(engineCases, files) {
|
|
133
|
+
const existingCaseIds = new Set(files.flatMap((file) => {
|
|
134
|
+
const normalized = file.path.replace(/\\/gu, "/").replace(/^\/+/u, "");
|
|
135
|
+
const slash = normalized.indexOf("/");
|
|
136
|
+
return slash > 0 ? [normalized.slice(0, slash)] : [];
|
|
137
|
+
}));
|
|
138
|
+
return [
|
|
139
|
+
...files.map((file) => ({ ...file })),
|
|
140
|
+
...engineCases
|
|
141
|
+
.filter((engineCase) => !existingCaseIds.has(engineCase.id))
|
|
142
|
+
.map((engineCase) => ({
|
|
143
|
+
path: `${engineCase.id}/.workbench-case.json`,
|
|
144
|
+
encoding: "utf8",
|
|
145
|
+
content: `${JSON.stringify({ id: engineCase.id })}\n`,
|
|
146
|
+
executable: false,
|
|
147
|
+
})),
|
|
148
|
+
];
|
|
149
|
+
}
|
|
150
|
+
function localHistoricalBenchmarkDocument(snapshot, benchmarkFingerprint) {
|
|
151
|
+
const candidate = snapshot.candidates.find((entry) => entry.benchmarkFingerprint === benchmarkFingerprint && readBenchmarkSourceMetadata(entry));
|
|
152
|
+
const source = candidate ? readBenchmarkSourceMetadata(candidate) : null;
|
|
153
|
+
if (!source?.sourceYaml) {
|
|
154
|
+
return null;
|
|
155
|
+
}
|
|
156
|
+
return loadAuthoredWorkbenchSourceDocument({
|
|
157
|
+
sourceYaml: source.sourceYaml,
|
|
158
|
+
path: WORKBENCH_BENCHMARK_FILE,
|
|
159
|
+
sourceFiles: source.files,
|
|
160
|
+
cases: localHistoricalBenchmarkFiles(snapshot, benchmarkFingerprint),
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
function localHistoricalBenchmarkFiles(_snapshot, _benchmarkFingerprint) {
|
|
164
|
+
return [];
|
|
165
|
+
}
|
|
166
|
+
function inspectableEngineCaseFiles(engineCases) {
|
|
167
|
+
return engineCases.flatMap((bundle) => engineCaseFiles(bundle).map((file) => ({
|
|
168
|
+
...file,
|
|
169
|
+
path: `${bundle.id}/${file.path}`,
|
|
170
|
+
}))).sort((left, right) => left.path.localeCompare(right.path));
|
|
171
|
+
}
|
|
172
|
+
function engineCaseFiles(bundle) {
|
|
173
|
+
const buckets = bundle.files;
|
|
174
|
+
return buckets.source?.length
|
|
175
|
+
? buckets.source
|
|
176
|
+
: [...(buckets.public ?? []), ...(buckets.private ?? [])];
|
|
177
|
+
}
|
|
178
|
+
function isInspectableCandidateRecord(candidate) {
|
|
179
|
+
return Boolean(candidate.eval || asRecord(asRecord(candidate.meta)?.source));
|
|
180
|
+
}
|
|
181
|
+
function evaluationSummary(evaluation) {
|
|
182
|
+
const { evaluation: _evaluation, ...summary } = evaluation;
|
|
183
|
+
return summary;
|
|
184
|
+
}
|
|
185
|
+
async function readCandidateForInspection(workspace, candidateId) {
|
|
186
|
+
const candidate = await readArchiveRecord("Candidate", candidateId, () => readLocalCandidateRecord(workspace, candidateId));
|
|
187
|
+
return candidateRecordWithoutDerivedFields(candidate);
|
|
188
|
+
}
|
|
189
|
+
async function readEvaluationForInspection(workspace, evaluationId) {
|
|
190
|
+
return await readArchiveRecord("Evaluation", evaluationId, () => readLocalEvaluationRecord(workspace, evaluationId));
|
|
191
|
+
}
|
|
192
|
+
async function readRunForInspection(workspace, runId) {
|
|
193
|
+
return await readArchiveRecord("Run", runId, () => readLocalRunRecord(workspace, runId));
|
|
194
|
+
}
|
|
195
|
+
async function readCandidateFilesForInspection(workspace, candidateId) {
|
|
196
|
+
return await readArchiveRecord("Candidate", candidateId, () => readLocalCandidateFilesForId(workspace, candidateId));
|
|
197
|
+
}
|
|
198
|
+
function readBenchmarkSourceMetadata(candidate) {
|
|
199
|
+
const benchmark = asRecord(asRecord(candidate.meta)?.benchmark);
|
|
200
|
+
const files = Array.isArray(benchmark?.files)
|
|
201
|
+
? benchmark.files
|
|
202
|
+
.map(readSurfaceSnapshotFile)
|
|
203
|
+
.filter((file) => file !== null)
|
|
204
|
+
: [];
|
|
205
|
+
const sourceYaml = files.find((file) => file.path === WORKBENCH_BENCHMARK_FILE)?.content ?? null;
|
|
206
|
+
if (!sourceYaml) {
|
|
207
|
+
return null;
|
|
208
|
+
}
|
|
209
|
+
return { sourceYaml, files };
|
|
210
|
+
}
|
|
211
|
+
function readSurfaceSnapshotFile(value) {
|
|
212
|
+
const record = asRecord(value);
|
|
213
|
+
if (!record) {
|
|
214
|
+
return null;
|
|
215
|
+
}
|
|
216
|
+
const filePath = typeof record?.path === "string" ? record.path : "";
|
|
217
|
+
const content = typeof record?.content === "string" ? record.content : null;
|
|
218
|
+
if (!filePath || content === null) {
|
|
219
|
+
return null;
|
|
220
|
+
}
|
|
221
|
+
return {
|
|
222
|
+
path: filePath,
|
|
223
|
+
kind: record.kind === "binary" ? "binary" : "text",
|
|
224
|
+
encoding: record.encoding === "base64" ? "base64" : "utf8",
|
|
225
|
+
content,
|
|
226
|
+
executable: record.executable === true,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
async function readArchiveRecord(kind, id, read) {
|
|
230
|
+
try {
|
|
231
|
+
return await read();
|
|
232
|
+
}
|
|
233
|
+
catch (error) {
|
|
234
|
+
if (error instanceof Error && error.message === `${kind} not found: ${id}`) {
|
|
235
|
+
throw new WorkbenchInspectionError(error.message, { status: 404 });
|
|
236
|
+
}
|
|
237
|
+
throw error;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
async function readExecutionFilesForRun(workspace, runId, jobId) {
|
|
241
|
+
await readExecutionJobForRun(workspace, runId, jobId);
|
|
242
|
+
return await readLocalExecutionFiles(workspace, jobId);
|
|
243
|
+
}
|
|
244
|
+
async function readExecutionJobForRun(workspace, runId, jobId) {
|
|
245
|
+
const job = await readLocalJobInRun(workspace, runId, jobId);
|
|
246
|
+
if (!job) {
|
|
247
|
+
throw new WorkbenchInspectionError(`Execution job not found: ${jobId}`, { status: 404 });
|
|
248
|
+
}
|
|
249
|
+
return job;
|
|
250
|
+
}
|
|
251
|
+
function readLocalAggregateTrace(job) {
|
|
252
|
+
const trace = job.trace;
|
|
253
|
+
if (!trace || typeof trace !== "object" || Array.isArray(trace)) {
|
|
254
|
+
return { trace_id: job.id, spans: [], events: [], summaries: [] };
|
|
255
|
+
}
|
|
256
|
+
const record = trace;
|
|
257
|
+
return {
|
|
258
|
+
trace_id: typeof record.trace_id === "string" ? record.trace_id : job.id,
|
|
259
|
+
spans: Array.isArray(record.spans) ? record.spans : [],
|
|
260
|
+
events: Array.isArray(record.events) ? record.events : [],
|
|
261
|
+
summaries: Array.isArray(record.summaries) ? record.summaries : [],
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
function readLocalTraceSessions(job, role) {
|
|
265
|
+
if (!Array.isArray(job.traceSessions)) {
|
|
266
|
+
return [];
|
|
267
|
+
}
|
|
268
|
+
return job.traceSessions.map((session) => ({
|
|
269
|
+
id: typeof session.id === "string" && session.id.length > 0
|
|
270
|
+
? session.id
|
|
271
|
+
: `${job.id}:trace`,
|
|
272
|
+
jobId: typeof session.jobId === "string" && session.jobId.length > 0
|
|
273
|
+
? session.jobId
|
|
274
|
+
: job.id,
|
|
275
|
+
role: session.role === "improver" || session.role === "runner" || session.role === "engine"
|
|
276
|
+
? session.role
|
|
277
|
+
: role,
|
|
278
|
+
kind: typeof session.kind === "string" && session.kind.length > 0 ? session.kind : "trace",
|
|
279
|
+
label: traceSessionDisplayLabel(session, role),
|
|
280
|
+
sourcePath: typeof session.sourcePath === "string" ? session.sourcePath : null,
|
|
281
|
+
trace: sanitizeLocalTrace(session.trace, session.id),
|
|
282
|
+
...(session.metadata && typeof session.metadata === "object" && !Array.isArray(session.metadata)
|
|
283
|
+
? { metadata: session.metadata }
|
|
284
|
+
: {}),
|
|
285
|
+
}));
|
|
286
|
+
}
|
|
287
|
+
function traceSessionDisplayLabel(session, fallbackRole) {
|
|
288
|
+
const role = session.role === "improver" || session.role === "runner" || session.role === "engine"
|
|
289
|
+
? session.role
|
|
290
|
+
: fallbackRole;
|
|
291
|
+
return typeof session.label === "string" && session.label.length > 0
|
|
292
|
+
? session.label
|
|
293
|
+
: typeof session.sourcePath === "string" && session.sourcePath.length > 0
|
|
294
|
+
? traceSessionLabel(session.sourcePath, role)
|
|
295
|
+
: "Trace";
|
|
296
|
+
}
|
|
297
|
+
function sanitizeLocalTrace(trace, fallbackId) {
|
|
298
|
+
if (!trace || typeof trace !== "object" || Array.isArray(trace)) {
|
|
299
|
+
return { trace_id: fallbackId, spans: [], events: [], summaries: [] };
|
|
300
|
+
}
|
|
301
|
+
const record = trace;
|
|
302
|
+
return {
|
|
303
|
+
trace_id: typeof record.trace_id === "string" ? record.trace_id : fallbackId,
|
|
304
|
+
spans: Array.isArray(record.spans) ? record.spans : [],
|
|
305
|
+
events: Array.isArray(record.events) ? record.events : [],
|
|
306
|
+
summaries: Array.isArray(record.summaries) ? record.summaries : [],
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
function asRecord(value) {
|
|
310
|
+
return value && typeof value === "object" && !Array.isArray(value)
|
|
311
|
+
? value
|
|
312
|
+
: null;
|
|
313
|
+
}
|
|
314
|
+
function normalizeOptionalFingerprint(value) {
|
|
315
|
+
const normalized = value?.trim();
|
|
316
|
+
return normalized ? normalized : null;
|
|
317
|
+
}
|
package/dist/project-source.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { type ResolvedWorkbenchAdapter } from "./adapter-project.js";
|
|
|
5
5
|
export declare const WORKBENCH_BENCHMARK_FILE = "benchmark.yaml";
|
|
6
6
|
export declare const WORKBENCH_CANDIDATES_DIR = "candidates";
|
|
7
7
|
export declare const WORKBENCH_CANDIDATE_FILE = "candidate.yaml";
|
|
8
|
-
export type
|
|
8
|
+
export type RemoteFile = WorkspaceSnapshotFile;
|
|
9
9
|
export interface LocalProjectSource {
|
|
10
10
|
dir: string;
|
|
11
11
|
specPath: string;
|
|
@@ -25,11 +25,11 @@ export interface LocalProjectSource {
|
|
|
25
25
|
dockerfilePath: string;
|
|
26
26
|
dockerfile: string;
|
|
27
27
|
runtimeDockerfile: string;
|
|
28
|
-
dockerfileFiles:
|
|
29
|
-
candidateFiles:
|
|
30
|
-
engineResolveFiles:
|
|
28
|
+
dockerfileFiles: RemoteFile[];
|
|
29
|
+
candidateFiles: RemoteFile[];
|
|
30
|
+
engineResolveFiles: RemoteFile[];
|
|
31
31
|
adapters: ResolvedWorkbenchAdapter[];
|
|
32
|
-
adapterFiles:
|
|
32
|
+
adapterFiles: RemoteFile[];
|
|
33
33
|
caseIds: string[];
|
|
34
34
|
engineCases: WorkbenchEngineCase[];
|
|
35
35
|
engineResolve: LocalEngineResolveInvocation;
|
|
@@ -58,6 +58,6 @@ interface LocalProjectSourceOptions {
|
|
|
58
58
|
}
|
|
59
59
|
export declare function readLocalProjectSource(source: string, options?: LocalProjectSourceOptions): Promise<LocalProjectSource>;
|
|
60
60
|
export declare function readLocalAuthoredProjectSource(source: string, options?: LocalProjectSourceOptions): Promise<LocalAuthoredProjectSource>;
|
|
61
|
-
export declare function
|
|
61
|
+
export declare function remoteEngineResolveFiles(source: LocalProjectSource): RemoteFile[];
|
|
62
62
|
export {};
|
|
63
63
|
//# sourceMappingURL=project-source.d.ts.map
|
package/dist/project-source.js
CHANGED
|
@@ -60,7 +60,7 @@ export async function readLocalProjectSource(source, options = {}) {
|
|
|
60
60
|
? normalizeSurfaceFiles(await readSnapshotFiles(absoluteCandidateFilesPath))
|
|
61
61
|
: [];
|
|
62
62
|
const rawEngineResolveFiles = engineResolveFilesFromBundles(normalizedSources.engineCases);
|
|
63
|
-
const engineResolveFiles =
|
|
63
|
+
const engineResolveFiles = toRemoteFiles(rawEngineResolveFiles);
|
|
64
64
|
const engineCases = normalizedSources.engineCases;
|
|
65
65
|
if (engineCases.length === 0) {
|
|
66
66
|
throw new WorkspaceSnapshotError(`Engine resolver ${normalizedSources.engineResolve.use} did not emit any cases.`);
|
|
@@ -85,11 +85,11 @@ export async function readLocalProjectSource(source, options = {}) {
|
|
|
85
85
|
dockerfilePath,
|
|
86
86
|
dockerfile,
|
|
87
87
|
runtimeDockerfile: composedDockerfile,
|
|
88
|
-
dockerfileFiles:
|
|
89
|
-
candidateFiles:
|
|
88
|
+
dockerfileFiles: toRemoteFiles(dockerfileSourceFiles(dockerfileSources)),
|
|
89
|
+
candidateFiles: toRemoteFiles(candidateFiles),
|
|
90
90
|
engineResolveFiles,
|
|
91
91
|
adapters,
|
|
92
|
-
adapterFiles:
|
|
92
|
+
adapterFiles: toRemoteFiles(adapterFiles),
|
|
93
93
|
caseIds,
|
|
94
94
|
engineCases,
|
|
95
95
|
engineResolve: normalizedSources.engineResolve,
|
|
@@ -136,7 +136,7 @@ export async function readLocalAuthoredProjectSource(source, options = {}) {
|
|
|
136
136
|
],
|
|
137
137
|
};
|
|
138
138
|
}
|
|
139
|
-
export function
|
|
139
|
+
export function remoteEngineResolveFiles(source) {
|
|
140
140
|
return [
|
|
141
141
|
...source.engineResolveFiles,
|
|
142
142
|
{
|
|
@@ -672,7 +672,7 @@ function adapterSourceSnapshotPath(adapter, filePath) {
|
|
|
672
672
|
? `${adapter.source}/${filePath}`
|
|
673
673
|
: `adapters/${adapter.manifest.id}/${filePath}`;
|
|
674
674
|
}
|
|
675
|
-
function
|
|
675
|
+
function toRemoteFiles(files) {
|
|
676
676
|
return files.map((file) => ({
|
|
677
677
|
path: file.path,
|
|
678
678
|
content: file.content,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@workbench-ai/workbench",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.65",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"registry": "https://registry.npmjs.org/",
|
|
@@ -16,9 +16,9 @@
|
|
|
16
16
|
],
|
|
17
17
|
"dependencies": {
|
|
18
18
|
"yaml": "^2.8.2",
|
|
19
|
-
"@workbench-ai/workbench-built-in-adapters": "0.0.
|
|
20
|
-
"@workbench-ai/workbench-protocol": "0.0.
|
|
21
|
-
"@workbench-ai/workbench-core": "0.0.
|
|
19
|
+
"@workbench-ai/workbench-built-in-adapters": "0.0.65",
|
|
20
|
+
"@workbench-ai/workbench-protocol": "0.0.65",
|
|
21
|
+
"@workbench-ai/workbench-core": "0.0.65"
|
|
22
22
|
},
|
|
23
23
|
"devDependencies": {
|
|
24
24
|
"@tailwindcss/postcss": "^4.2.2",
|