@workbench-ai/workbench 0.0.66 → 0.0.68

package/dist/local-inspection.d.ts

@@ -1,9 +0,0 @@
-import { type WorkbenchInspection } from "@workbench-ai/workbench-core";
-import { type LocalProjectSource } from "./project-source.js";
-export interface LocalWorkbenchInspectionOptions {
-    workspace: string;
-    readProjectSource?: () => Promise<LocalProjectSource>;
-}
-export declare function createLocalWorkbenchInspection(options: LocalWorkbenchInspectionOptions): WorkbenchInspection;
-export declare function createLocalProjectSourceReader(workspace: string): () => Promise<LocalProjectSource>;
-//# sourceMappingURL=local-inspection.d.ts.map

package/dist/local-inspection.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"local-inspection.d.ts","sourceRoot":"","sources":["../src/local-inspection.ts"],"names":[],"mappings":"AAEA,OAAO,EAaL,KAAK,mBAAmB,EAGzB,MAAM,8BAA8B,CAAC;AAetC,OAAO,EAGL,KAAK,kBAAkB,EAExB,MAAM,qBAAqB,CAAC;AAgB7B,MAAM,WAAW,+BAA+B;IAC9C,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,MAAM,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACvD;AAED,wBAAgB,8BAA8B,CAC5C,OAAO,EAAE,+BAA+B,GACvC,mBAAmB,CAqCrB;AAED,wBAAgB,8BAA8B,CAC5C,SAAS,EAAE,MAAM,GAChB,MAAM,OAAO,CAAC,kBAAkB,CAAC,CAiBnC"}

package/dist/local-inspection.js

@@ -1,317 +0,0 @@
-import path from "node:path";
-import { WorkbenchInspectionError, candidateRecordWithoutDerivedFields, candidateSummaryFromRecord, createWorkbenchInspection, loadAuthoredWorkbenchSourceDocument, traceSessionLabel, } from "@workbench-ai/workbench-core";
-import { localBenchmarkFingerprint } from "./benchmark-fingerprint.js";
-import { loadLocalArchiveIndex, readLocalCandidateFilesForId, readLocalCandidateRecord, readLocalEvaluationRecord, readLocalExecutionFiles, readLocalJobInRun, readLocalRunJobs, readLocalRunRecord, } from "./local-archive.js";
-import { readLocalAuthoredProjectSource, readLocalProjectSource, WORKBENCH_BENCHMARK_FILE, } from "./project-source.js";
-const PROJECT_SOURCE_CACHE_TTL_MS = 1000;
-export function createLocalWorkbenchInspection(options) {
-    const context = {
-        workspace: path.resolve(options.workspace),
-        readProjectSource: options.readProjectSource ??
-            createLocalProjectSourceReader(options.workspace),
-    };
-    const backend = {
-        projectId: "local",
-        snapshot: () => localBenchmarkSnapshot(context),
-        spec: (input) => localSpecDocument(context, input.fingerprint),
-        sourceFiles: (input) => localBenchmarkMountedFiles(context, input.fingerprint),
-        candidate: (input) => readCandidateForInspection(context.workspace, input.id),
-        candidateFiles: async (input) => {
-            const candidate = await readCandidateForInspection(context.workspace, input.id);
-            return {
-                files: await readCandidateFilesForInspection(context.workspace, input.id),
-                changedPaths: candidate.fileChanges,
-            };
-        },
-        evaluation: (input) => readEvaluationForInspection(context.workspace, input.id),
-        run: async (input) => {
-            const run = await readRunForInspection(context.workspace, input.id);
-            return {
-                run,
-                ...(input.includeJobs
-                    ? { jobs: await readLocalRunJobs(context.workspace, input.id) }
-                    : {}),
-            };
-        },
-        jobInRun: (input) => readExecutionJobForRun(context.workspace, input.runId, input.jobId),
-        executionFiles: (input) => readExecutionFilesForRun(context.workspace, input.runId, input.jobId),
-        traceForJob: readLocalAggregateTrace,
-        traceSessionsForJob: readLocalTraceSessions,
-    };
-    return createWorkbenchInspection(backend);
-}
-export function createLocalProjectSourceReader(workspace) {
-    const resolvedWorkspace = path.resolve(workspace);
-    let cached = null;
-    return () => {
-        const now = Date.now();
-        if (cached && now - cached.loadedAt < PROJECT_SOURCE_CACHE_TTL_MS) {
-            return cached.promise;
-        }
-        const promise = readLocalProjectSource(resolvedWorkspace);
-        cached = { loadedAt: now, promise };
-        promise.catch(() => {
-            if (cached?.promise === promise) {
-                cached = null;
-            }
-        });
-        return promise;
-    };
-}
-async function localBenchmarkSnapshot(context) {
-    const { workspace } = context;
-    const snapshot = await loadLocalArchiveIndex(workspace);
-    const candidates = snapshot.candidates.filter(isInspectableCandidateRecord);
-    const summaries = candidates.map(candidateSummaryFromRecord);
-    const activeId = snapshot.activeId && candidates.some((candidate) => candidate.id === snapshot.activeId)
-        ? snapshot.activeId
-        : null;
-    const currentBenchmarkFingerprint = await readCurrentBenchmarkFingerprint(context);
-    return {
-        workspaceRoot: path.resolve(workspace),
-        activeId,
-        currentBenchmarkFingerprint,
-        summaries,
-        evaluations: snapshot.evaluations.map(evaluationSummary),
-        runs: snapshot.runs.map(publicLocalRunSummary),
-    };
-}
-async function localSpecDocument(context, benchmarkFingerprint) {
-    const { workspace } = context;
-    const projectSource = await context.readProjectSource().catch(() => null);
-    const authoredSource = projectSource
-        ? null
-        : await readLocalAuthoredProjectSource(workspace).catch(() => null);
-    const requestedFingerprint = normalizeOptionalFingerprint(benchmarkFingerprint);
-    const currentFingerprint = projectSource
-        ? localBenchmarkFingerprint(projectSource)
-        : null;
-    if (requestedFingerprint &&
-        currentFingerprint &&
-        requestedFingerprint !== currentFingerprint) {
-        const snapshot = await loadLocalArchiveIndex(workspace);
-        const document = localHistoricalBenchmarkDocument(snapshot, requestedFingerprint);
-        if (document) {
-            return document;
-        }
-        throw new WorkbenchInspectionError(`Benchmark version not found: ${requestedFingerprint}`, { status: 404 });
-    }
-    const sourceYaml = projectSource?.specSource ?? authoredSource?.specSource ?? "";
-    const cases = projectSource
-        ? caseSummaryFilesFromEngineCases(projectSource.engineCases, projectSource.engineResolveFiles)
-        : [];
-    return loadAuthoredWorkbenchSourceDocument({
-        sourceYaml,
-        path: WORKBENCH_BENCHMARK_FILE,
-        sourceFiles: projectSource?.sourceFiles ?? authoredSource?.sourceFiles,
-        cases,
-    });
-}
-async function localBenchmarkMountedFiles(context, benchmarkFingerprint) {
-    const requestedFingerprint = normalizeOptionalFingerprint(benchmarkFingerprint);
-    const { workspace } = context;
-    const projectSource = await context.readProjectSource();
-    const currentFingerprint = localBenchmarkFingerprint(projectSource);
-    if (requestedFingerprint &&
-        currentFingerprint &&
-        requestedFingerprint !== currentFingerprint) {
-        const snapshot = await loadLocalArchiveIndex(workspace);
-        return localHistoricalBenchmarkFiles(snapshot, requestedFingerprint);
-    }
-    return inspectableEngineCaseFiles(projectSource.engineCases);
-}
-function publicLocalRunSummary(run) {
-    const { executionFingerprint: _executionFingerprint, ...summary } = run;
-    return summary;
-}
-async function readCurrentBenchmarkFingerprint(context) {
-    return await context.readProjectSource()
-        .then(localBenchmarkFingerprint)
-        .catch(() => null);
-}
-function caseSummaryFilesFromEngineCases(engineCases, files) {
-    const existingCaseIds = new Set(files.flatMap((file) => {
-        const normalized = file.path.replace(/\\/gu, "/").replace(/^\/+/u, "");
-        const slash = normalized.indexOf("/");
-        return slash > 0 ? [normalized.slice(0, slash)] : [];
-    }));
-    return [
-        ...files.map((file) => ({ ...file })),
-        ...engineCases
-            .filter((engineCase) => !existingCaseIds.has(engineCase.id))
-            .map((engineCase) => ({
-            path: `${engineCase.id}/.workbench-case.json`,
-            encoding: "utf8",
-            content: `${JSON.stringify({ id: engineCase.id })}\n`,
-            executable: false,
-        })),
-    ];
-}
-function localHistoricalBenchmarkDocument(snapshot, benchmarkFingerprint) {
-    const candidate = snapshot.candidates.find((entry) => entry.benchmarkFingerprint === benchmarkFingerprint && readBenchmarkSourceMetadata(entry));
-    const source = candidate ? readBenchmarkSourceMetadata(candidate) : null;
-    if (!source?.sourceYaml) {
-        return null;
-    }
-    return loadAuthoredWorkbenchSourceDocument({
-        sourceYaml: source.sourceYaml,
-        path: WORKBENCH_BENCHMARK_FILE,
-        sourceFiles: source.files,
-        cases: localHistoricalBenchmarkFiles(snapshot, benchmarkFingerprint),
-    });
-}
-function localHistoricalBenchmarkFiles(_snapshot, _benchmarkFingerprint) {
-    return [];
-}
-function inspectableEngineCaseFiles(engineCases) {
-    return engineCases.flatMap((bundle) => engineCaseFiles(bundle).map((file) => ({
-        ...file,
-        path: `${bundle.id}/${file.path}`,
-    }))).sort((left, right) => left.path.localeCompare(right.path));
-}
-function engineCaseFiles(bundle) {
-    const buckets = bundle.files;
-    return buckets.source?.length
-        ? buckets.source
-        : [...(buckets.public ?? []), ...(buckets.private ?? [])];
-}
-function isInspectableCandidateRecord(candidate) {
-    return Boolean(candidate.eval || asRecord(asRecord(candidate.meta)?.source));
-}
-function evaluationSummary(evaluation) {
-    const { evaluation: _evaluation, ...summary } = evaluation;
-    return summary;
-}
-async function readCandidateForInspection(workspace, candidateId) {
-    const candidate = await readArchiveRecord("Candidate", candidateId, () => readLocalCandidateRecord(workspace, candidateId));
-    return candidateRecordWithoutDerivedFields(candidate);
-}
-async function readEvaluationForInspection(workspace, evaluationId) {
-    return await readArchiveRecord("Evaluation", evaluationId, () => readLocalEvaluationRecord(workspace, evaluationId));
-}
-async function readRunForInspection(workspace, runId) {
-    return await readArchiveRecord("Run", runId, () => readLocalRunRecord(workspace, runId));
-}
-async function readCandidateFilesForInspection(workspace, candidateId) {
-    return await readArchiveRecord("Candidate", candidateId, () => readLocalCandidateFilesForId(workspace, candidateId));
-}
-function readBenchmarkSourceMetadata(candidate) {
-    const benchmark = asRecord(asRecord(candidate.meta)?.benchmark);
-    const files = Array.isArray(benchmark?.files)
-        ? benchmark.files
-            .map(readSurfaceSnapshotFile)
-            .filter((file) => file !== null)
-        : [];
-    const sourceYaml = files.find((file) => file.path === WORKBENCH_BENCHMARK_FILE)?.content ?? null;
-    if (!sourceYaml) {
-        return null;
-    }
-    return { sourceYaml, files };
-}
-function readSurfaceSnapshotFile(value) {
-    const record = asRecord(value);
-    if (!record) {
-        return null;
-    }
-    const filePath = typeof record?.path === "string" ? record.path : "";
-    const content = typeof record?.content === "string" ? record.content : null;
-    if (!filePath || content === null) {
-        return null;
-    }
-    return {
-        path: filePath,
-        kind: record.kind === "binary" ? "binary" : "text",
-        encoding: record.encoding === "base64" ? "base64" : "utf8",
-        content,
-        executable: record.executable === true,
-    };
-}
-async function readArchiveRecord(kind, id, read) {
-    try {
-        return await read();
-    }
-    catch (error) {
-        if (error instanceof Error && error.message === `${kind} not found: ${id}`) {
-            throw new WorkbenchInspectionError(error.message, { status: 404 });
-        }
-        throw error;
-    }
-}
-async function readExecutionFilesForRun(workspace, runId, jobId) {
-    await readExecutionJobForRun(workspace, runId, jobId);
-    return await readLocalExecutionFiles(workspace, jobId);
-}
-async function readExecutionJobForRun(workspace, runId, jobId) {
-    const job = await readLocalJobInRun(workspace, runId, jobId);
-    if (!job) {
-        throw new WorkbenchInspectionError(`Execution job not found: ${jobId}`, { status: 404 });
-    }
-    return job;
-}
-function readLocalAggregateTrace(job) {
-    const trace = job.trace;
-    if (!trace || typeof trace !== "object" || Array.isArray(trace)) {
-        return { trace_id: job.id, spans: [], events: [], summaries: [] };
-    }
-    const record = trace;
-    return {
-        trace_id: typeof record.trace_id === "string" ? record.trace_id : job.id,
-        spans: Array.isArray(record.spans) ? record.spans : [],
-        events: Array.isArray(record.events) ? record.events : [],
-        summaries: Array.isArray(record.summaries) ? record.summaries : [],
-    };
-}
-function readLocalTraceSessions(job, role) {
-    if (!Array.isArray(job.traceSessions)) {
-        return [];
-    }
-    return job.traceSessions.map((session) => ({
-        id: typeof session.id === "string" && session.id.length > 0
-            ? session.id
-            : `${job.id}:trace`,
-        jobId: typeof session.jobId === "string" && session.jobId.length > 0
-            ? session.jobId
-            : job.id,
-        role: session.role === "improver" || session.role === "runner" || session.role === "engine"
-            ? session.role
-            : role,
-        kind: typeof session.kind === "string" && session.kind.length > 0 ? session.kind : "trace",
-        label: traceSessionDisplayLabel(session, role),
-        sourcePath: typeof session.sourcePath === "string" ? session.sourcePath : null,
-        trace: sanitizeLocalTrace(session.trace, session.id),
-        ...(session.metadata && typeof session.metadata === "object" && !Array.isArray(session.metadata)
-            ? { metadata: session.metadata }
-            : {}),
-    }));
-}
-function traceSessionDisplayLabel(session, fallbackRole) {
-    const role = session.role === "improver" || session.role === "runner" || session.role === "engine"
-        ? session.role
-        : fallbackRole;
-    return typeof session.label === "string" && session.label.length > 0
-        ? session.label
-        : typeof session.sourcePath === "string" && session.sourcePath.length > 0
-            ? traceSessionLabel(session.sourcePath, role)
-            : "Trace";
-}
-function sanitizeLocalTrace(trace, fallbackId) {
-    if (!trace || typeof trace !== "object" || Array.isArray(trace)) {
-        return { trace_id: fallbackId, spans: [], events: [], summaries: [] };
-    }
-    const record = trace;
-    return {
-        trace_id: typeof record.trace_id === "string" ? record.trace_id : fallbackId,
-        spans: Array.isArray(record.spans) ? record.spans : [],
-        events: Array.isArray(record.events) ? record.events : [],
-        summaries: Array.isArray(record.summaries) ? record.summaries : [],
-    };
-}
-function asRecord(value) {
-    return value && typeof value === "object" && !Array.isArray(value)
-        ? value
-        : null;
-}
-function normalizeOptionalFingerprint(value) {
-    const normalized = value?.trim();
-    return normalized ? normalized : null;
-}

package/dist/project-source.d.ts

@@ -1,63 +0,0 @@
-import { resolveWorkbenchResolvedSourceYaml, type Json, type SurfaceSnapshotFile } from "@workbench-ai/workbench-core";
-import { type WorkbenchEngineCase, type WorkbenchEngineResolveResult } from "@workbench-ai/workbench-protocol";
-import { type WorkspaceSnapshotFile } from "./workspace-snapshot.js";
-import { type ResolvedWorkbenchAdapter } from "./adapter-project.js";
-export declare const WORKBENCH_BENCHMARK_FILE = "benchmark.yaml";
-export declare const WORKBENCH_CANDIDATES_DIR = "candidates";
-export declare const WORKBENCH_CANDIDATE_FILE = "candidate.yaml";
-export type RemoteFile = WorkspaceSnapshotFile;
-export interface LocalProjectSource {
-    dir: string;
-    specPath: string;
-    specSource: string;
-    spec: ReturnType<typeof resolveWorkbenchResolvedSourceYaml>;
-    benchmarkPath: string;
-    benchmarkSource: string;
-    candidateName: string;
-    candidateDir: string;
-    candidateFilesPath: string;
-    candidateSpecPath: string;
-    candidateSource: string;
-    candidateRunId: string;
-    candidateRunIds: string[];
-    benchmarkAdapterSources: string[];
-    benchmarkAdapterIds: string[];
-    dockerfilePath: string;
-    dockerfile: string;
-    runtimeDockerfile: string;
-    dockerfileFiles: RemoteFile[];
-    candidateFiles: RemoteFile[];
-    engineResolveFiles: RemoteFile[];
-    adapters: ResolvedWorkbenchAdapter[];
-    adapterFiles: RemoteFile[];
-    caseIds: string[];
-    engineCases: WorkbenchEngineCase[];
-    engineResolve: LocalEngineResolveInvocation;
-    engineResolveFingerprintPath: string;
-    engineResolveEnvironment?: WorkbenchEngineResolveResult["environment"];
-    sourceFiles: SurfaceSnapshotFile[];
-}
-export interface LocalAuthoredProjectSource {
-    dir: string;
-    specPath: string;
-    specSource: string;
-    benchmarkPath: string;
-    benchmarkSource: string;
-    candidateDir: string;
-    candidateSpecPath: string;
-    candidateSource: string;
-    sourceFiles: SurfaceSnapshotFile[];
-}
-export interface LocalEngineResolveInvocation {
-    use: string;
-    with: Json;
-    auth?: Json;
-}
-interface LocalProjectSourceOptions {
-    runId?: string;
-}
-export declare function readLocalProjectSource(source: string, options?: LocalProjectSourceOptions): Promise<LocalProjectSource>;
-export declare function readLocalAuthoredProjectSource(source: string, options?: LocalProjectSourceOptions): Promise<LocalAuthoredProjectSource>;
-export declare function remoteEngineResolveFiles(source: LocalProjectSource): RemoteFile[];
-export {};
-//# sourceMappingURL=project-source.d.ts.map

package/dist/project-source.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"project-source.d.ts","sourceRoot":"","sources":["../src/project-source.ts"],"names":[],"mappings":"AAKA,OAAO,EAOL,kCAAkC,EAGlC,KAAK,IAAI,EACT,KAAK,mBAAmB,EACzB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAUL,KAAK,mBAAmB,EACxB,KAAK,4BAA4B,EAClC,MAAM,kCAAkC,CAAC;AAE1C,OAAO,EAGL,KAAK,qBAAqB,EAC3B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAML,KAAK,wBAAwB,EAC9B,MAAM,sBAAsB,CAAC;AAI9B,eAAO,MAAM,wBAAwB,mBAAsB,CAAC;AAC5D,eAAO,MAAM,wBAAwB,eAAe,CAAC;AACrD,eAAO,MAAM,wBAAwB,mBAAsB,CAAC;AAE5D,MAAM,MAAM,UAAU,GAAG,qBAAqB,CAAC;AAE/C,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,CAAC,OAAO,kCAAkC,CAAC,CAAC;IAC5D,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,uBAAuB,EAAE,MAAM,EAAE,CAAC;IAClC,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,UAAU,EAAE,CAAC;IAC9B,cAAc,EAAE,UAAU,EAAE,CAAC;IAC7B,kBAAkB,EAAE,UAAU,EAAE,CAAC;IACjC,QAAQ,EAAE,wBAAwB,EAAE,CAAC;IACrC,YAAY,EAAE,UAAU,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,aAAa,EAAE,4BAA4B,CAAC;IAC5C,4BAA4B,EAAE,MAAM,CAAC;IACrC,wBAAwB,CAAC,EAAE,4BAA4B,CAAC,aAAa,CAAC,CAAC;IACvE,WAAW,EAAE,mBAAmB,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,0BAA0B;IACzC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,mBAAmB,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,4BAA4B;IAC3C,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,IAAI,CAAC;IACX,IAAI,CAAC,EAAE,IAAI,CAAC;CACb;AAED,UAAU,yBAAyB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAaD,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,kBAAkB,CAAC,CA8G7B;AAED,wBAAsB,8BAA8B,CAClD,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,0BAA0B,CAAC,CA6BrC;AAED,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,kBAAkB,GAAG,UAAU,EAAE,CAqBjF"}