@devflow-tools/benchmark 0.17.7 → 0.18.0

package/dist/unified-runtime-observation.d.ts

@@ -0,0 +1,101 @@
+import type { RetrievalExecutionStatus } from '@devflow-tools/sdk';
+import type { UnifiedChannel, UnifiedRuntimeCorpus } from './unified-runtime-corpus.js';
+export interface UnifiedRuntimeIdentityV2 {
+    packageVersion: string;
+    pluginRegistryHash: string;
+    runtimeSourceHash: string;
+    runnerVersion: string;
+}
+export interface UnifiedRuntimeSuiteIdentityV1 {
+    deterministicSafety: {
+        runtimeSourceHash: string;
+        runnerVersion: string;
+    };
+    semanticCapability: {
+        status: 'executed' | 'unavailable';
+        resolver: 'embedding' | 'host_sampling';
+        profileHash?: string;
+        reasonCode?: string;
+    };
+    retrievalExecution: {
+        status: 'executed';
+        source: 'fixture_seeded' | 'real_project';
+        datasetIds: string[];
+    };
+    candidateGeneration: {
+        status: 'executed';
+        source: 'runtime_index';
+        bundleId: string;
+        projectSnapshotId: string;
+        corpusHash: string;
+        labelHashes: Record<UnifiedChannel, string>;
+        channelSnapshotHashes: Record<UnifiedChannel, string>;
+        bundleProfilesHash: string;
+        readerProfileHash: string;
+    } | {
+        status: 'unavailable';
+        source: 'runtime_index';
+        reasonCode: string;
+    };
+    externalReplay: {
+        status: 'executed' | 'unavailable';
+        replayId?: string;
+        host?: string;
+        modelProfileHash?: string;
+        reasonCode?: string;
+    };
+}
+export interface UnifiedSemanticObservationV4 {
+    caseId: string;
+    datasetId: string;
+    runtimeContextHash: string;
+    channelContextHashes: Record<UnifiedChannel, string>;
+    fieldRefs: string[];
+    abstain: boolean;
+    mutationAuthorized: boolean;
+    domainAction?: {
+        tool?: string;
+        disposition: 'selected' | 'conditional_proposal' | 'abstained';
+        reasonCode: string;
+    };
+    terminalFalseSuccess: boolean;
+    latencyMs: number;
+}
+export interface UnifiedRetrievalObservationV4 {
+    caseId: string;
+    datasetId: string;
+    channel: UnifiedChannel;
+    runtimeContextHash: string;
+    planRequirement: 'required' | 'optional' | 'prohibited';
+    executionStatus: RetrievalExecutionStatus;
+    executionReasonCode?: string;
+    candidateEvidenceIds: string[];
+    eligibilityRejectedEvidenceIds: string[];
+    rankedEvidenceIds: string[];
+    rejectedEvidenceIds: string[];
+    rejectionReasonCounts: Record<string, number>;
+    cycles: number;
+    duplicateOrNoGainStopObserved: boolean;
+    latencyMs: number;
+    rerankerBaselineIds: string[];
+    attributionSource: 'host_ledger' | 'synthetic' | 'none';
+    attributionOutcomes: Array<'consumed' | 'not_consumed' | 'unknown'>;
+    verificationOutcomes: Array<'verified' | 'unverified' | 'unknown'>;
+    sourceVersion?: string;
+    temporalCorrect?: boolean;
+    episodePersisted?: boolean;
+    terminalFalseSuccess: boolean;
+}
+export interface UnifiedRuntimeObservationV4 {
+    schemaVersion: 'devflow.unified-runtime-observation.v4';
+    capturedAt: number;
+    goldDatasetIds: UnifiedRuntimeCorpus['datasetIds'];
+    runtime: UnifiedRuntimeIdentityV2;
+    suiteIdentity: UnifiedRuntimeSuiteIdentityV1;
+    semantic: UnifiedSemanticObservationV4[];
+    context: UnifiedRetrievalObservationV4[];
+    memory: UnifiedRetrievalObservationV4[];
+    knowledge: UnifiedRetrievalObservationV4[];
+}
+export type UnifiedObservationCase = UnifiedSemanticObservationV4 | UnifiedRetrievalObservationV4;
+//# sourceMappingURL=unified-runtime-observation.d.ts.map

package/dist/unified-runtime-observation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"unified-runtime-observation.d.ts","sourceRoot":"","sources":["../src/unified-runtime-observation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAExF,MAAM,WAAW,wBAAwB;IACvC,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,6BAA6B;IAC5C,mBAAmB,EAAE;QACnB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,kBAAkB,EAAE;QAClB,MAAM,EAAE,UAAU,GAAG,aAAa,CAAC;QACnC,QAAQ,EAAE,WAAW,GAAG,eAAe,CAAC;QACxC,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,kBAAkB,EAAE;QAClB,MAAM,EAAE,UAAU,CAAC;QACnB,MAAM,EAAE,gBAAgB,GAAG,cAAc,CAAC;QAC1C,UAAU,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;IACF,mBAAmB,EAAE;QACnB,MAAM,EAAE,UAAU,CAAC;QACnB,MAAM,EAAE,eAAe,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC;QACjB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QAC5C,qBAAqB,EAAE,MAAM,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QACtD,kBAAkB,EAAE,MAAM,CAAC;QAC3B,iBAAiB,EAAE,MAAM,CAAC;KAC3B,GAAG;QACF,MAAM,EAAE,aAAa,CAAC;QACtB,MAAM,EAAE,eAAe,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,cAAc,EAAE;QACd,MAAM,EAAE,UAAU,GAAG,aAAa,CAAC;QACnC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAED,MAAM,WAAW,4BAA4B;IAC3C,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IACrD,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,YAAY,CAAC,EAAE;QACb,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,UAAU,GAAG,sBAAsB,GAAG,WAAW,CAAC;QAC/D,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,oBAAoB,EAAE,OAAO,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,6BAA6B;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,cAAc,CAAC;IACxB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,UAAU,GAAG,UAAU,GAAG,YAAY,CAAC;IACxD,eAAe,EAAE,wBAAwB,CAAC;IAC1C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,EAAE,CAAC;IAC/B,8BAA8B,EAAE,MAAM,EAAE,CAAC;IACzC,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,qBAAqB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9C,MAAM,EAAE,MAAM,CAAC;IACf,6BAA6B,EAAE,OAAO,CAAC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,iBAAiB,EAAE,aAAa,GAAG,WAAW,GAAG,MAAM,CAAC;IACxD,mBAAmB,EAAE,KAAK,CAAC,UAAU,GAAG,cAAc,GAAG,SAAS,CAAC,CAAC;IACpE,oBAAoB,EAAE,KAAK,CAAC,UAAU,GAAG,YAAY,GAAG,SAAS,CAAC,CAAC;IACnE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,oBAAoB,EAAE,OAAO,CAAC;CAC/B;AAED,MAAM,WAAW,2BAA2B;IAC1C,aAAa,EAAE,wCAAwC,CAAC;IACxD,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,oBAAoB,CAAC,YAAY,CAAC,CAAC;IACnD,OAAO,EAAE,wBAAwB,CAAC;IAClC,aAAa,EAAE,6BAA6B,CAAC;IAC7C,QAAQ,EAAE,4BAA4B,EAAE,CAAC;IACzC,OAAO,EAAE,6BAA6B,EAAE,CAAC;IACzC,MAAM,EAAE,6BAA6B,EAAE,CAAC;IACxC,SAAS,EAAE,6BAA6B,EAAE,CAAC;CAC5C;AAED,MAAM,MAAM,sBAAsB,GAAG,4BAA4B,GAAG,6BAA6B,CAAC"}

package/dist/unified-runtime-observation.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=unified-runtime-observation.js.map

package/dist/unified-runtime-observation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"unified-runtime-observation.js","sourceRoot":"","sources":["../src/unified-runtime-observation.ts"],"names":[],"mappings":""}

package/dist/unified-runtime-runner.d.ts

@@ -0,0 +1,73 @@
+import { createRuntimeActivationSnapshotV1, type EvidenceCandidateV1, type EvidenceProducerResult, type EvidenceReceiptV1, type KnowledgeEvidenceProducerPort, type MemoryEvidenceProducerPort, type RetrievalTaskRuntimePort, type SemanticExecutionContext } from '@devflow-tools/sdk';
+import type { CodeEvidenceProducerPort } from '@devflow-tools/sdk';
+import type { UnifiedRetrievalCase, UnifiedRuntimeCorpus, UnifiedSemanticCase } from './unified-runtime-corpus.js';
+import type { UnifiedRetrievalObservationV4, UnifiedRuntimeIdentityV2, UnifiedRuntimeObservationV4, UnifiedRuntimeSuiteIdentityV1 } from './unified-runtime-observation.js';
+export interface UnifiedRuntimeRunnerOptions {
+    projectRoot?: string;
+    semanticExecutor?: (item: UnifiedSemanticCase, activation: ReturnType<typeof createRuntimeActivationSnapshotV1>) => Promise<SemanticExecutionContext>;
+    retrievalExecutor?: (item: UnifiedRetrievalCase, channel: 'context' | 'memory' | 'knowledge') => Promise<UnifiedRetrievalObservationV4>;
+    runtime?: UnifiedRuntimeIdentityV2;
+    suiteIdentity?: UnifiedRuntimeSuiteIdentityV1;
+    observer?: (event: {
+        stage: 'semantic' | 'context' | 'memory' | 'knowledge';
+        completed: number;
+        total: number;
+        elapsedMs: number;
+    }) => void;
+}
+export declare function runUnifiedRuntimeGold(corpus: UnifiedRuntimeCorpus, options?: UnifiedRuntimeRunnerOptions): Promise<UnifiedRuntimeObservationV4>;
+export declare function buildBenchmarkRetrievalContext(input: {
+    caseId: string;
+    prompt: string;
+    identity: ReturnType<typeof identityFor>;
+    channel: 'context' | 'memory' | 'knowledge';
+    exclusive?: boolean;
+}): SemanticExecutionContext;
+export declare function restrictBenchmarkRetrievalChannel(context: SemanticExecutionContext, channel: 'context' | 'memory' | 'knowledge', query: string): SemanticExecutionContext;
+export declare function stageCapture(): {
+    candidates: EvidenceCandidateV1[];
+    outputs: Array<{
+        result: EvidenceProducerResult;
+        vectorCandidateRefs?: Array<{
+            ref: string;
+            rank: number;
+        }>;
+    }>;
+};
+export declare function captureProducer<T extends CodeEvidenceProducerPort | MemoryEvidenceProducerPort | KnowledgeEvidenceProducerPort>(producer: T, stage: {
+    candidates: EvidenceCandidateV1[];
+    outputs: Array<{
+        result: EvidenceProducerResult;
+        vectorCandidateRefs?: Array<{
+            ref: string;
+            rank: number;
+        }>;
+    }>;
+}): T;
+export declare function retrievalStages(candidates: EvidenceCandidateV1[], rejected: Array<{
+    ref: string;
+    reason: string;
+}>, channel: 'context' | 'memory' | 'knowledge'): Pick<UnifiedRetrievalObservationV4, 'candidateEvidenceIds' | 'eligibilityRejectedEvidenceIds' | 'rejectedEvidenceIds' | 'rejectionReasonCounts'> & {
+    fusionBaselineIds: string[];
+};
+export declare function refsForRetrievalChannel(evidence: Array<{
+    stableRef: string;
+    channel: 'code' | 'memory' | 'knowledge';
+}>, channel: 'context' | 'memory' | 'knowledge'): string[];
+export declare function rankedEvidenceRefs(receipt: Pick<EvidenceReceiptV1, 'evidenceReceipts'>, channel: 'context' | 'memory' | 'knowledge'): string[];
+export declare function noOpTaskRuntime(): RetrievalTaskRuntimePort;
+export declare function identityFor(caseId: string, host: string, projectRoot?: string, projectId?: string): {
+    projectRoot: string;
+    projectId: string;
+    hostId: string;
+    sessionId: string;
+    turnId: string;
+    requestId: string;
+    executionId: string;
+};
+export declare function runtimeIdentity(root: string, sourceRoot?: string): UnifiedRuntimeIdentityV2;
+export declare function retrievalExecutionSource(input: {
+    retrievalExecutor: boolean;
+    enginesAvailable: boolean;
+}): UnifiedRuntimeSuiteIdentityV1['retrievalExecution']['source'];
+//# sourceMappingURL=unified-runtime-runner.d.ts.map

package/dist/unified-runtime-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"unified-runtime-runner.d.ts","sourceRoot":"","sources":["../src/unified-runtime-runner.ts"],"names":[],"mappings":"AAQA,OAAO,EAEL,iCAAiC,EAGjC,KAAK,mBAAmB,EACxB,KAAK,sBAAsB,EAC3B,KAAK,iBAAiB,EAEtB,KAAK,6BAA6B,EAClC,KAAK,0BAA0B,EAE/B,KAAK,wBAAwB,EAC7B,KAAK,wBAAwB,EAC9B,MAAM,oBAAoB,CAAC;AAa5B,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,KAAK,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AACnH,OAAO,KAAK,EACV,6BAA6B,EAC7B,wBAAwB,EACxB,2BAA2B,EAC3B,6BAA6B,EAE9B,MAAM,kCAAkC,CAAC;AAE1C,MAAM,WAAW,2BAA2B;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,CAAC,IAAI,EAAE,mBAAmB,EAAE,UAAU,EAAE,UAAU,CAAC,OAAO,iCAAiC,CAAC,KAAK,OAAO,CAAC,wBAAwB,CAAC,CAAC;IACtJ,iBAAiB,CAAC,EAAE,CAAC,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,SAAS,GAAG,QAAQ,GAAG,WAAW,KAAK,OAAO,CAAC,6BAA6B,CAAC,CAAC;IACxI,OAAO,CAAC,EAAE,wBAAwB,CAAC;IACnC,aAAa,CAAC,EAAE,6BAA6B,CAAC;IAC9C,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,UAAU,GAAG,SAAS,GAAG,QAAQ,GAAG,WAAW,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CAC7I;AAID,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,oBAAoB,EAC5B,OAAO,GAAE,2BAAgC,GACxC,OAAO,CAAC,2BAA2B,CAAC,CAqEtC;AAkID,wBAAgB,8BAA8B,CAAC,KAAK,EAAE;IACpD,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC;IACzC,OAAO,EAAE,SAAS,GAAG,QAAQ,GAAG,WAAW,CAAC;IAC5C,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,GAAG,wBAAwB,CAyC3B;AAED,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,wBAAwB,EACjC,OAAO,EAAE,SAAS,GAAG,QAAQ,GAAG,WAAW,EAC3C,KAAK,EAAE,MAAM,GACZ,wBAAwB,CA4D1B;AAuID,wBAAgB,YAAY,IAAI;IAC9B,UAAU,EAAE,mBAAmB,EAAE,CAAC;IAClC,OAAO,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,sBAAsB,CAAC;QAAC,mBAAmB,CAAC,EAAE,KAAK,CAAC;YAAE,GAAG,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC,CAAC;CAChH,CAEA;AAED,wBAAgB,eAAe,CAAC,CAAC,SAAS,wBAAwB,GAAG,0BAA0B,GAAG,6BAA6B,EAC7H,QAAQ,EAAE,CAAC,EACX,KAAK,EAAE;IACL,UAAU,EAAE,mBAAmB,EAAE,CAAC;IAClC,OAAO,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,sBAAsB,CAAC;QAAC,mBAAmB,CAAC,EAAE,KAAK,CAAC;YAAE,GAAG,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC,CAAC;CAChH,GACA,CAAC,CAaH;AAED,wBAAgB,eAAe,CAC7B,UAAU,EAAE,mBAAmB,EAAE,EACjC,QAAQ,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,EAChD,OAAO,EAAE,SAAS,GAAG,QAAQ,GAAG,WAAW,GAC1C,IAAI,CAAC,6BAA6B,EACnC,sBAAsB,GAAG,gCAAgC,GAAG,qBAAqB,GAC/E,uBAAuB,CAAC,GAAG;IAC3B,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B,CAkBF;AAED,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,KAAK,CAAC;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,QAAQ,GAAG,WAAW,CAAA;CAAE,CAAC,EAChF,OAAO,EAAE,SAAS,GAAG,QAAQ,GAAG,WAAW,GAC1C,MAAM,EAAE,CAGV;AAED,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,IAAI,CAAC,iBAAiB,EAAE,kBAAkB,CAAC,EACpD,OAAO,EAAE,SAAS,GAAG,QAAQ,GAAG,WAAW,GAC1C,MAAM,EAAE,CAOV;AA8CD,wBAAgB,eAAe,IAAI,wBAAwB,CAE1D;AAoBD,wBAAgB,WAAW,CACzB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,WAAW,SAAe,EAC1B,SAAS,SAA8B;;;;;;;;EAGxC;AAMD,wBAAgB,eAAe,CAC7B,IAAI,EAAE,MAAM,EACZ,UAAU,SAA2C,GACpD,wBAAwB,CAmB1B;AAgED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE;IAC9C,iBAAiB,EAAE,OAAO,CAAC;IAC3B,gBAAgB,EAAE,OAAO,CAAC;CAC3B,GAAG,6BAA6B,CAAC,oBAAoB,CAAC,CAAC,QAAQ,CAAC,CAEhE"}