agentic-qe 3.14.0 → 3.14.2

package/dist/learning/structural-retrieval/evaluate.js

@@ -0,0 +1,124 @@
+import { verifyStructuralRetrievalCorpus } from './corpus.js';
+function queryScores(query, result, k) {
+    const relevant = new Set(query.relevantPatternIds);
+    const rank = result.candidates.findIndex(candidate => relevant.has(candidate.patternId));
+    const topKHits = result.candidates.slice(0, k).filter(candidate => relevant.has(candidate.patternId)).length;
+    const idealCount = Math.min(k, relevant.size);
+    const dcg = result.candidates.slice(0, k).reduce((sum, candidate, index) => sum + (relevant.has(candidate.patternId) ? 1 / Math.log2(index + 2) : 0), 0);
+    const idcg = Array.from({ length: idealCount }, (_, index) => 1 / Math.log2(index + 2)).reduce((a, b) => a + b, 0);
+    const relevantScore = rank >= 0 ? result.candidates[rank].score : 0;
+    const bestWrong = Math.max(0, ...result.candidates.filter(c => !relevant.has(c.patternId)).map(c => c.score));
+    return {
+        recall: relevant.size === 0 ? 0 : topKHits / relevant.size,
+        hit: rank === 0 ? 1 : 0,
+        reciprocalRank: rank < 0 ? 0 : 1 / (rank + 1),
+        ndcg: idcg === 0 ? 0 : dcg / idcg,
+        margin: relevantScore - bestWrong,
+    };
+}
+function mean(values) {
+    return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+function estimate(values, seed) {
+    if (values.length === 0)
+        return { value: 0, confidenceInterval: { lower: 0, upper: 0, level: 0.95 } };
+    let state = seed >>> 0;
+    const random = () => ((state = (1664525 * state + 1013904223) >>> 0) / 0x100000000);
+    const bootstraps = Array.from({ length: 1000 }, () => mean(Array.from({ length: values.length }, () => values[Math.floor(random() * values.length)]))).sort((a, b) => a - b);
+    return {
+        value: mean(values),
+        confidenceInterval: {
+            lower: bootstraps[Math.floor(bootstraps.length * 0.025)],
+            upper: bootstraps[Math.floor(bootstraps.length * 0.975)],
+            level: 0.95,
+        },
+    };
+}
+function metricSet(scores) {
+    return {
+        support: scores.length,
+        recallAtK: estimate(scores.map(s => s.recall), 11),
+        hitAt1: estimate(scores.map(s => s.hit), 13),
+        mrr: estimate(scores.map(s => s.reciprocalRank), 17),
+        ndcg: estimate(scores.map(s => s.ndcg), 19),
+        top1Margin: estimate(scores.map(s => s.margin), 23),
+    };
+}
+export function diagnoseRetrieval(query, result, k) {
+    const relevant = new Set(query.relevantPatternIds);
+    const relevantCandidates = result.candidates.slice(0, k).filter(c => relevant.has(c.patternId));
+    if (relevantCandidates.length === 0)
+        return 'not_retrieved';
+    if (!relevant.has(result.candidates[0]?.patternId ?? ''))
+        return 'retrieved_misranked';
+    const applied = result.appliedPatternIds ?? [];
+    if (applied.length === 0 || !applied.some(id => relevant.has(id)))
+        return 'ranked_not_used';
+    if (result.outcomeImproved !== true)
+        return 'used_no_benefit';
+    return 'success';
+}
+export function evaluateStructuralRetrieval(corpus, results, options) {
+    const k = options.k ?? 10;
+    if (!Number.isInteger(k) || k < 1)
+        throw new Error('k must be a positive integer');
+    if (!verifyStructuralRetrievalCorpus(corpus))
+        throw new Error('corpus lineage hash mismatch');
+    if (results.length !== corpus.queries.length) {
+        throw new Error('results must contain exactly one entry for every corpus query');
+    }
+    const byQuery = new Map(results.map(result => [result.queryId, result]));
+    if (byQuery.size !== corpus.queries.length || corpus.queries.some(query => !byQuery.has(query.id))) {
+        throw new Error('results must contain exactly one entry for every corpus query');
+    }
+    const knownPatterns = new Set(corpus.patterns.map(pattern => pattern.id));
+    for (const result of results) {
+        const candidateIds = new Set();
+        for (const candidate of result.candidates) {
+            if (!knownPatterns.has(candidate.patternId))
+                throw new Error(`unknown candidate pattern: ${candidate.patternId}`);
+            if (!Number.isFinite(candidate.score))
+                throw new Error(`candidate score must be finite: ${candidate.patternId}`);
+            if (candidateIds.has(candidate.patternId))
+                throw new Error(`duplicate candidate pattern: ${candidate.patternId}`);
+            candidateIds.add(candidate.patternId);
+        }
+        for (const appliedId of result.appliedPatternIds ?? []) {
+            if (!candidateIds.has(appliedId))
+                throw new Error(`applied pattern was not a candidate: ${appliedId}`);
+        }
+    }
+    const rows = corpus.queries.map(query => {
+        const result = byQuery.get(query.id);
+        return { query, result, scores: queryScores(query, result, k) };
+    });
+    const diagnosticNames = [
+        'success', 'not_retrieved', 'retrieved_misranked', 'ranked_not_used', 'used_no_benefit',
+    ];
+    const diagnostics = Object.fromEntries(diagnosticNames.map(name => [name, 0]));
+    for (const row of rows)
+        diagnostics[diagnoseRetrieval(row.query, row.result, k)] += 1;
+    const relationViolations = rows.reduce((count, { query, result }) => {
+        const ranks = new Map(result.candidates.map((candidate, index) => [candidate.patternId, index]));
+        return count + query.mustNotRankAbove.filter(relation => {
+            const low = ranks.get(relation.lowerPatternId);
+            const high = ranks.get(relation.higherPatternId);
+            return low !== undefined && (high === undefined || low < high);
+        }).length;
+    }, 0);
+    const slices = {};
+    for (const transformation of new Set(corpus.queries.map(query => query.transformation))) {
+        slices[transformation] = metricSet(rows.filter(row => row.query.transformation === transformation).map(row => row.scores));
+    }
+    return {
+        corpusRevision: corpus.corpusRevision,
+        corpusLineageHash: corpus.lineageHash,
+        rankerVersion: options.rankerVersion,
+        k,
+        overall: metricSet(rows.map(row => row.scores)),
+        slices,
+        diagnostics,
+        relationViolations,
+    };
+}
+//# sourceMappingURL=evaluate.js.map

package/dist/learning/structural-retrieval/index.d.ts

@@ -0,0 +1,6 @@
+export { createStructuralRetrievalCorpus, verifyStructuralRetrievalCorpus, STRUCTURAL_RETRIEVAL_TRANSFORMATIONS, } from './corpus.js';
+export { diagnoseRetrieval, evaluateStructuralRetrieval } from './evaluate.js';
+export { createRetrievalReceipt } from './receipt.js';
+export type * from './types.js';
+export type { CreateRetrievalReceiptInput } from './receipt.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/learning/structural-retrieval/index.js

@@ -0,0 +1,4 @@
+export { createStructuralRetrievalCorpus, verifyStructuralRetrievalCorpus, STRUCTURAL_RETRIEVAL_TRANSFORMATIONS, } from './corpus.js';
+export { diagnoseRetrieval, evaluateStructuralRetrieval } from './evaluate.js';
+export { createRetrievalReceipt } from './receipt.js';
+//# sourceMappingURL=index.js.map

package/dist/learning/structural-retrieval/receipt.d.ts

@@ -0,0 +1,14 @@
+import type { RankedCandidate, RetrievalReceipt } from './types.js';
+export interface CreateRetrievalReceiptInput {
+    revision: string;
+    queryId: string;
+    queryRepresentationVersion: string;
+    embeddingSpaceId: string;
+    rankerVersion: string;
+    candidates: RankedCandidate[];
+    appliedPatternIds?: string[];
+    outcomeRef?: string | null;
+    outcomeImproved?: boolean | null;
+}
+export declare function createRetrievalReceipt(input: CreateRetrievalReceiptInput): RetrievalReceipt;
+//# sourceMappingURL=receipt.d.ts.map

package/dist/learning/structural-retrieval/receipt.js

@@ -0,0 +1,36 @@
+import { createHash } from 'node:crypto';
+function deepFreeze(value) {
+    if (value && typeof value === 'object' && !Object.isFrozen(value)) {
+        Object.freeze(value);
+        for (const child of Object.values(value))
+            deepFreeze(child);
+    }
+    return value;
+}
+export function createRetrievalReceipt(input) {
+    if (!input.revision || !input.queryId || !input.queryRepresentationVersion || !input.embeddingSpaceId || !input.rankerVersion) {
+        throw new Error('retrieval receipt provenance fields must be non-empty');
+    }
+    const candidateIds = new Set(input.candidates.map(candidate => candidate.patternId));
+    if (candidateIds.size !== input.candidates.length || input.candidates.some(candidate => !candidate.patternId || !Number.isFinite(candidate.score))) {
+        throw new Error('retrieval receipt candidates must have an id and finite score');
+    }
+    if ((input.appliedPatternIds ?? []).some(id => !candidateIds.has(id))) {
+        throw new Error('retrieval receipt applied patterns must be present in candidates');
+    }
+    const payload = {
+        schemaVersion: 'aqe-retrieval-receipt/v1',
+        revision: input.revision,
+        queryId: input.queryId,
+        queryRepresentationVersion: input.queryRepresentationVersion,
+        embeddingSpaceId: input.embeddingSpaceId,
+        rankerVersion: input.rankerVersion,
+        candidates: input.candidates.map(candidate => ({ ...candidate })),
+        appliedPatternIds: [...(input.appliedPatternIds ?? [])],
+        outcomeRef: input.outcomeRef ?? null,
+        outcomeImproved: input.outcomeImproved ?? null,
+    };
+    const receiptId = createHash('sha256').update(JSON.stringify(payload)).digest('hex');
+    return deepFreeze({ ...payload, receiptId });
+}
+//# sourceMappingURL=receipt.js.map

package/dist/learning/structural-retrieval/types.d.ts

@@ -0,0 +1,83 @@
+export type StructuralTransformation = 'rename' | 'framework-transfer' | 'reordered-steps' | 'distractor-terminology' | 'cross-domain-transfer';
+export type CorpusSplit = 'train' | 'calibration' | 'audit';
+export interface StructuralPattern {
+    id: string;
+    familyId: string;
+    framework: string;
+    domain: string;
+    failureMechanism: string;
+    invariant: string;
+    actions: string[];
+    oracle: string;
+    text: string;
+}
+export interface StructuralRetrievalQuery {
+    id: string;
+    familyId: string;
+    split: CorpusSplit;
+    transformation: StructuralTransformation;
+    text: string;
+    relevantPatternIds: string[];
+    mustNotRankAbove: Array<{
+        lowerPatternId: string;
+        higherPatternId: string;
+    }>;
+}
+export interface StructuralRetrievalCorpus {
+    schemaVersion: 'aqe-structural-retrieval/v1';
+    corpusRevision: string;
+    lineageHash: string;
+    patterns: StructuralPattern[];
+    queries: StructuralRetrievalQuery[];
+}
+export interface RankedCandidate {
+    patternId: string;
+    score: number;
+}
+export type RetrievalDiagnostic = 'success' | 'not_retrieved' | 'retrieved_misranked' | 'ranked_not_used' | 'used_no_benefit';
+export interface QueryRetrievalResult {
+    queryId: string;
+    candidates: RankedCandidate[];
+    appliedPatternIds?: string[];
+    outcomeImproved?: boolean;
+}
+export interface MetricEstimate {
+    value: number;
+    confidenceInterval: {
+        lower: number;
+        upper: number;
+        level: 0.95;
+    };
+}
+export interface RetrievalMetricSet {
+    support: number;
+    recallAtK: MetricEstimate;
+    hitAt1: MetricEstimate;
+    mrr: MetricEstimate;
+    ndcg: MetricEstimate;
+    top1Margin: MetricEstimate;
+}
+export interface StructuralRetrievalReport {
+    corpusRevision: string;
+    corpusLineageHash: string;
+    rankerVersion: string;
+    k: number;
+    overall: RetrievalMetricSet;
+    slices: Partial<Record<StructuralTransformation, RetrievalMetricSet>>;
+    diagnostics: Record<RetrievalDiagnostic, number>;
+    relationViolations: number;
+}
+export interface RetrievalReceipt {
+    schemaVersion: 'aqe-retrieval-receipt/v1';
+    receiptId: string;
+    revision: string;
+    queryId: string;
+    queryRepresentationVersion: string;
+    embeddingSpaceId: string;
+    rankerVersion: string;
+    candidates: RankedCandidate[];
+    appliedPatternIds: string[];
+    outcomeRef: string | null;
+    outcomeImproved: boolean | null;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/learning/structural-retrieval/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map