@devflow-tools/benchmark 0.17.8 → 0.18.1

package/dist/evaluation-bundle.d.ts

@@ -0,0 +1,55 @@
+import { type EvaluationBundleAssetV1, type EvaluationBundleManifestV1, type EvaluationChannel, type EmbeddingProfileV1, type ProjectSnapshotIdentityV1 } from '@devflow-tools/sdk';
+export interface EvaluationLabelCaseV1 {
+    caseId: string;
+    goldSets: string[][];
+    forbiddenRefs: string[];
+    reviewer: string;
+    labelledAt: string;
+    sourceEvidence: string[];
+}
+export interface EvaluationLabelSetV1 {
+    schemaVersion: 'devflow.evaluation-labels.v1';
+    projectSnapshotId: string;
+    channel: EvaluationChannel;
+    channelSnapshotHash: string;
+    datasetHash: string;
+    labels: EvaluationLabelCaseV1[];
+}
+export interface EvaluationCorpusV1 {
+    schemaVersion: 'devflow.evaluation-corpus.v1';
+    cases: Array<{
+        caseId: string;
+        channel: EvaluationChannel;
+        [key: string]: unknown;
+    }>;
+}
+export interface EvaluationProfilesV1 {
+    context: string;
+    memory: string;
+    knowledge: string;
+    embeddingProfiles: {
+        code_retrieval: EmbeddingProfileV1;
+    };
+}
+export type EvaluationBundleValidation = {
+    status: 'valid';
+    manifest: EvaluationBundleManifestV1;
+} | {
+    status: 'invalid';
+    reasonCode: 'manifest_missing' | 'manifest_invalid' | 'asset_hash_mismatch' | 'bundle_identity_mismatch' | 'channel_snapshot_mismatch';
+    asset?: string;
+};
+export declare function readProjectSnapshotIdentity(projectRoot: string): ProjectSnapshotIdentityV1;
+export declare function assertCleanGitProject(projectRoot: string): void;
+export declare function logicalDatabaseHash(databasePath: string): string;
+export declare function collectBundleAssets(bundleRoot: string): EvaluationBundleAssetV1[];
+export declare function extractEvaluationStableRefs(input: {
+    channel: EvaluationChannel;
+    databasePath: string;
+    projectRoot: string;
+    projectSnapshotId: string;
+    channelSnapshotHash: string;
+}): string[];
+export declare function validateEvaluationBundle(bundleRoot: string): EvaluationBundleValidation;
+export declare function vectorAssetIdentity(bundleRoot: string): EvaluationBundleAssetV1[];
+//# sourceMappingURL=evaluation-bundle.d.ts.map

package/dist/evaluation-bundle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluation-bundle.d.ts","sourceRoot":"","sources":["../src/evaluation-bundle.ts"],"names":[],"mappings":"AAMA,OAAO,EAYL,KAAK,uBAAuB,EAC5B,KAAK,0BAA0B,EAC/B,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,yBAAyB,EAC/B,MAAM,oBAAoB,CAAC;AAE5B,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAC;IACrB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,oBAAoB;IACnC,aAAa,EAAE,8BAA8B,CAAC;IAC9C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,qBAAqB,EAAE,CAAC;CACjC;AAED,MAAM,WAAW,kBAAkB;IACjC,aAAa,EAAE,8BAA8B,CAAC;IAC9C,KAAK,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,iBAAiB,CAAC;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC,CAAC;CACtF;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE;QAAE,cAAc,EAAE,kBAAkB,CAAA;KAAE,CAAC;CAC3D;AAED,MAAM,MAAM,0BAA0B,GAAG;IACvC,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,0BAA0B,CAAC;CACtC,GAAG;IACF,MAAM,EAAE,SAAS,CAAC;IAClB,UAAU,EAAE,kBAAkB,GAAG,kBAAkB,GAAG,qBAAqB,GACvE,0BAA0B,GAAG,2BAA2B,CAAC;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,wBAAgB,2BAA2B,CAAC,WAAW,EAAE,MAAM,GAAG,yBAAyB,CAY1F;AAED,wBAAgB,qBAAqB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAI/D;AAED,wBAAgB,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAmBhE;AAED,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,uBAAuB,EAAE,CAYjF;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE;IACjD,OAAO,EAAE,iBAAiB,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;CAC7B,GAAG,MAAM,EAAE,CAwDX;AAED,wBAAgB,wBAAwB,CAAC,UAAU,EAAE,MAAM,GAAG,0BAA0B,CAqEvF;AA4BD,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,uBAAuB,EAAE,CAQjF"}

package/dist/evaluation-bundle.js

@@ -0,0 +1,272 @@
+import { execFileSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { existsSync, lstatSync, readFileSync, readlinkSync, readdirSync } from 'node:fs';
+import { join, relative, resolve } from 'node:path';
+import Database from 'better-sqlite3';
+import * as sqliteVec from 'sqlite-vec';
+import { EVALUATION_BUNDLE_SCHEMA, assetTreeHash, channelSnapshotHash, createEmbeddingProfile, datasetHash, evaluationBundleId, evaluationContextStableRef, evaluationKnowledgeStableRef, evaluationMemoryStableRef, projectSnapshotId, stableRuntimeJson, } from '@devflow-tools/sdk';
+export function readProjectSnapshotIdentity(projectRoot) {
+    const snapshotPath = join(projectRoot, '.devflow/source-snapshot.json');
+    if (existsSync(snapshotPath))
+        return JSON.parse(readFileSync(snapshotPath, 'utf8'));
+    const commit = git(projectRoot, ['rev-parse', 'HEAD']).trim();
+    const tree = git(projectRoot, ['rev-parse', 'HEAD^{tree}']).trim();
+    const output = execFileSync('git', ['-C', projectRoot, 'ls-tree', '-r', '-z', '--full-tree', 'HEAD']);
+    const trackedFiles = output.toString('utf8').split('\0').filter(Boolean).map(entry => {
+        const match = /^(\d+)\s+(?:blob|commit)\s+([a-f0-9]+)\t(.+)$/u.exec(entry);
+        if (!match)
+            throw new Error(`EVALUATION_GIT_TREE_ENTRY_INVALID:${entry}`);
+        return { mode: match[1], blob: match[2], path: match[3] };
+    });
+    return { commit, tree, trackedFiles };
+}
+export function assertCleanGitProject(projectRoot) {
+    if (existsSync(join(projectRoot, '.devflow/source-snapshot.json')))
+        return;
+    const status = git(projectRoot, ['status', '--porcelain', '--untracked-files=all']).trim();
+    if (status)
+        throw new Error(`EVALUATION_PROJECT_NOT_CLEAN\n${status}`);
+}
+export function logicalDatabaseHash(databasePath) {
+    const database = new Database(databasePath, { readonly: true, fileMustExist: true });
+    try {
+        sqliteVec.load(database);
+        const tables = database.prepare(`SELECT name, sql FROM sqlite_master
+      WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name`).all();
+        const body = tables.map(table => {
+            const rows = database.prepare(`SELECT * FROM ${quoteIdentifier(table.name)}`).all()
+                .map(row => typedSqliteValue(row))
+                .sort((left, right) => stableRuntimeJson(left).localeCompare(stableRuntimeJson(right)));
+            return { name: table.name, sql: table.sql ?? '', rows };
+        });
+        return channelSnapshotHash({ userVersion: database.pragma('user_version', { simple: true }), tables: body });
+    }
+    finally {
+        database.close();
+    }
+}
+export function collectBundleAssets(bundleRoot) {
+    return walkAssets(bundleRoot).filter(path => path !== 'manifest.json').map((path) => {
+        const absolute = join(bundleRoot, path);
+        const stat = lstatSync(absolute);
+        const content = stat.isSymbolicLink() ? Buffer.from(readlinkSync(absolute)) : readFileSync(absolute);
+        return {
+            path,
+            kind: stat.isSymbolicLink() ? 'symlink' : 'file',
+            sha256: createHash('sha256').update(content).digest('hex'),
+            size: content.length,
+        };
+    }).sort((left, right) => left.path.localeCompare(right.path));
+}
+export function extractEvaluationStableRefs(input) {
+    const database = new Database(input.databasePath, { readonly: true, fileMustExist: true });
+    try {
+        if (input.channel === 'context') {
+            requireColumns(database, 'nodes', ['id', 'file_path', 'start_line', 'end_line']);
+            const rows = database.prepare(`SELECT id, file_path, start_line, end_line FROM nodes ORDER BY id`).all();
+            return unique(rows.map(row => evaluationContextStableRef({
+                projectSnapshotId: input.projectSnapshotId,
+                normalizedPath: projectRelativePath(input.projectRoot, row.file_path),
+                symbolId: row.id,
+                range: [row.start_line ?? 0, row.end_line ?? 0],
+            })));
+        }
+        if (input.channel === 'memory') {
+            requireColumns(database, 'memories', ['id', 'version']);
+            const facts = database.prepare('SELECT id, version FROM memories ORDER BY id').all().map(row => evaluationMemoryStableRef({
+                channelSnapshotHash: input.channelSnapshotHash,
+                kind: 'fact', immutableId: row.id, version: row.version,
+            }));
+            const episodes = tableExists(database, 'memory_episodes_v1')
+                ? database.prepare(`SELECT episode_id, episode_hash FROM memory_episodes_v1
+            ORDER BY episode_id`).all().map(row => evaluationMemoryStableRef({
+                    channelSnapshotHash: input.channelSnapshotHash,
+                    kind: 'episode', immutableId: row.episode_id, version: row.episode_hash,
+                }))
+                : [];
+            return unique([...facts, ...episodes]);
+        }
+        requireColumns(database, 'knowledge_docs', ['id', 'source', 'version', 'status']);
+        requireColumns(database, 'knowledge_chunks', ['id', 'doc_id', 'content_hash']);
+        const rows = database.prepare(`SELECT c.id, c.content_hash, d.source, d.version
+      FROM knowledge_chunks c JOIN knowledge_docs d ON d.id = c.doc_id
+      WHERE d.status = 'active' ORDER BY c.id`).all();
+        return unique(rows.map(row => evaluationKnowledgeStableRef({
+            sourceId: row.source,
+            sourceVersion: required(row.version, 'EVALUATION_KNOWLEDGE_VERSION_REQUIRED'),
+            contentHash: row.content_hash,
+            chunkId: row.id,
+        })));
+    }
+    finally {
+        database.close();
+    }
+}
+export function validateEvaluationBundle(bundleRoot) {
+    const manifestPath = join(bundleRoot, 'manifest.json');
+    if (!existsSync(manifestPath))
+        return { status: 'invalid', reasonCode: 'manifest_missing' };
+    let manifest;
+    let project;
+    let profiles;
+    try {
+        manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
+        project = JSON.parse(readFileSync(join(bundleRoot, 'project.json'), 'utf8'));
+        profiles = JSON.parse(readFileSync(join(bundleRoot, 'profiles.json'), 'utf8'));
+        if (manifest.schemaVersion !== EVALUATION_BUNDLE_SCHEMA
+            || typeof manifest.projectId !== 'string' || !manifest.projectId.trim()
+            || manifest.projectSnapshotId !== projectSnapshotId(project)
+            || !isEvaluationProfiles(profiles)
+            || datasetHash(profiles) !== manifest.profilesHash) {
+            return { status: 'invalid', reasonCode: 'manifest_invalid' };
+        }
+    }
+    catch {
+        return { status: 'invalid', reasonCode: 'manifest_invalid' };
+    }
+    const actualAssets = collectBundleAssets(bundleRoot);
+    const expected = new Map(manifest.assets.map(asset => [asset.path, asset]));
+    for (const asset of actualAssets) {
+        const recorded = expected.get(asset.path);
+        if (!recorded || recorded.sha256 !== asset.sha256 || recorded.size !== asset.size) {
+            return { status: 'invalid', reasonCode: 'asset_hash_mismatch', asset: asset.path };
+        }
+        expected.delete(asset.path);
+    }
+    if (expected.size > 0) {
+        return { status: 'invalid', reasonCode: 'asset_hash_mismatch', asset: [...expected.keys()].sort()[0] };
+    }
+    const channelPaths = {
+        context: join(bundleRoot, 'channels/codegraph.db'),
+        contextVectors: join(bundleRoot, 'channels/context-symbols.db'),
+        memory: join(bundleRoot, 'channels/memory.db'),
+        knowledge: join(bundleRoot, 'channels/knowledge.db'),
+    };
+    for (const channel of ['context', 'memory', 'knowledge']) {
+        const databaseHash = logicalDatabaseHash(channelPaths[channel]);
+        const logicalHash = channel === 'context'
+            ? channelSnapshotHash({
+                databaseHash,
+                vectorDatabaseHash: logicalDatabaseHash(channelPaths.contextVectors),
+            })
+            : channel === 'knowledge'
+                ? channelSnapshotHash({ databaseHash, vectorAssets: vectorAssetIdentity(bundleRoot) })
+                : databaseHash;
+        if (logicalHash !== manifest.channelSnapshotHashes[channel]) {
+            return { status: 'invalid', reasonCode: 'channel_snapshot_mismatch', asset: channel };
+        }
+    }
+    const bundleId = evaluationBundleId({
+        schemaVersion: manifest.schemaVersion,
+        projectId: manifest.projectId,
+        project,
+        channels: manifest.channelSnapshotHashes,
+        corpusHash: manifest.corpusHash,
+        labelHashes: manifest.labelHashes,
+        assetTreeHash: assetTreeHash(manifest.assets),
+        builderIdentity: manifest.builderIdentity,
+        profilesHash: manifest.profilesHash,
+    });
+    return bundleId === manifest.bundleId
+        ? { status: 'valid', manifest }
+        : { status: 'invalid', reasonCode: 'bundle_identity_mismatch' };
+}
+function isEvaluationProfiles(value) {
+    if (!value || typeof value !== 'object'
+        || stableRuntimeJson(Object.keys(value).sort()) !== stableRuntimeJson([
+            'context', 'embeddingProfiles', 'knowledge', 'memory',
+        ])
+        || typeof value.context !== 'string' || !value.context.trim()
+        || typeof value.memory !== 'string' || !value.memory.trim()
+        || typeof value.knowledge !== 'string' || !value.knowledge.trim()
+        || !value.embeddingProfiles || typeof value.embeddingProfiles !== 'object'
+        || stableRuntimeJson(Object.keys(value.embeddingProfiles).sort()) !== stableRuntimeJson(['code_retrieval'])) {
+        return false;
+    }
+    const profile = value.embeddingProfiles.code_retrieval;
+    if (!profile || profile.purpose !== 'code_retrieval'
+        || stableRuntimeJson(Object.keys(profile).sort()) !== stableRuntimeJson([
+            'dimension', 'generation', 'modelId', 'modelRevision', 'normalization',
+            'profileHash', 'provider', 'purpose', 'tokenizerConfigHash',
+        ]))
+        return false;
+    const { profileHash, ...body } = profile;
+    try {
+        return createEmbeddingProfile(body).profileHash === profileHash;
+    }
+    catch {
+        return false;
+    }
+}
+export function vectorAssetIdentity(bundleRoot) {
+    const vectorRoot = join(bundleRoot, 'channels/knowledge-assets');
+    if (!existsSync(vectorRoot))
+        return [];
+    return walkAssets(vectorRoot).map((path) => {
+        const absolute = join(vectorRoot, path);
+        const content = readFileSync(absolute);
+        return { path, kind: 'file', sha256: createHash('sha256').update(content).digest('hex'), size: content.length };
+    }).sort((left, right) => left.path.localeCompare(right.path));
+}
+function typedSqliteValue(value) {
+    if (Buffer.isBuffer(value))
+        return { type: 'blob', hex: value.toString('hex') };
+    if (Array.isArray(value))
+        return value.map(typedSqliteValue);
+    if (value && typeof value === 'object') {
+        return Object.fromEntries(Object.entries(value).sort(([left], [right]) => left.localeCompare(right))
+            .map(([key, child]) => [key, typedSqliteValue(child)]));
+    }
+    if (typeof value === 'bigint')
+        return { type: 'integer', value: value.toString() };
+    return value;
+}
+function walkAssets(root, prefix = '') {
+    if (!existsSync(root))
+        return [];
+    return readdirSync(join(root, prefix), { withFileTypes: true }).flatMap(entry => {
+        const path = prefix ? `${prefix}/${entry.name}` : entry.name;
+        if (entry.isDirectory())
+            return walkAssets(root, path);
+        if (entry.isFile() || entry.isSymbolicLink())
+            return [path];
+        throw new Error(`EVALUATION_ASSET_TYPE_UNSUPPORTED:${path}`);
+    }).sort();
+}
+function git(root, args) {
+    return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' });
+}
+function quoteIdentifier(value) {
+    return `"${value.replaceAll('"', '""')}"`;
+}
+function tableExists(database, table) {
+    return Boolean(database.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`).get(table));
+}
+function requireColumns(database, table, columns) {
+    if (!tableExists(database, table))
+        throw new Error(`EVALUATION_CHANNEL_TABLE_REQUIRED:${table}`);
+    const actual = new Set(database.pragma(`table_info(${quoteIdentifier(table)})`)
+        .map(column => column.name));
+    for (const column of columns) {
+        if (!actual.has(column))
+            throw new Error(`EVALUATION_CHANNEL_COLUMN_REQUIRED:${table}.${column}`);
+    }
+}
+function projectRelativePath(projectRoot, value) {
+    const normalized = value.replaceAll('\\', '/');
+    if (!normalized.startsWith('/') && !/^[A-Za-z]:\//u.test(normalized))
+        return normalized;
+    const path = relative(resolve(projectRoot), resolve(value)).replaceAll('\\', '/');
+    if (!path || path === '..' || path.startsWith('../'))
+        throw new Error('EVALUATION_CHANNEL_PATH_OUTSIDE_SOURCE');
+    return path;
+}
+function unique(values) {
+    return [...new Set(values)].sort();
+}
+function required(value, error) {
+    if (!value?.trim())
+        throw new Error(error);
+    return value;
+}
+//# sourceMappingURL=evaluation-bundle.js.map

package/dist/evaluation-bundle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluation-bundle.js","sourceRoot":"","sources":["../src/evaluation-bundle.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACzF,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,KAAK,SAAS,MAAM,YAAY,CAAC;AACxC,OAAO,EACL,wBAAwB,EACxB,aAAa,EACb,mBAAmB,EACnB,sBAAsB,EACtB,WAAW,EACX,kBAAkB,EAClB,0BAA0B,EAC1B,4BAA4B,EAC5B,yBAAyB,EACzB,iBAAiB,EACjB,iBAAiB,GAMlB,MAAM,oBAAoB,CAAC;AA0C5B,MAAM,UAAU,2BAA2B,CAAC,WAAmB;IAC7D,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,EAAE,+BAA+B,CAAC,CAAC;IACxE,IAAI,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAA8B,CAAC;IACjH,MAAM,MAAM,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9D,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACnE,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;IACtG,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QACnF,MAAM,KAAK,GAAG,gDAAgD,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3E,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,KAAK,EAAE,CAAC,CAAC;QAC1E,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5D,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,WAAmB;IACvD,IAAI,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,+BAA+B,CAAC,CAAC;QAAE,OAAO;IAC3E,MAAM,MAAM,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,aAAa,EAAE,uBAAuB,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3F,IAAI,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,MAAM,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,YAAoB;IACtD,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACrF,IAAI,CAAC;QACH,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC;sEACkC,CAAC,CAAC,GAAG,EAGnE,CAAC;QACL,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;YAC9B,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,iBAAiB,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE;iBAChF,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;iBACjC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,aAAa,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1F,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC;QAC1D,CAAC,CAAC,CAAC;QACH,OAAO,mBAAmB,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/G,CAAC;YAAS,CAAC;QACT,QAAQ,CAAC,KAAK,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,UAAkB;IACpD,OAAO,UAAU,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAA2B,EAAE;QAC3G,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QACrG,OAAO;YACL,IAAI;YACJ,IAAI,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM;YAChD,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;YAC1D,IAAI,EAAE,OAAO,CAAC,MAAM;SACrB,CAAC;IACJ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,KAM3C;IACC,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3F,IAAI,CAAC;QACH,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAChC,cAAc,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC,IAAI,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC,CAAC;YACjF,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,mEAAmE,CAAC,CAAC,GAAG,EAKpG,CAAC;YACH,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,0BAA0B,CAAC;gBACvD,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;gBAC1C,cAAc,EAAE,mBAAmB,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,CAAC,SAAS,CAAC;gBACrE,QAAQ,EAAE,GAAG,CAAC,EAAE;gBAChB,KAAK,EAAE,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,EAAE,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC;aAChD,CAAC,CAAC,CAAC,CAAC;QACP,CAAC;QACD,IAAI,KAAK,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC/B,cAAc,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;YACxD,MAAM,KAAK,GAAI,QAAQ,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC,GAAG,EAGhF,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,yBAAyB,CAAC;gBACvC,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;gBAC9C,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO;aACxD,CAAC,CAAC,CAAC;YACJ,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,EAAE,oBAAoB,CAAC;gBAC1D,CAAC,CAAE,QAAQ,CAAC,OAAO,CAAC;gCACI,CAAC,CAAC,GAAG,EAA0D,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAC9F,yBAAyB,CAAC;oBACxB,mBAAmB,EAAE,KAAK,CAAC,mBAAmB;oBAC9C,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,GAAG,CAAC,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,YAAY;iBACxE,CAAC,CAAC;gBACP,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,MAAM,CAAC,CAAC,GAAG,KAAK,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC;QACzC,CAAC;QACD,cAAc,CAAC,QAAQ,EAAE,gBAAgB,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;QAClF,cAAc,CAAC,QAAQ,EAAE,kBAAkB,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC;QAC/E,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC;;8CAEY,CAAC,CAAC,GAAG,EAK3C,CAAC;QACL,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,4BAA4B,CAAC;YACzD,QAAQ,EAAE,GAAG,CAAC,MAAM;YACpB,aAAa,EAAE,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,uCAAuC,CAAC;YAC7E,WAAW,EAAE,GAAG,CAAC,YAAY;YAC7B,OAAO,EAAE,GAAG,CAAC,EAAE;SAChB,CAAC,CAAC,CAAC,CAAC;IACP,CAAC;YAAS,CAAC;QACT,QAAQ,CAAC,KAAK,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,UAAkB;IACzD,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;IACvD,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,kBAAkB,EAAE,CAAC;IAC5F,IAAI,QAAoC,CAAC;IACzC,IAAI,OAAkC,CAAC;IACvC,IAAI,QAA8B,CAAC;IACnC,IAAI,CAAC;QACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAA+B,CAAC;QACxF,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAA8B,CAAC;QAC1G,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,EAAE,MAAM,CAAC,CAAyB,CAAC;QACvG,IAAI,QAAQ,CAAC,aAAa,KAAK,wBAAwB;eAClD,OAAO,QAAQ,CAAC,SAAS,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,EAAE;eACpE,QAAQ,CAAC,iBAAiB,KAAK,iBAAiB,CAAC,OAAO,CAAC;eACzD,CAAC,oBAAoB,CAAC,QAAQ,CAAC;eAC/B,WAAW,CAAC,QAAQ,CAAC,KAAK,QAAQ,CAAC,YAAY,EAAE,CAAC;YACrD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,kBAAkB,EAAE,CAAC;QAC/D,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,kBAAkB,EAAE,CAAC;IAC/D,CAAC;IAED,MAAM,YAAY,GAAG,mBAAmB,CAAC,UAAU,CAAC,CAAC;IACrD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5E,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC;YAClF,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,qBAAqB,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;QACrF,CAAC;QACD,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,qBAAqB,EAAE,KAAK,EAAE,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACzG,CAAC;IAED,MAAM,YAAY,GAAG;QACnB,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE,uBAAuB,CAAC;QAClD,cAAc,EAAE,IAAI,CAAC,UAAU,EAAE,6BAA6B,CAAC;QAC/D,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,oBAAoB,CAAC;QAC9C,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,uBAAuB,CAAC;KACrD,CAAC;IACF,KAAK,MAAM,OAAO,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAU,EAAE,CAAC;QAClE,MAAM,YAAY,GAAG,mBAAmB,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;QAChE,MAAM,WAAW,GAAG,OAAO,KAAK,SAAS;YACvC,CAAC,CAAC,mBAAmB,CAAC;gBACpB,YAAY;gBACZ,kBAAkB,EAAE,mBAAmB,CAAC,YAAY,CAAC,cAAc,CAAC;aACrE,CAAC;YACF,CAAC,CAAC,OAAO,KAAK,WAAW;gBACvB,CAAC,CAAC,mBAAmB,CAAC,EAAE,YAAY,EAAE,YAAY,EAAE,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;gBACtF,CAAC,CAAC,YAAY,CAAC;QACnB,IAAI,WAAW,KAAK,QAAQ,CAAC,qBAAqB,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5D,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,2BAA2B,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QACxF,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,kBAAkB,CAAC;QAClC,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,OAAO;QACP,QAAQ,EAAE,QAAQ,CAAC,qBAAqB;QACxC,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,aAAa,EAAE,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC7C,eAAe,EAAE,QAAQ,CAAC,eAAe;QACzC,YAAY,EAAE,QAAQ,CAAC,YAAY;KACpC,CAAC,CAAC;IACH,OAAO,QAAQ,KAAK,QAAQ,CAAC,QAAQ;QACnC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE;QAC/B,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,0BAA0B,EAAE,CAAC;AACpE,CAAC;AAED,SAAS,oBAAoB,CAAC,KAA2B;IACvD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;WAClC,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,iBAAiB,CAAC;YACpE,SAAS,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ;SACtD,CAAC;WACC,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;WAC1D,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE;WACxD,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE;WAC9D,CAAC,KAAK,CAAC,iBAAiB,IAAI,OAAO,KAAK,CAAC,iBAAiB,KAAK,QAAQ;WACvE,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,iBAAiB,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC;QAC9G,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,iBAAiB,CAAC,cAAc,CAAC;IACvD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,KAAK,gBAAgB;WAC/C,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,iBAAiB,CAAC;YACtE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe;YACtE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,qBAAqB;SAC5D,CAAC;QAAE,OAAO,KAAK,CAAC;IACnB,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;IACzC,IAAI,CAAC;QACH,OAAO,sBAAsB,CAAC,IAAI,CAAC,CAAC,WAAW,KAAK,WAAW,CAAC;IAClE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,UAAkB;IACpD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,2BAA2B,CAAC,CAAC;IACjE,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,CAAC;IACvC,OAAO,UAAU,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAA2B,EAAE;QAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QACvC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;IAClH,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;IAChF,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC7D,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;aACjG,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC;IACnF,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,MAAM,GAAG,EAAE;IAC3C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;QAC9E,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;QAC7D,IAAI,KAAK,CAAC,WAAW,EAAE;YAAE,OAAO,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACvD,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,cAAc,EAAE;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5D,MAAM,IAAI,KAAK,CAAC,qCAAqC,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,GAAG,CAAC,IAAY,EAAE,IAAc;IACvC,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;AAC1E,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACpC,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC;AAC5C,CAAC;AAED,SAAS,WAAW,CAAC,QAA2B,EAAE,KAAa;IAC7D,OAAO,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,+DAA+D,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AAC/G,CAAC;AAED,SAAS,cAAc,CAAC,QAA2B,EAAE,KAAa,EAAE,OAAiB;IACnF,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,KAAK,EAAE,CAAC,CAAC;IACjG,MAAM,MAAM,GAAG,IAAI,GAAG,CAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,eAAe,CAAC,KAAK,CAAC,GAAG,CAA6B;SACzG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,KAAK,IAAI,MAAM,EAAE,CAAC,CAAC;IACpG,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,WAAmB,EAAE,KAAa;IAC7D,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC/C,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,UAAU,CAAC;QAAE,OAAO,UAAU,CAAC;IACxF,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAClF,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAChH,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,MAAM,CAAC,MAAgB;IAC9B,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,QAAQ,CAAC,KAAoB,EAAE,KAAa;IACnD,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3C,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/evaluation-run-state.d.ts

@@ -0,0 +1,23 @@
+export interface RuntimeDataSources {
+    sourceRoot: string;
+    codegraphDatabasePath: string;
+    contextVectorDatabasePath: string;
+    memoryDatabasePath: string;
+    knowledgeDatabasePath: string;
+    knowledgeVectorPath: string;
+    operationalDatabasePath: string;
+}
+export interface EvaluationRunState {
+    root: string;
+    kind: 'retrieval' | 'lifecycle';
+    caseId: string;
+    dataSources: RuntimeDataSources;
+    verifyBundleUnchanged(): Promise<void>;
+    dispose(): void;
+}
+export declare function createEvaluationRunState(input: {
+    bundlePath: string;
+    kind: EvaluationRunState['kind'];
+    caseId: string;
+}): EvaluationRunState;
+//# sourceMappingURL=evaluation-run-state.d.ts.map

package/dist/evaluation-run-state.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluation-run-state.d.ts","sourceRoot":"","sources":["../src/evaluation-run-state.ts"],"names":[],"mappings":"AAiBA,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,yBAAyB,EAAE,MAAM,CAAC;IAClC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,WAAW,GAAG,WAAW,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,kBAAkB,CAAC;IAChC,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,OAAO,IAAI,IAAI,CAAC;CACjB;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE;IAC9C,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,kBAAkB,CAAC,MAAM,CAAC,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;CAChB,GAAG,kBAAkB,CAgErB"}

package/dist/evaluation-run-state.js

@@ -0,0 +1,115 @@
+import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, lstatSync, writeFileSync, } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import Database from 'better-sqlite3';
+import * as sqliteVec from 'sqlite-vec';
+import { validateEvaluationBundle } from './evaluation-bundle.js';
+export function createEvaluationRunState(input) {
+    if (!input.caseId.trim() || input.caseId.includes('/') || input.caseId.includes('\\')) {
+        throw new Error('EVALUATION_RUN_CASE_ID_INVALID');
+    }
+    const validation = validateEvaluationBundle(input.bundlePath);
+    if (validation.status !== 'valid')
+        throw new Error(`EVALUATION_BUNDLE_INVALID:${validation.reasonCode}`);
+    const bundleId = validation.manifest.bundleId;
+    const root = mkdtempSync(join(tmpdir(), `devflow-evaluation-${input.kind}-`));
+    const sourceRoot = join(root, 'source');
+    const codegraphDatabasePath = join(sourceRoot, '.codegraph', 'codegraph.db');
+    const contextVectorDatabasePath = join(sourceRoot, '.devflow', 'symbols-vec.db');
+    const memoryDatabasePath = join(sourceRoot, '.devflow', 'memory', 'memories.db');
+    const knowledgeDatabasePath = join(sourceRoot, '.devflow', 'knowledge', 'knowledge.db');
+    const knowledgeVectorPath = join(sourceRoot, '.devflow', 'knowledge', 'assets');
+    const operationalDatabasePath = join(root, 'operational', `${input.caseId}.db`);
+    try {
+        cpSync(join(input.bundlePath, 'source'), sourceRoot, { recursive: true, dereference: false });
+        copyFile(join(input.bundlePath, 'channels/codegraph.db'), codegraphDatabasePath);
+        copyFile(join(input.bundlePath, 'channels/context-symbols.db'), contextVectorDatabasePath);
+        copyFile(join(input.bundlePath, 'channels/memory.db'), memoryDatabasePath);
+        copyFile(join(input.bundlePath, 'channels/knowledge.db'), knowledgeDatabasePath);
+        cpSync(join(input.bundlePath, 'channels/knowledge-assets'), knowledgeVectorPath, {
+            recursive: true,
+            dereference: false,
+        });
+        makeWritable(root);
+        mkdirSync(join(sourceRoot, '.devflow'), { recursive: true });
+        writeFileSync(join(sourceRoot, '.devflow', 'config.json'), `${JSON.stringify({
+            projectId: validation.manifest.projectId,
+            memoryNamespace: `memory:${validation.manifest.projectId}`,
+        })}\n`);
+        bindPortableProjectScope(memoryDatabasePath, sourceRoot);
+        bindPortableProjectScope(knowledgeDatabasePath, sourceRoot);
+        mkdirSync(join(root, 'operational'), { recursive: true });
+    }
+    catch (error) {
+        makeWritable(root);
+        rmSync(root, { recursive: true, force: true });
+        throw error;
+    }
+    return {
+        root,
+        kind: input.kind,
+        caseId: input.caseId,
+        dataSources: {
+            sourceRoot,
+            codegraphDatabasePath,
+            contextVectorDatabasePath,
+            memoryDatabasePath,
+            knowledgeDatabasePath,
+            knowledgeVectorPath,
+            operationalDatabasePath,
+        },
+        async verifyBundleUnchanged() {
+            const current = validateEvaluationBundle(input.bundlePath);
+            if (current.status !== 'valid' || current.manifest.bundleId !== bundleId) {
+                throw new Error(`EVALUATION_BUNDLE_CHANGED:${current.status === 'invalid' ? current.reasonCode : 'bundle_id'}`);
+            }
+        },
+        dispose() {
+            makeWritable(root);
+            rmSync(root, { recursive: true, force: true });
+        },
+    };
+}
+function bindPortableProjectScope(databasePath, sourceRoot) {
+    const database = new Database(databasePath);
+    try {
+        sqliteVec.load(database);
+        database.transaction(() => {
+            const tables = database.prepare(`SELECT name FROM sqlite_master
+        WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name`).all();
+            for (const { name } of tables) {
+                const columns = new Set(database.pragma(`table_info(${quoteIdentifier(name)})`)
+                    .map(column => column.name));
+                if (columns.has('project_root')) {
+                    database.prepare(`UPDATE ${quoteIdentifier(name)} SET project_root = ? WHERE project_root = '.'`)
+                        .run(sourceRoot);
+                }
+            }
+        })();
+    }
+    finally {
+        database.close();
+    }
+}
+function quoteIdentifier(value) {
+    return `"${value.replaceAll('"', '""')}"`;
+}
+function copyFile(source, destination) {
+    mkdirSync(dirname(destination), { recursive: true });
+    cpSync(source, destination, { force: false, errorOnExist: true });
+}
+function makeWritable(path) {
+    if (!existsSync(path))
+        return;
+    const stat = lstatSync(path);
+    if (stat.isSymbolicLink())
+        return;
+    if (!stat.isDirectory()) {
+        chmodSync(path, 0o644);
+        return;
+    }
+    chmodSync(path, 0o755);
+    for (const name of readdirSync(path))
+        makeWritable(join(path, name));
+}
+//# sourceMappingURL=evaluation-run-state.js.map

package/dist/evaluation-run-state.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluation-run-state.js","sourceRoot":"","sources":["../src/evaluation-run-state.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,SAAS,EACT,MAAM,EACN,UAAU,EACV,SAAS,EACT,WAAW,EACX,WAAW,EACX,MAAM,EACN,SAAS,EACT,aAAa,GACd,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,KAAK,SAAS,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAqBlE,MAAM,UAAU,wBAAwB,CAAC,KAIxC;IACC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACtF,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IACD,MAAM,UAAU,GAAG,wBAAwB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC9D,IAAI,UAAU,CAAC,MAAM,KAAK,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,UAAU,CAAC,UAAU,EAAE,CAAC,CAAC;IACzG,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAC9C,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,sBAAsB,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IAC9E,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACxC,MAAM,qBAAqB,GAAG,IAAI,CAAC,UAAU,EAAE,YAAY,EAAE,cAAc,CAAC,CAAC;IAC7E,MAAM,yBAAyB,GAAG,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,gBAAgB,CAAC,CAAC;IACjF,MAAM,kBAAkB,GAAG,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IACjF,MAAM,qBAAqB,GAAG,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC;IACxF,MAAM,mBAAmB,GAAG,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC;IAChF,MAAM,uBAAuB,GAAG,IAAI,CAAC,IAAI,EAAE,aAAa,EAAE,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC;IAChF,IAAI,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9F,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,EAAE,qBAAqB,CAAC,CAAC;QACjF,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,6BAA6B,CAAC,EAAE,yBAAyB,CAAC,CAAC;QAC3F,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,oBAAoB,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAC3E,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,EAAE,qBAAqB,CAAC,CAAC;QACjF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,2BAA2B,CAAC,EAAE,mBAAmB,EAAE;YAC/E,SAAS,EAAE,IAAI;YACf,WAAW,EAAE,KAAK;SACnB,CAAC,CAAC;QACH,YAAY,CAAC,IAAI,CAAC,CAAC;QACnB,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,aAAa,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;YAC3E,SAAS,EAAE,UAAU,CAAC,QAAQ,CAAC,SAAS;YACxC,eAAe,EAAE,UAAU,UAAU,CAAC,QAAQ,CAAC,SAAS,EAAE;SAC3D,CAAC,IAAI,CAAC,CAAC;QACR,wBAAwB,CAAC,kBAAkB,EAAE,UAAU,CAAC,CAAC;QACzD,wBAAwB,CAAC,qBAAqB,EAAE,UAAU,CAAC,CAAC;QAC5D,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,aAAa,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,YAAY,CAAC,IAAI,CAAC,CAAC;QACnB,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,KAAK,CAAC;IACd,CAAC;IAED,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,WAAW,EAAE;YACX,UAAU;YACV,qBAAqB;YACrB,yBAAyB;YACzB,kBAAkB;YAClB,qBAAqB;YACrB,mBAAmB;YACnB,uBAAuB;SACxB;QACD,KAAK,CAAC,qBAAqB;YACzB,MAAM,OAAO,GAAG,wBAAwB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAC3D,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBACzE,MAAM,IAAI,KAAK,CAAC,6BAA6B,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YAClH,CAAC;QACH,CAAC;QACD,OAAO;YACL,YAAY,CAAC,IAAI,CAAC,CAAC;YACnB,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjD,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAAC,YAAoB,EAAE,UAAkB;IACxE,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC5C,IAAI,CAAC;QACH,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,QAAQ,CAAC,WAAW,CAAC,GAAG,EAAE;YACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC;wEACkC,CAAC,CAAC,GAAG,EAA6B,CAAC;YACrG,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC;gBAC9B,MAAM,OAAO,GAAG,IAAI,GAAG,CAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,eAAe,CAAC,IAAI,CAAC,GAAG,CAA6B;qBACzG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC/B,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;oBAChC,QAAQ,CAAC,OAAO,CAAC,UAAU,eAAe,CAAC,IAAI,CAAC,gDAAgD,CAAC;yBAC9F,GAAG,CAAC,UAAU,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;YAAS,CAAC;QACT,QAAQ,CAAC,KAAK,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACpC,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC;AAC5C,CAAC;AAED,SAAS,QAAQ,CAAC,MAAc,EAAE,WAAmB;IACnD,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACrD,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;AACpE,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO;IAC9B,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,IAAI,CAAC,cAAc,EAAE;QAAE,OAAO;IAClC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;QACxB,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACvB,OAAO;IACT,CAAC;IACD,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACvB,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,IAAI,CAAC;QAAE,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AACvE,CAAC"}

package/dist/evidence-quality-metrics.d.ts

@@ -16,5 +16,20 @@ export interface EvidenceQualityMetadata {
     latencyBudgetMs: number;
     observedLatencyMs: number;
 }
+export interface AggregatedEvidenceQualityMetric {
+    caseMacro: number | null;
+    refMicro: EvidenceQualityMetric;
+}
 export declare function evidenceQualityMetric(numerator: number, denominator: number): EvidenceQualityMetric;
+export declare function candidateRecallAt20(candidateRefs: string[], goldRefs: Set<string>): EvidenceQualityMetric;
+export declare function eligibilitySurvival(candidateRefs: string[], eligibilityRejectedRefs: string[], goldRefs: Set<string>): EvidenceQualityMetric;
+export declare function recallAt5(selectedRefs: string[], goldRefs: Set<string>): EvidenceQualityMetric;
+export declare function precisionAt5(selectedRefs: string[], goldRefs: Set<string>): EvidenceQualityMetric;
+export declare function forbiddenContaminationAt5(selectedRefs: string[], forbiddenRefs: Set<string>): EvidenceQualityMetric;
+export declare function selectBestGoldSet(input: {
+    goldSets: string[][];
+    candidateRefs: string[];
+    selectedRefs: string[];
+}): string[];
+export declare function aggregateEvidenceQualityMetrics(metrics: EvidenceQualityMetric[]): AggregatedEvidenceQualityMetric;
 //# sourceMappingURL=evidence-quality-metrics.d.ts.map

package/dist/evidence-quality-metrics.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evidence-quality-metrics.d.ts","sourceRoot":"","sources":["../src/evidence-quality-metrics.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,QAAQ,EAAE,gBAAgB,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,qBAAqB,CAuBnG"}
+{"version":3,"file":"evidence-quality-metrics.d.ts","sourceRoot":"","sources":["../src/evidence-quality-metrics.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,QAAQ,EAAE,gBAAgB,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,+BAA+B;IAC9C,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,qBAAqB,CAAC;CACjC;AAED,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,qBAAqB,CAuBnG;AAED,wBAAgB,mBAAmB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,qBAAqB,CAGzG;AAED,wBAAgB,mBAAmB,CACjC,aAAa,EAAE,MAAM,EAAE,EACvB,uBAAuB,EAAE,MAAM,EAAE,EACjC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,GACpB,qBAAqB,CAIvB;AAED,wBAAgB,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,qBAAqB,CAG9F;AAED,wBAAgB,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,qBAAqB,CAEjG;AAED,wBAAgB,yBAAyB,CACvC,YAAY,EAAE,MAAM,EAAE,EACtB,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,GACzB,qBAAqB,CAEvB;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE;IACvC,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAC;IACrB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,GAAG,MAAM,EAAE,CAWX;AAED,wBAAgB,+BAA+B,CAC7C,OAAO,EAAE,qBAAqB,EAAE,GAC/B,+BAA+B,CAWjC"}

package/dist/evidence-quality-metrics.js

@@ -1,5 +1,6 @@
+import { stableRuntimeHash } from '@devflow-tools/sdk';
 export function evidenceQualityMetric(numerator, denominator) {
-    if (!Number.isFinite(numerator) || !Number.isSafeInteger(denominator)
+    if (!Number.isSafeInteger(numerator) || !Number.isSafeInteger(denominator)
         || numerator < 0 || denominator < 0 || numerator > denominator) {
         throw new Error(`Invalid evidence quality metric: ${numerator}/${denominator}`);
     }
@@ -22,4 +23,53 @@ export function evidenceQualityMetric(numerator, denominator) {
         },
     };
 }
+export function candidateRecallAt20(candidateRefs, goldRefs) {
+    const candidates = new Set(uniqueTopK(candidateRefs, 20));
+    return evidenceQualityMetric([...goldRefs].filter(ref => candidates.has(ref)).length, goldRefs.size);
+}
+export function eligibilitySurvival(candidateRefs, eligibilityRejectedRefs, goldRefs) {
+    const candidateGold = uniqueTopK(candidateRefs, 20).filter(ref => goldRefs.has(ref));
+    const rejected = new Set(eligibilityRejectedRefs);
+    return evidenceQualityMetric(candidateGold.filter(ref => !rejected.has(ref)).length, candidateGold.length);
+}
+export function recallAt5(selectedRefs, goldRefs) {
+    const selected = new Set(uniqueTopK(selectedRefs, 5));
+    return evidenceQualityMetric([...goldRefs].filter(ref => selected.has(ref)).length, goldRefs.size);
+}
+export function precisionAt5(selectedRefs, goldRefs) {
+    return evidenceQualityMetric(uniqueTopK(selectedRefs, 5).filter(ref => goldRefs.has(ref)).length, 5);
+}
+export function forbiddenContaminationAt5(selectedRefs, forbiddenRefs) {
+    return evidenceQualityMetric(uniqueTopK(selectedRefs, 5).filter(ref => forbiddenRefs.has(ref)).length, 5);
+}
+export function selectBestGoldSet(input) {
+    if (input.goldSets.length === 0)
+        return [];
+    const selected = new Set(uniqueTopK(input.selectedRefs, 5));
+    const candidates = new Set(uniqueTopK(input.candidateRefs, 20));
+    return input.goldSets.map(refs => [...new Set(refs)].sort()).sort((left, right) => {
+        const selectedOverlap = overlap(right, selected) - overlap(left, selected);
+        if (selectedOverlap !== 0)
+            return selectedOverlap;
+        const candidateOverlap = overlap(right, candidates) - overlap(left, candidates);
+        if (candidateOverlap !== 0)
+            return candidateOverlap;
+        return stableRuntimeHash({ refs: left }).localeCompare(stableRuntimeHash({ refs: right }));
+    })[0];
+}
+export function aggregateEvidenceQualityMetrics(metrics) {
+    const measurable = metrics.filter(metric => metric.denominator > 0);
+    return {
+        caseMacro: measurable.length === 0
+            ? null
+            : measurable.reduce((total, metric) => total + metric.numerator / metric.denominator, 0) / measurable.length,
+        refMicro: evidenceQualityMetric(measurable.reduce((total, metric) => total + metric.numerator, 0), measurable.reduce((total, metric) => total + metric.denominator, 0)),
+    };
+}
+function uniqueTopK(refs, limit) {
+    return [...new Set(refs)].slice(0, limit);
+}
+function overlap(refs, selected) {
+    return refs.filter(ref => selected.has(ref)).length;
+}
 //# sourceMappingURL=evidence-quality-metrics.js.map

package/dist/evidence-quality-metrics.js.map

@@ -1 +1 @@
-{"version":3,"file":"evidence-quality-metrics.js","sourceRoot":"","sources":["../src/evidence-quality-metrics.ts"],"names":[],"mappings":"AAqBA,MAAM,UAAU,qBAAqB,CAAC,SAAiB,EAAE,WAAmB;IAC1E,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC;WAChE,SAAS,GAAG,CAAC,IAAI,WAAW,GAAG,CAAC,IAAI,SAAS,GAAG,WAAW,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,oCAAoC,SAAS,IAAI,WAAW,EAAE,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IACjE,CAAC;IACD,MAAM,KAAK,GAAG,SAAS,GAAG,WAAW,CAAC;IACtC,MAAM,CAAC,GAAG,iBAAiB,CAAC;IAC5B,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACjB,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;IACnC,MAAM,MAAM,GAAG,CAAC,KAAK,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,KAAK,CAAC;IACxD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,WAAW,CAAC,GAAG,KAAK,CAAC;IACnG,OAAO;QACL,SAAS;QACT,WAAW;QACX,KAAK;QACL,QAAQ,EAAE;YACR,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;YACnC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;SACpC;KACF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"evidence-quality-metrics.js","sourceRoot":"","sources":["../src/evidence-quality-metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AA4BvD,MAAM,UAAU,qBAAqB,CAAC,SAAiB,EAAE,WAAmB;IAC1E,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC;WACrE,SAAS,GAAG,CAAC,IAAI,WAAW,GAAG,CAAC,IAAI,SAAS,GAAG,WAAW,EAAE,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,oCAAoC,SAAS,IAAI,WAAW,EAAE,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IACjE,CAAC;IACD,MAAM,KAAK,GAAG,SAAS,GAAG,WAAW,CAAC;IACtC,MAAM,CAAC,GAAG,iBAAiB,CAAC;IAC5B,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACjB,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;IACnC,MAAM,MAAM,GAAG,CAAC,KAAK,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,KAAK,CAAC;IACxD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,WAAW,CAAC,GAAG,KAAK,CAAC;IACnG,OAAO;QACL,SAAS;QACT,WAAW;QACX,KAAK;QACL,QAAQ,EAAE;YACR,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;YACnC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;SACpC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,aAAuB,EAAE,QAAqB;IAChF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1D,OAAO,qBAAqB,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;AACvG,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,aAAuB,EACvB,uBAAiC,EACjC,QAAqB;IAErB,MAAM,aAAa,GAAG,UAAU,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IACrF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,uBAAuB,CAAC,CAAC;IAClD,OAAO,qBAAqB,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;AAC7G,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,YAAsB,EAAE,QAAqB;IACrE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC;IACtD,OAAO,qBAAqB,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;AACrG,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,YAAsB,EAAE,QAAqB;IACxE,OAAO,qBAAqB,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AACvG,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,YAAsB,EACtB,aAA0B;IAE1B,OAAO,qBAAqB,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AAC5G,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAIjC;IACC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAC3C,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,CAAC;IAChE,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAChF,MAAM,eAAe,GAAG,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC3E,IAAI,eAAe,KAAK,CAAC;YAAE,OAAO,eAAe,CAAC;QAClD,MAAM,gBAAgB,GAAG,OAAO,CAAC,KAAK,EAAE,UAAU,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAChF,IAAI,gBAAgB,KAAK,CAAC;YAAE,OAAO,gBAAgB,CAAC;QACpD,OAAO,iBAAiB,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,aAAa,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IAC7F,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC;AACT,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,OAAgC;IAEhC,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;IACpE,OAAO;QACL,SAAS,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC;YAChC,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM;QAC9G,QAAQ,EAAE,qBAAqB,CAC7B,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,EACjE,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CACpE;KACF,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,IAAc,EAAE,KAAa;IAC/C,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,OAAO,CAAC,IAAc,EAAE,QAAqB;IACpD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;AACtD,CAAC"}

package/dist/graders/deterministic.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../src/graders/deterministic.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAiB,gBAAgB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAKhF,eAAO,MAAM,qBAAqB,EAAE,gBAAgB,EAWnD,CAAC;AAEF,wBAAgB,uBAAuB,IAAI,gBAAgB,CAuC1D;AAED,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAA;CAAE,GACpE,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,OAAO,CAAA;CAAE,CAAC,CAgC1F;AAmHD,wBAAgB,MAAM,CACpB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,YAAY,CAAC,QAAQ,CAAC,EAC9B,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,YAAY,CAAC,UAAU,CAAC,EAClC,KAAK,CAAC,EAAE,MAAM,EACd,SAAS,CAAC,EAAE,MAAM,GACjB,YAAY,CAMd"}
+{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../src/graders/deterministic.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAiB,gBAAgB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAKhF,eAAO,MAAM,qBAAqB,EAAE,gBAAgB,EAWnD,CAAC;AAEF,wBAAgB,uBAAuB,IAAI,gBAAgB,CAuC1D;AAED,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAA;CAAE,GACpE,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,OAAO,CAAA;CAAE,CAAC,CAgC1F;AAqHD,wBAAgB,MAAM,CACpB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,YAAY,CAAC,QAAQ,CAAC,EAC9B,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,YAAY,CAAC,UAAU,CAAC,EAClC,KAAK,CAAC,EAAE,MAAM,EACd,SAAS,CAAC,EAAE,MAAM,GACjB,YAAY,CAMd"}