@kaelio/ktx 0.0.0 → 0.1.0-rc.1

package/node_modules/@ktx/context/dist/scan/relationship-benchmark-report.test.js

@@ -0,0 +1,389 @@
+import { describe, expect, it } from 'vitest';
+import { buildKtxRelationshipBenchmarkReport, formatKtxRelationshipBenchmarkReportMarkdown, } from './relationship-benchmark-report.js';
+function caseResult(overrides = {}) {
+    return {
+        fixtureId: overrides.fixtureId ?? 'demo_b2b_no_declared_constraints',
+        mode: overrides.mode ?? 'declared_pks_and_declared_fks_removed',
+        metrics: {
+            pkPrecision: 1,
+            pkRecall: 0.5,
+            pkF1: 0.6666666666666666,
+            fkPrecision: 1,
+            fkRecall: 1,
+            fkF1: 1,
+            acceptedFalsePositiveCount: 0,
+            reviewRecall: 0,
+            acceptedOrReviewRecall: 1,
+            runtimeSeconds: 0.012345,
+            sqlQueries: 14,
+            llmCalls: 0,
+            ...(overrides.metrics ?? {}),
+        },
+        expected: overrides.expected ?? {
+            pk: ['accounts.(id)', 'users.(id)'],
+            fk: ['users.(account_id)->accounts.(id)'],
+        },
+        predicted: overrides.predicted ?? {
+            pk: ['accounts.(id)'],
+            fk: ['users.(account_id)->accounts.(id)'],
+            acceptedFk: ['users.(account_id)->accounts.(id)'],
+            reviewFk: [],
+        },
+        falsePositives: overrides.falsePositives ?? { pk: [], fk: [] },
+        falseNegatives: overrides.falseNegatives ?? { pk: ['users.(id)'], fk: [] },
+        skippedComposite: overrides.skippedComposite ?? { pk: [], fk: [] },
+        validationBlocked: overrides.validationBlocked ?? false,
+    };
+}
+function fixture(overrides = {}) {
+    return {
+        id: overrides.id ?? 'demo_b2b_no_declared_constraints',
+        name: overrides.name ?? 'Packaged B2B demo with declared PK and FK metadata masked',
+        tier: overrides.tier ?? 'smoke',
+        origin: overrides.origin ?? 'synthetic',
+        thresholdEligible: overrides.thresholdEligible,
+        validationBudget: overrides.validationBudget,
+        snapshot: overrides.snapshot ?? {
+            connectionId: 'demo_b2b',
+            driver: 'sqlite',
+            extractedAt: '2026-05-07T00:00:00.000Z',
+            scope: {},
+            metadata: {},
+            tables: [],
+        },
+        expected: overrides.expected ?? { expectedPks: [], expectedLinks: [] },
+        defaultModes: overrides.defaultModes ?? ['declared_pks_and_declared_fks_removed', 'validation_disabled'],
+        dataPath: overrides.dataPath ?? '/tmp/demo.sqlite',
+        columnEmbeddings: overrides.columnEmbeddings ?? {},
+    };
+}
+describe('relationship benchmark report', () => {
+    it('classifies run, validation-blocked, and not-run benchmark cases', () => {
+        const suite = {
+            cases: [
+                caseResult(),
+                caseResult({
+                    mode: 'validation_disabled',
+                    validationBlocked: true,
+                    metrics: { fkRecall: 0, acceptedOrReviewRecall: 1, sqlQueries: 0 },
+                    predicted: {
+                        pk: ['accounts.(id)'],
+                        fk: ['users.(account_id)->accounts.(id)'],
+                        acceptedFk: [],
+                        reviewFk: ['users.(account_id)->accounts.(id)'],
+                    },
+                }),
+            ],
+            validationBlockedCases: ['demo_b2b_no_declared_constraints:validation_disabled'],
+            aggregate: {
+                caseCount: 2,
+                headlineCaseCount: 1,
+                headlinePkRecall: 0.5,
+                headlineFkRecall: 1,
+                headlineAcceptedOrReviewRecall: 1,
+                meanPkRecall: 0.5,
+                meanFkRecall: 0.5,
+                meanAcceptedOrReviewRecall: 1,
+            },
+        };
+        const report = buildKtxRelationshipBenchmarkReport({
+            fixtures: [fixture()],
+            suite,
+            modes: ['declared_pks_and_declared_fks_removed', 'validation_disabled', 'profiling_disabled'],
+        });
+        expect(report.headline).toEqual({
+            caseCount: 2,
+            headlineCaseCount: 1,
+            headlinePkRecall: 0.5,
+            headlineFkRecall: 1,
+            headlineAcceptedOrReviewRecall: 1,
+            acceptedFalsePositiveCount: 0,
+            validationBlockedCount: 1,
+        });
+        expect(report.cases.map((item) => `${item.fixtureId}:${item.mode}:${item.status}`)).toEqual([
+            'demo_b2b_no_declared_constraints:declared_pks_and_declared_fks_removed:run',
+            'demo_b2b_no_declared_constraints:validation_disabled:validation_blocked',
+            'demo_b2b_no_declared_constraints:profiling_disabled:not_run',
+        ]);
+        expect(report.cases[2]?.reason).toBe('mode not selected by fixture defaultModes');
+    });
+    it('surfaces validation budget review candidates in the report reason', () => {
+        const suite = {
+            cases: [
+                caseResult({
+                    fixtureId: 'scale_stress_no_declared_constraints',
+                    metrics: { fkRecall: 0.5, acceptedOrReviewRecall: 1 },
+                    predicted: {
+                        pk: ['dim_entity_00.(entity_00_key)'],
+                        fk: [
+                            'fact_activity_000.(entity_00_key)->dim_entity_00.(entity_00_key)',
+                            'fact_activity_001.(entity_00_key)->dim_entity_00.(entity_00_key)',
+                        ],
+                        acceptedFk: ['fact_activity_000.(entity_00_key)->dim_entity_00.(entity_00_key)'],
+                        reviewFk: ['fact_activity_001.(entity_00_key)->dim_entity_00.(entity_00_key)'],
+                    },
+                }),
+            ],
+            validationBlockedCases: [],
+            aggregate: {
+                caseCount: 1,
+                headlineCaseCount: 0,
+                headlinePkRecall: 1,
+                headlineFkRecall: 0.5,
+                headlineAcceptedOrReviewRecall: 1,
+                meanPkRecall: 1,
+                meanFkRecall: 0.5,
+                meanAcceptedOrReviewRecall: 1,
+            },
+        };
+        const report = buildKtxRelationshipBenchmarkReport({
+            fixtures: [
+                fixture({
+                    id: 'scale_stress_no_declared_constraints',
+                    name: 'Scale stress fixture',
+                    tier: 'row_bearing',
+                    validationBudget: 800,
+                    defaultModes: ['declared_pks_and_declared_fks_removed'],
+                }),
+            ],
+            suite,
+            modes: ['declared_pks_and_declared_fks_removed'],
+        });
+        expect(report.cases[0]?.reason).toBe('review candidate validation reasons: validation_unattempted (1)');
+        expect(formatKtxRelationshipBenchmarkReportMarkdown(report)).toContain('validation_unattempted');
+    });
+    it('uses benchmark suite eligibility for product and smoke report rows', () => {
+        const productCase = caseResult({ fixtureId: 'product_curated' });
+        const productBlocked = caseResult({
+            fixtureId: 'product_curated',
+            mode: 'validation_disabled',
+            validationBlocked: true,
+            metrics: { fkRecall: 0, acceptedOrReviewRecall: 1, sqlQueries: 0 },
+        });
+        const smokeCase = caseResult({ fixtureId: 'smoke_even_if_marked' });
+        const suite = {
+            cases: [productCase, productBlocked, smokeCase],
+            validationBlockedCases: ['product_curated:validation_disabled'],
+            aggregate: {
+                caseCount: 3,
+                headlineCaseCount: 1,
+                headlinePkRecall: 0.5,
+                headlineFkRecall: 1,
+                headlineAcceptedOrReviewRecall: 1,
+                meanPkRecall: 0.5,
+                meanFkRecall: 0.6666666666666666,
+                meanAcceptedOrReviewRecall: 1,
+            },
+        };
+        const report = buildKtxRelationshipBenchmarkReport({
+            fixtures: [
+                fixture({
+                    id: 'product_curated',
+                    name: 'Curated product fixture',
+                    tier: 'product',
+                    thresholdEligible: true,
+                    defaultModes: ['declared_pks_and_declared_fks_removed', 'validation_disabled'],
+                }),
+                fixture({
+                    id: 'smoke_even_if_marked',
+                    name: 'Marked smoke fixture',
+                    tier: 'smoke',
+                    thresholdEligible: true,
+                    defaultModes: ['declared_pks_and_declared_fks_removed'],
+                }),
+            ],
+            suite,
+            modes: ['declared_pks_and_declared_fks_removed', 'validation_disabled'],
+        });
+        expect(report.cases.map((item) => `${item.fixtureId}:${item.mode}:${item.tuningEligible}`)).toEqual([
+            'product_curated:declared_pks_and_declared_fks_removed:true',
+            'product_curated:validation_disabled:false',
+            'smoke_even_if_marked:declared_pks_and_declared_fks_removed:false',
+            'smoke_even_if_marked:validation_disabled:false',
+        ]);
+        expect(formatKtxRelationshipBenchmarkReportMarkdown(report)).toContain('| product_curated | product | declared_pks_and_declared_fks_removed | run | yes |');
+    });
+    it('formats a compact Markdown report with false negatives and blocked modes', () => {
+        const suite = {
+            cases: [
+                caseResult({
+                    metrics: { fkRecall: 0, acceptedOrReviewRecall: 0 },
+                    falseNegatives: { pk: ['users.(id)'], fk: ['users.(account_id)->accounts.(id)'] },
+                }),
+            ],
+            validationBlockedCases: [],
+            aggregate: {
+                caseCount: 1,
+                headlineCaseCount: 1,
+                headlinePkRecall: 0.5,
+                headlineFkRecall: 0,
+                headlineAcceptedOrReviewRecall: 0,
+                meanPkRecall: 0.5,
+                meanFkRecall: 0,
+                meanAcceptedOrReviewRecall: 0,
+            },
+        };
+        const markdown = formatKtxRelationshipBenchmarkReportMarkdown(buildKtxRelationshipBenchmarkReport({
+            fixtures: [fixture()],
+            suite,
+            modes: ['declared_pks_and_declared_fks_removed'],
+        }));
+        expect(markdown).toContain('# KTX Relationship Discovery Benchmark Evidence');
+        expect(markdown).toContain('| demo_b2b_no_declared_constraints | smoke | declared_pks_and_declared_fks_removed | run | no | 0.500 | 0.000 | 0.000 | 0 |');
+        expect(markdown).toContain('- `demo_b2b_no_declared_constraints` / `declared_pks_and_declared_fks_removed` / `run`: users.(id)');
+        expect(markdown).toContain('- `demo_b2b_no_declared_constraints` / `declared_pks_and_declared_fks_removed` / `run`: users.(account_id)->accounts.(id)');
+    });
+    it('keeps headline failures separate from non-headline failure details', () => {
+        const suite = {
+            cases: [
+                caseResult({
+                    fixtureId: 'product_curated',
+                    falseNegatives: { pk: [], fk: [] },
+                    metrics: { pkRecall: 1, fkRecall: 1, acceptedOrReviewRecall: 1 },
+                }),
+                caseResult({
+                    fixtureId: 'product_curated',
+                    mode: 'embeddings_disabled',
+                    falseNegatives: {
+                        pk: ['customers.(id)'],
+                        fk: ['orders.(buyer_ref)->customers.(id)'],
+                    },
+                    metrics: { pkRecall: 0.5, fkRecall: 0, acceptedOrReviewRecall: 0 },
+                }),
+            ],
+            validationBlockedCases: [],
+            aggregate: {
+                caseCount: 2,
+                headlineCaseCount: 1,
+                headlinePkRecall: 1,
+                headlineFkRecall: 1,
+                headlineAcceptedOrReviewRecall: 1,
+                meanPkRecall: 0.75,
+                meanFkRecall: 0.5,
+                meanAcceptedOrReviewRecall: 0.5,
+            },
+        };
+        const markdown = formatKtxRelationshipBenchmarkReportMarkdown(buildKtxRelationshipBenchmarkReport({
+            fixtures: [
+                fixture({
+                    id: 'product_curated',
+                    name: 'Curated product fixture',
+                    tier: 'product',
+                    thresholdEligible: true,
+                    defaultModes: ['declared_pks_and_declared_fks_removed', 'embeddings_disabled'],
+                }),
+            ],
+            suite,
+            modes: ['declared_pks_and_declared_fks_removed', 'embeddings_disabled'],
+        }));
+        expect(markdown).toContain('## Failure Details');
+        expect(markdown).toContain('### Headline False Negative FKs\n\n- none');
+        expect(markdown).toContain('- `product_curated` / `embeddings_disabled` / `run`: orders.(buyer_ref)->customers.(id)');
+        expect(markdown).toContain('- `product_curated` / `embeddings_disabled` / `run`: customers.(id)');
+    });
+    it('formats headline failure context from remaining headline false negatives', () => {
+        const suite = {
+            cases: [
+                caseResult({
+                    fixtureId: 'public_headline_fixture',
+                    metrics: { pkRecall: 0.5, fkRecall: 0, acceptedOrReviewRecall: 0 },
+                    falseNegatives: {
+                        pk: ['parent_table.(opaque_key)'],
+                        fk: ['child_table.(parent_table_id)->parent_table.(opaque_key)'],
+                    },
+                }),
+            ],
+            validationBlockedCases: [],
+            aggregate: {
+                caseCount: 1,
+                headlineCaseCount: 1,
+                headlinePkRecall: 0.5,
+                headlineFkRecall: 0,
+                headlineAcceptedOrReviewRecall: 0,
+                meanPkRecall: 0.5,
+                meanFkRecall: 0,
+                meanAcceptedOrReviewRecall: 0,
+            },
+        };
+        const markdown = formatKtxRelationshipBenchmarkReportMarkdown(buildKtxRelationshipBenchmarkReport({
+            fixtures: [
+                fixture({
+                    id: 'public_headline_fixture',
+                    name: 'Public headline fixture',
+                    tier: 'row_bearing',
+                    thresholdEligible: true,
+                    defaultModes: ['declared_pks_and_declared_fks_removed'],
+                }),
+            ],
+            suite,
+            modes: ['declared_pks_and_declared_fks_removed'],
+        }));
+        expect(markdown).toContain('## Headline Failure Context');
+        expect(markdown).toContain('- Remaining headline false-negative PKs: 1');
+        expect(markdown).toContain('- Remaining headline false-negative FKs: 1');
+        expect(markdown).toContain('- `public_headline_fixture` / `declared_pks_and_declared_fks_removed` / `run`: parent_table.(opaque_key)');
+        expect(markdown).toContain('- `public_headline_fixture` / `declared_pks_and_declared_fks_removed` / `run`: child_table.(parent_table_id)->parent_table.(opaque_key)');
+    });
+    it('formats skipped composite ground truth separately from false-negative details', () => {
+        const compositePk = 'order_lines.(order_id,line_number)';
+        const compositeFk = 'order_line_allocations.(order_id,line_number)->order_lines.(order_id,line_number)';
+        const suite = {
+            cases: [
+                caseResult({
+                    fixtureId: 'composite_keys_no_declared_constraints',
+                    metrics: { pkRecall: 0, fkRecall: 0, acceptedOrReviewRecall: 0 },
+                    expected: {
+                        pk: [compositePk],
+                        fk: [compositeFk],
+                    },
+                    predicted: {
+                        pk: [],
+                        fk: [],
+                        acceptedFk: [],
+                        reviewFk: [],
+                    },
+                    falseNegatives: {
+                        pk: [compositePk],
+                        fk: [compositeFk],
+                    },
+                    skippedComposite: {
+                        pk: [compositePk],
+                        fk: [compositeFk],
+                    },
+                }),
+            ],
+            validationBlockedCases: [],
+            aggregate: {
+                caseCount: 1,
+                headlineCaseCount: 1,
+                headlinePkRecall: 0,
+                headlineFkRecall: 0,
+                headlineAcceptedOrReviewRecall: 0,
+                meanPkRecall: 0,
+                meanFkRecall: 0,
+                meanAcceptedOrReviewRecall: 0,
+            },
+        };
+        const report = buildKtxRelationshipBenchmarkReport({
+            fixtures: [
+                fixture({
+                    id: 'composite_keys_no_declared_constraints',
+                    name: 'Composite key fixture with no declared constraints',
+                    tier: 'row_bearing',
+                    defaultModes: ['declared_pks_and_declared_fks_removed'],
+                }),
+            ],
+            suite,
+            modes: ['declared_pks_and_declared_fks_removed'],
+        });
+        expect(report.cases[0]?.skippedComposite).toEqual({
+            pk: [compositePk],
+            fk: [compositeFk],
+        });
+        const markdown = formatKtxRelationshipBenchmarkReportMarkdown(report);
+        expect(markdown).toContain('## Composite Ground Truth Skips');
+        expect(markdown).toContain('### Skipped Composite PKs\n\n- `composite_keys_no_declared_constraints` / `declared_pks_and_declared_fks_removed` / `run`: order_lines.(order_id,line_number)');
+        expect(markdown).toContain('### Skipped Composite FKs\n\n- `composite_keys_no_declared_constraints` / `declared_pks_and_declared_fks_removed` / `run`: order_line_allocations.(order_id,line_number)->order_lines.(order_id,line_number)');
+        expect(markdown).toContain('### Headline False Negative FKs\n\n- `composite_keys_no_declared_constraints` / `declared_pks_and_declared_fks_removed` / `run`: order_line_allocations.(order_id,line_number)->order_lines.(order_id,line_number)');
+    });
+});

package/node_modules/@ktx/context/dist/scan/relationship-benchmarks.d.ts

@@ -0,0 +1,148 @@
+import type { KtxLlmRuntimePort } from '../llm/index.js';
+import type { KtxEnrichedSchema, KtxRelationshipType } from './enrichment-types.js';
+import type { KtxRelationshipValidationBudget } from './relationship-budget.js';
+import type { KtxRelationshipFixtureOrigin } from './relationship-scoring.js';
+import type { KtxSchemaSnapshot } from './types.js';
+export declare const KTX_RELATIONSHIP_BENCHMARK_MODES: readonly ["metadata_present", "declared_fks_removed", "declared_pks_removed", "declared_pks_and_declared_fks_removed", "llm_disabled", "profiling_disabled", "validation_disabled", "embeddings_disabled"];
+export type KtxRelationshipBenchmarkMode = (typeof KTX_RELATIONSHIP_BENCHMARK_MODES)[number];
+export declare const KTX_RELATIONSHIP_BENCHMARK_TIERS: readonly ["unit", "row_bearing", "schema_only", "smoke", "product"];
+export type KtxRelationshipBenchmarkTier = (typeof KTX_RELATIONSHIP_BENCHMARK_TIERS)[number];
+export type KtxRelationshipBenchmarkStatus = 'accepted' | 'review' | 'rejected';
+export interface KtxRelationshipBenchmarkExpectedPk {
+    table: string;
+    columns: string[];
+}
+export interface KtxRelationshipBenchmarkExpectedLink {
+    fromTable: string;
+    fromColumns: string[];
+    toTable: string;
+    toColumns: string[];
+    relationship: KtxRelationshipType;
+}
+export interface KtxRelationshipBenchmarkExpectedLinks {
+    expectedPks: KtxRelationshipBenchmarkExpectedPk[];
+    expectedLinks: KtxRelationshipBenchmarkExpectedLink[];
+}
+export interface KtxRelationshipBenchmarkFixture {
+    id: string;
+    name: string;
+    tier: KtxRelationshipBenchmarkTier;
+    origin: KtxRelationshipFixtureOrigin;
+    thresholdEligible?: boolean;
+    validationBudget?: KtxRelationshipValidationBudget;
+    snapshot: KtxSchemaSnapshot;
+    expected: KtxRelationshipBenchmarkExpectedLinks;
+    defaultModes: KtxRelationshipBenchmarkMode[];
+    dataPath: string | null;
+    columnEmbeddings: Record<string, number[]>;
+}
+export interface KtxRelationshipBenchmarkDetectedPk {
+    table: string;
+    columns: string[];
+    score: number;
+    status: KtxRelationshipBenchmarkStatus;
+}
+export interface KtxRelationshipBenchmarkDetectedLink {
+    fromTable: string;
+    fromColumns: string[];
+    toTable: string;
+    toColumns: string[];
+    relationship: KtxRelationshipType;
+    score: number;
+    status: KtxRelationshipBenchmarkStatus;
+    source: string;
+}
+export interface KtxRelationshipBenchmarkDetectorResult {
+    pks: KtxRelationshipBenchmarkDetectedPk[];
+    links: KtxRelationshipBenchmarkDetectedLink[];
+    validationBlocked: boolean;
+    sqlQueries: number;
+    llmCalls: number;
+    runtimeSeconds: number;
+}
+export interface KtxRelationshipBenchmarkDetectorInput {
+    fixtureId: string;
+    mode: KtxRelationshipBenchmarkMode;
+    snapshot: KtxSchemaSnapshot;
+    schema: KtxEnrichedSchema;
+    dataPath: string | null;
+    validationBudget?: KtxRelationshipValidationBudget;
+}
+export interface KtxRelationshipBenchmarkDetector {
+    detect(input: KtxRelationshipBenchmarkDetectorInput): Promise<KtxRelationshipBenchmarkDetectorResult>;
+}
+export interface KtxRelationshipBenchmarkMetrics {
+    pkPrecision: number;
+    pkRecall: number;
+    pkF1: number;
+    fkPrecision: number;
+    fkRecall: number;
+    fkF1: number;
+    acceptedFalsePositiveCount: number;
+    reviewRecall: number;
+    acceptedOrReviewRecall: number;
+    runtimeSeconds: number;
+    sqlQueries: number;
+    llmCalls: number;
+}
+export interface KtxRelationshipBenchmarkCaseResult {
+    fixtureId: string;
+    mode: KtxRelationshipBenchmarkMode;
+    metrics: KtxRelationshipBenchmarkMetrics;
+    expected: {
+        pk: string[];
+        fk: string[];
+    };
+    predicted: {
+        pk: string[];
+        fk: string[];
+        acceptedFk: string[];
+        reviewFk: string[];
+    };
+    falsePositives: {
+        pk: string[];
+        fk: string[];
+    };
+    falseNegatives: {
+        pk: string[];
+        fk: string[];
+    };
+    skippedComposite: {
+        pk: string[];
+        fk: string[];
+    };
+    validationBlocked: boolean;
+}
+export interface KtxRelationshipBenchmarkSuiteResult {
+    cases: KtxRelationshipBenchmarkCaseResult[];
+    validationBlockedCases: string[];
+    aggregate: {
+        caseCount: number;
+        headlineCaseCount: number;
+        headlinePkRecall: number;
+        headlineFkRecall: number;
+        headlineAcceptedOrReviewRecall: number;
+        meanPkRecall: number;
+        meanFkRecall: number;
+        meanAcceptedOrReviewRecall: number;
+    };
+}
+export declare function maskKtxRelationshipBenchmarkSnapshot(snapshot: KtxSchemaSnapshot, mode: KtxRelationshipBenchmarkMode): KtxSchemaSnapshot;
+export declare function isKtxRelationshipBenchmarkTuningEligible(input: {
+    fixture: Pick<KtxRelationshipBenchmarkFixture, 'tier' | 'thresholdEligible'>;
+    mode: KtxRelationshipBenchmarkMode;
+    validationBlocked: boolean;
+}): boolean;
+export declare function ktxRelationshipBenchmarkDetectorWithLlm(llmRuntime: KtxLlmRuntimePort): KtxRelationshipBenchmarkDetector;
+export declare function currentKtxRelationshipBenchmarkDetector(): KtxRelationshipBenchmarkDetector;
+export declare function loadKtxRelationshipBenchmarkFixture(fixtureDir: string): Promise<KtxRelationshipBenchmarkFixture>;
+export declare function loadKtxRelationshipBenchmarkFixtures(fixtureRoot: string): Promise<KtxRelationshipBenchmarkFixture[]>;
+export declare function runKtxRelationshipBenchmarkCase(input: {
+    fixture: KtxRelationshipBenchmarkFixture;
+    mode: KtxRelationshipBenchmarkMode;
+    detector?: KtxRelationshipBenchmarkDetector;
+}): Promise<KtxRelationshipBenchmarkCaseResult>;
+export declare function runKtxRelationshipBenchmarkSuite(input: {
+    fixtures: KtxRelationshipBenchmarkFixture[];
+    detector?: KtxRelationshipBenchmarkDetector;
+}): Promise<KtxRelationshipBenchmarkSuiteResult>;