@kaelio/ktx 0.1.0-rc.6 → 0.1.0

package/node_modules/@ktx/context/dist/ingest/reports.d.ts

@@ -29,31 +29,13 @@ export interface IngestReportToolTranscriptSummary {
     errorCount: number;
     toolNames: string[];
 }
-export interface IngestReportFinalizationMismatch {
-    artifactKind: 'sl' | 'wiki';
-    key: string;
-    direction: 'missing_from_adapter_declaration' | 'extra_in_adapter_declaration';
-}
-export interface IngestReportFinalizationProvenanceExclusion {
-    action: MemoryAction;
-    reason: 'missing_raw_paths' | 'raw_path_not_defensible';
-    invalidRawPaths?: string[];
-}
-export interface IngestReportFinalizationOutcome {
+export interface IngestReportPostProcessorOutcome {
     sourceKey: string;
-    status: 'success' | 'failed' | 'skipped';
-    commitSha: string | null;
-    touchedPaths: string[];
-    declaredTouchedSources: TouchedSlSource[];
-    derivedTouchedSources: TouchedSlSource[];
-    declaredChangedWikiPageKeys: string[];
-    derivedChangedWikiPageKeys: string[];
-    mismatches: IngestReportFinalizationMismatch[];
+    status: 'success' | 'failed';
     result?: unknown;
     errors: string[];
     warnings: string[];
-    actions: MemoryAction[];
-    provenanceExclusions: IngestReportFinalizationProvenanceExclusion[];
+    touchedSources: TouchedSlSource[];
 }
 export interface IngestReportFailure {
     phase: string;
@@ -97,7 +79,7 @@ export interface IngestReportBody {
     overrideOf: string | null;
     provenanceRows: IngestReportProvenanceDetail[];
     toolTranscripts: IngestReportToolTranscriptSummary[];
-    finalization?: IngestReportFinalizationOutcome;
+    postProcessor?: IngestReportPostProcessorOutcome;
     wikiSlRefRepairs?: WikiSlRefRepair[];
     wikiSlRefRepairWarnings?: string[];
     memoryFlow?: MemoryFlowReplayInput;
@@ -115,6 +97,6 @@ export interface IngestSavedMemoryCounts {
     wikiCount: number;
     slCount: number;
 }
-export declare function finalizationSavedMemoryCounts(finalization: IngestReportFinalizationOutcome | undefined): IngestSavedMemoryCounts;
+export declare function postProcessorSavedMemoryCounts(postProcessor: IngestReportPostProcessorOutcome | undefined): IngestSavedMemoryCounts;
 export declare function savedMemoryCountsForReport(report: IngestReportSnapshot): IngestSavedMemoryCounts;
 export declare function buildStageIndexFromReportBody(jobId: string, connectionId: string, body: IngestReportBody): StageIndex;

package/node_modules/@ktx/context/dist/ingest/reports.js

@@ -1,19 +1,36 @@
-export function finalizationSavedMemoryCounts(finalization) {
-    const actions = finalization?.actions ?? [];
+function numericResultField(result, field) {
+    const value = result[field];
+    return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0;
+}
+export function postProcessorSavedMemoryCounts(postProcessor) {
+    if (!postProcessor || postProcessor.sourceKey !== 'historic-sql') {
+        return { wikiCount: 0, slCount: 0 };
+    }
+    const result = postProcessor.result;
+    if (!result || typeof result !== 'object' || Array.isArray(result)) {
+        return { wikiCount: 0, slCount: 0 };
+    }
+    const record = result;
     return {
-        wikiCount: actions.filter((action) => action.target === 'wiki').length,
-        slCount: actions.filter((action) => action.target === 'sl').length,
+        wikiCount: numericResultField(record, 'patternPagesWritten') +
+            numericResultField(record, 'stalePatternPagesMarked') +
+            numericResultField(record, 'archivedPatternPages'),
+        slCount: numericResultField(record, 'tableUsageMerged') + numericResultField(record, 'staleTablesMarked'),
     };
 }
 export function savedMemoryCountsForReport(report) {
     const workUnitActions = report.body.workUnits.flatMap((workUnit) => workUnit.actions);
     const reconciliationActions = report.body.reconciliationActions ?? [];
-    const finalizationActions = report.body.finalization?.actions ?? [];
-    const actions = [...workUnitActions, ...reconciliationActions, ...finalizationActions];
-    return {
+    const actions = [...workUnitActions, ...reconciliationActions];
+    const directCounts = {
         wikiCount: actions.filter((action) => action.target === 'wiki').length,
         slCount: actions.filter((action) => action.target === 'sl').length,
     };
+    const postProcessorCounts = postProcessorSavedMemoryCounts(report.body.postProcessor);
+    return {
+        wikiCount: directCounts.wikiCount + postProcessorCounts.wikiCount,
+        slCount: directCounts.slCount + postProcessorCounts.slCount,
+    };
 }
 export function buildStageIndexFromReportBody(jobId, connectionId, body) {
     return {

package/node_modules/@ktx/context/dist/ingest/types.d.ts

@@ -1,10 +1,6 @@
 import type { KtxEmbeddingPort } from '../core/embedding.js';
-import type { MemoryAction } from '../memory/index.js';
 import type { SemanticLayerService } from '../sl/index.js';
-import type { TouchedSlSource } from '../tools/index.js';
 import type { MemoryFlowEventSink } from './memory-flow/types.js';
-import type { StageIndex } from './stages/stage-index.types.js';
-import type { WorkUnitOutcome } from './stages/stage-3-work-units.js';
 export type IngestTrigger = 'upload' | 'scheduled_pull' | 'manual_resync' | 'manual_override';
 export interface DiffSet {
     added: string[];
@@ -103,34 +99,6 @@ export interface ProjectionResult {
     changedWikiPageKeys: string[];
     result?: unknown;
 }
-export interface FinalizationOverrideReplay {
-    priorJobId: string;
-    priorRunId: string;
-    priorSyncId: string;
-    evictionRawPaths: string[];
-}
-export interface DeterministicFinalizationContext {
-    connectionId: string;
-    sourceKey: string;
-    syncId: string;
-    jobId: string;
-    runId: string;
-    stagedDir: string;
-    workdir: string;
-    parseArtifacts?: unknown;
-    stageIndex: StageIndex;
-    workUnitOutcomes: WorkUnitOutcome[];
-    reconciliationActions: MemoryAction[];
-    overrideReplay?: FinalizationOverrideReplay;
-}
-export interface FinalizationResult {
-    warnings: string[];
-    errors: string[];
-    touchedSources: TouchedSlSource[];
-    changedWikiPageKeys: string[];
-    actions?: MemoryAction[];
-    result?: unknown;
-}
 export interface SourceAdapter {
     readonly source: string;
     readonly skillNames: string[];
@@ -145,7 +113,6 @@ export interface SourceAdapter {
     chunk(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult>;
     clusterWorkUnits?(ctx: ClusterWorkUnitsContext): Promise<WorkUnit[]>;
     project?(ctx: DeterministicProjectionContext): Promise<ProjectionResult>;
-    finalize?(ctx: DeterministicFinalizationContext): Promise<FinalizationResult>;
     describeScope?(stagedDir: string): Promise<ScopeDescriptor>;
     onPullSucceeded?(ctx: {
         connectionId: string;

package/node_modules/@ktx/context/dist/package-exports.test.js

@@ -1,6 +1,5 @@
 import { describe, expect, it } from 'vitest';
 const scanTypeExportCoverage = {};
-const ingestFinalizationTypeExportCoverage = {};
 describe('@ktx/context package exports', () => {
     it('exports package entry points used by host adapters', async () => {
         const core = await import('./core/index.js');
@@ -32,7 +31,6 @@ describe('@ktx/context package exports', () => {
         expect(connections.notionConnectionToPullConfig).toBeTypeOf('function');
         expect(scan).toBeDefined();
         expect(scanTypeExportCoverage).toEqual({});
-        expect(ingestFinalizationTypeExportCoverage).toEqual({});
         expect(scan.createKtxConnectorCapabilities).toBeTypeOf('function');
         expect(`liveDatabaseSnapshotToKtx${'SchemaSnapshot'}` in scan).toBe(false);
         expect(scan.normalizeKtxNativeType).toBeTypeOf('function');
@@ -233,6 +231,7 @@ describe('@ktx/context package exports', () => {
         expect(ingest.historicSqlEvidenceEnvelopeSchema).toBeDefined();
         expect(ingest.historicSqlEvidencePath).toBeTypeOf('function');
         expect(ingest.createEmitHistoricSqlEvidenceTool).toBeTypeOf('function');
+        expect(ingest.HistoricSqlProjectionPostProcessor).toBeTypeOf('function');
         expect(ingest.SqliteContextEvidenceStore).toBeTypeOf('function');
         expect(ingest.SqliteBundleIngestStore).toBeTypeOf('function');
         expect(ingest.CuratorPaginationService).toBeTypeOf('function');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kaelio/ktx",
-  "version": "0.1.0-rc.6",
+  "version": "0.1.0",
   "description": "Standalone KTX context layer for database agents",
   "private": false,
   "type": "module",
@@ -71,10 +71,10 @@
   "license": "Apache-2.0",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/kaelio/ktx.git"
+    "url": "https://github.com/Kaelio/ktx"
   },
   "bugs": {
-    "url": "https://github.com/kaelio/ktx/issues"
+    "url": "https://github.com/Kaelio/ktx/issues"
   },
-  "homepage": "https://github.com/kaelio/ktx#readme"
+  "homepage": "https://github.com/Kaelio/ktx#readme"
 }

package/node_modules/@ktx/context/dist/ingest/finalization-scope.d.ts

@@ -1,22 +0,0 @@
-import type { SemanticLayerSource } from '../sl/index.js';
-import type { TouchedSlSource } from '../tools/index.js';
-import type { IngestReportFinalizationMismatch } from './reports.js';
-interface DeriveTouchedSourcesInput {
-    changedPaths: string[];
-    beforeSourcesByConnection: Map<string, SemanticLayerSource[]>;
-    afterSourcesByConnection: Map<string, SemanticLayerSource[]>;
-}
-interface DeriveTouchedSourcesResult {
-    touchedSources: TouchedSlSource[];
-    unresolvedPaths: string[];
-}
-interface CompareFinalizationDeclarationsInput {
-    declaredTouchedSources: TouchedSlSource[];
-    derivedTouchedSources: TouchedSlSource[];
-    declaredChangedWikiPageKeys: string[];
-    derivedChangedWikiPageKeys: string[];
-}
-export declare function deriveFinalizationWikiPageKeys(paths: string[]): string[];
-export declare function deriveFinalizationTouchedSources(input: DeriveTouchedSourcesInput): Promise<DeriveTouchedSourcesResult>;
-export declare function compareFinalizationDeclarations(input: CompareFinalizationDeclarationsInput): IngestReportFinalizationMismatch[];
-export {};

package/node_modules/@ktx/context/dist/ingest/finalization-scope.js

@@ -1,95 +0,0 @@
-function uniqueSorted(values) {
-    return [...new Set(values.filter((value) => value.length > 0))].sort();
-}
-function touchedKey(source) {
-    return `${source.connectionId}:${source.sourceName}`;
-}
-function stableJson(value) {
-    if (Array.isArray(value)) {
-        return `[${value.map((entry) => stableJson(entry)).join(',')}]`;
-    }
-    if (value && typeof value === 'object') {
-        const record = value;
-        return `{${Object.keys(record)
-            .sort()
-            .map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`)
-            .join(',')}}`;
-    }
-    return JSON.stringify(value);
-}
-function changedSourceNames(beforeSources, afterSources) {
-    const before = new Map(beforeSources.map((source) => [source.name, stableJson(source)]));
-    const after = new Map(afterSources.map((source) => [source.name, stableJson(source)]));
-    return uniqueSorted(uniqueSorted([...before.keys(), ...after.keys()]).filter((sourceName) => before.get(sourceName) !== after.get(sourceName)));
-}
-export function deriveFinalizationWikiPageKeys(paths) {
-    return uniqueSorted(paths
-        .filter((path) => path.startsWith('wiki/global/') && path.endsWith('.md'))
-        .filter((path) => !path.slice('wiki/global/'.length, -'.md'.length).includes('/'))
-        .map((path) => path.slice('wiki/global/'.length, -'.md'.length)));
-}
-export async function deriveFinalizationTouchedSources(input) {
-    const touched = new Map();
-    const unresolvedPaths = [];
-    for (const path of input.changedPaths) {
-        if (!path.startsWith('semantic-layer/') || !(path.endsWith('.yaml') || path.endsWith('.yml'))) {
-            continue;
-        }
-        const parts = path.split('/');
-        const connectionId = parts[1] ?? '';
-        if (!connectionId) {
-            unresolvedPaths.push(path);
-            continue;
-        }
-        if (parts[2] !== '_schema') {
-            const fileName = parts.at(-1) ?? '';
-            const sourceName = fileName.replace(/\.ya?ml$/, '');
-            if (!sourceName) {
-                unresolvedPaths.push(path);
-                continue;
-            }
-            touched.set(`${connectionId}:${sourceName}`, { connectionId, sourceName });
-            continue;
-        }
-        const changedNames = changedSourceNames(input.beforeSourcesByConnection.get(connectionId) ?? [], input.afterSourcesByConnection.get(connectionId) ?? []);
-        if (changedNames.length === 0) {
-            unresolvedPaths.push(path);
-            continue;
-        }
-        for (const sourceName of changedNames) {
-            touched.set(`${connectionId}:${sourceName}`, { connectionId, sourceName });
-        }
-    }
-    return {
-        touchedSources: [...touched.values()].sort((left, right) => touchedKey(left).localeCompare(touchedKey(right))),
-        unresolvedPaths: uniqueSorted(unresolvedPaths),
-    };
-}
-export function compareFinalizationDeclarations(input) {
-    const mismatches = [];
-    const declaredSl = new Set(input.declaredTouchedSources.map(touchedKey));
-    const derivedSl = new Set(input.derivedTouchedSources.map(touchedKey));
-    const declaredWiki = new Set(input.declaredChangedWikiPageKeys);
-    const derivedWiki = new Set(input.derivedChangedWikiPageKeys);
-    for (const key of [...derivedSl].sort()) {
-        if (!declaredSl.has(key)) {
-            mismatches.push({ artifactKind: 'sl', key, direction: 'missing_from_adapter_declaration' });
-        }
-    }
-    for (const key of [...declaredSl].sort()) {
-        if (!derivedSl.has(key)) {
-            mismatches.push({ artifactKind: 'sl', key, direction: 'extra_in_adapter_declaration' });
-        }
-    }
-    for (const key of [...derivedWiki].sort()) {
-        if (!declaredWiki.has(key)) {
-            mismatches.push({ artifactKind: 'wiki', key, direction: 'missing_from_adapter_declaration' });
-        }
-    }
-    for (const key of [...declaredWiki].sort()) {
-        if (!derivedWiki.has(key)) {
-            mismatches.push({ artifactKind: 'wiki', key, direction: 'extra_in_adapter_declaration' });
-        }
-    }
-    return mismatches;
-}

package/node_modules/@ktx/context/dist/ingest/finalization-scope.test.js

@@ -1,114 +0,0 @@
-import { describe, expect, it } from 'vitest';
-import { compareFinalizationDeclarations, deriveFinalizationTouchedSources, deriveFinalizationWikiPageKeys, } from './finalization-scope.js';
-describe('deriveFinalizationWikiPageKeys', () => {
-    it('maps changed global wiki markdown paths to page keys', () => {
-        expect(deriveFinalizationWikiPageKeys([
-            'wiki/global/historic-sql-orders.md',
-            'wiki/global/nested/page.md',
-            'README.md',
-        ])).toEqual(['historic-sql-orders']);
-    });
-});
-describe('deriveFinalizationTouchedSources', () => {
-    it('maps standalone semantic-layer files directly', async () => {
-        const result = await deriveFinalizationTouchedSources({
-            changedPaths: ['semantic-layer/warehouse/orders.yaml'],
-            beforeSourcesByConnection: new Map(),
-            afterSourcesByConnection: new Map(),
-        });
-        expect(result).toEqual({
-            touchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
-            unresolvedPaths: [],
-        });
-    });
-    it('resolves aggregate _schema changes by comparing loaded source snapshots', async () => {
-        const beforeSourcesByConnection = new Map([
-            [
-                'warehouse',
-                [
-                    {
-                        name: 'orders',
-                        grain: ['order_id'],
-                        columns: [{ name: 'order_id', type: 'string' }],
-                        joins: [],
-                        measures: [],
-                        usage: {
-                            narrative: 'old',
-                            frequencyTier: 'low',
-                            commonFilters: [],
-                            commonJoins: [],
-                        },
-                    },
-                ],
-            ],
-        ]);
-        const afterSourcesByConnection = new Map([
-            [
-                'warehouse',
-                [
-                    {
-                        name: 'orders',
-                        grain: ['order_id'],
-                        columns: [{ name: 'order_id', type: 'string' }],
-                        joins: [],
-                        measures: [],
-                        usage: {
-                            narrative: 'new',
-                            frequencyTier: 'high',
-                            commonFilters: [],
-                            commonJoins: [],
-                        },
-                    },
-                ],
-            ],
-        ]);
-        const result = await deriveFinalizationTouchedSources({
-            changedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
-            beforeSourcesByConnection,
-            afterSourcesByConnection,
-        });
-        expect(result).toEqual({
-            touchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
-            unresolvedPaths: [],
-        });
-    });
-    it('flags aggregate _schema changes that cannot be resolved to logical sources', async () => {
-        const beforeSourcesByConnection = new Map([['warehouse', []]]);
-        const afterSourcesByConnection = new Map([['warehouse', []]]);
-        const result = await deriveFinalizationTouchedSources({
-            changedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
-            beforeSourcesByConnection,
-            afterSourcesByConnection,
-        });
-        expect(result).toEqual({
-            touchedSources: [],
-            unresolvedPaths: ['semantic-layer/warehouse/_schema/public.yaml'],
-        });
-    });
-});
-describe('compareFinalizationDeclarations', () => {
-    it('reports missing and extra adapter declarations', () => {
-        expect(compareFinalizationDeclarations({
-            declaredTouchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
-            derivedTouchedSources: [{ connectionId: 'warehouse', sourceName: 'customers' }],
-            declaredChangedWikiPageKeys: ['orders'],
-            derivedChangedWikiPageKeys: ['orders', 'patterns'],
-        })).toEqual([
-            {
-                artifactKind: 'sl',
-                key: 'warehouse:customers',
-                direction: 'missing_from_adapter_declaration',
-            },
-            {
-                artifactKind: 'sl',
-                key: 'warehouse:orders',
-                direction: 'extra_in_adapter_declaration',
-            },
-            {
-                artifactKind: 'wiki',
-                key: 'patterns',
-                direction: 'missing_from_adapter_declaration',
-            },
-        ]);
-    });
-});