@kaelio/ktx 0.13.1 → 0.15.0

package/dist/context/scan/description-generation.js

@@ -1,4 +1,5 @@
 import { z } from 'zod';
+import { KtxSubprocessDeadlineError, runGenerateObjectInSubprocess, } from '../../context/llm/subprocess-generate-object.js';
 const batchedTableDescriptionSchema = z.object({
     tableDescription: z.string(),
     columns: z.array(z.object({
@@ -281,11 +282,13 @@ export class KtxDescriptionGenerator {
     logger;
     onWarning;
     settings;
+    spawnSubprocessGenerateChild;
     constructor(options) {
         this.llmRuntime = options.llmRuntime;
         this.cache = options.cache;
         this.logger = options.logger;
         this.onWarning = options.onWarning;
+        this.spawnSubprocessGenerateChild = options.spawnSubprocessGenerateChild;
         this.settings = {
             columnMaxWords: options.settings.columnMaxWords,
             tableMaxWords: options.settings.tableMaxWords,
@@ -500,6 +503,20 @@ export class KtxDescriptionGenerator {
         const descriptions = new Map();
         let tableDescription = null;
         let structuredGenerationSucceeded = false;
+        // Bound + retry the per-table enrichment LLM call. A transient backend error
+        // (e.g. an "overloaded"/burst rejection when many tables enrich concurrently)
+        // otherwise nulls a whole table's descriptions on the FIRST failure — sampleTable
+        // already retries, this call did not, so transient errors silently dropped most
+        // tables of a db. retryAsync gives it the same 3-attempt backoff. A FRESH timeout
+        // per attempt still bounds a wedged wide table (it never returns a result message);
+        // a timeout is surfaced as KtxAbortedError so retryAsync does NOT retry it (one
+        // wedge stays one timeout, not 3×). Tune via KTX_ENRICH_LLM_TIMEOUT_MS (default
+        // 120s) and KTX_ENRICH_LLM_ATTEMPTS (default 3).
+        const rawEnrichTimeoutMs = Number(process.env.KTX_ENRICH_LLM_TIMEOUT_MS);
+        const enrichTimeoutMs = Number.isFinite(rawEnrichTimeoutMs) && rawEnrichTimeoutMs > 0 ? rawEnrichTimeoutMs : 120_000;
+        const enrichAttempts = Math.max(1, Number(process.env.KTX_ENRICH_LLM_ATTEMPTS ?? 3) || 3);
+        let llmStartedAt = 0;
+        let lastTimedOut = false;
         try {
             const prompt = batchedPrompt({
                 table: input.table,
@@ -508,12 +525,78 @@ export class KtxDescriptionGenerator {
                 tableMaxWords: this.settings.tableMaxWords,
                 columnMaxWords: this.settings.columnMaxWords,
             });
-            const generated = await this.llmRuntime.generateObject({
-                role: 'candidateExtraction',
-                system: prompt.system,
-                prompt: prompt.user,
-                schema: batchedTableDescriptionSchema,
-                temperature: this.settings.temperature,
+            llmStartedAt = Date.now();
+            this.logger?.info(`[enrich] llm:start table=${input.table.name} cols=${input.table.columns.length} promptChars=${prompt.user.length} timeoutMs=${enrichTimeoutMs} attempts=${enrichAttempts}`, { connectorId: input.connector.id, table: input.table.name, columns: input.table.columns.length });
+            // Subprocess backends (codex/claude-code) own an SDK child that ignores the
+            // in-process abort, so each attempt runs behind a tree-killable boundary;
+            // HTTP backends keep the native abortSignal -> fetch cancellation.
+            const enrichForkSpec = this.llmRuntime.subprocessForkSpec();
+            const enrichJsonSchema = enrichForkSpec
+                ? z.toJSONSchema(batchedTableDescriptionSchema, { target: 'draft-7' })
+                : null;
+            const generated = await retryAsync(async () => {
+                if (enrichForkSpec && enrichJsonSchema) {
+                    try {
+                        return await runGenerateObjectInSubprocess({
+                            forkSpec: enrichForkSpec,
+                            role: 'candidateExtraction',
+                            system: prompt.system,
+                            prompt: prompt.user,
+                            schema: batchedTableDescriptionSchema,
+                            jsonSchema: enrichJsonSchema,
+                            deadlineMs: enrichTimeoutMs,
+                            ...(input.context.signal ? { signal: input.context.signal } : {}),
+                            ...(this.spawnSubprocessGenerateChild
+                                ? { spawnChild: this.spawnSubprocessGenerateChild }
+                                : {}),
+                        });
+                    }
+                    catch (error) {
+                        // The boundary tree-kills the wedged child on deadline; a per-table
+                        // timeout is not worth retrying (it would just time out again), so
+                        // surface it as KtxAbortedError so retryAsync stops immediately.
+                        if (error instanceof KtxSubprocessDeadlineError && !input.context.signal?.aborted) {
+                            lastTimedOut = true;
+                            throw new KtxAbortedError();
+                        }
+                        throw error;
+                    }
+                }
+                const enrichTimeout = AbortSignal.timeout(enrichTimeoutMs);
+                const abortSignal = input.context.signal
+                    ? AbortSignal.any([enrichTimeout, input.context.signal])
+                    : enrichTimeout;
+                try {
+                    return await this.llmRuntime.generateObject({
+                        role: 'candidateExtraction',
+                        system: prompt.system,
+                        prompt: prompt.user,
+                        schema: batchedTableDescriptionSchema,
+                        temperature: this.settings.temperature,
+                        abortSignal,
+                    });
+                }
+                catch (error) {
+                    // A per-table timeout is not worth retrying (it would just time out
+                    // again); surface it as KtxAbortedError so retryAsync stops immediately.
+                    // A genuine context cancellation is handled by retryAsync's own signal check.
+                    if (enrichTimeout.aborted && !input.context.signal?.aborted) {
+                        lastTimedOut = true;
+                        throw new KtxAbortedError();
+                    }
+                    throw error;
+                }
+            }, {
+                attempts: enrichAttempts,
+                baseDelayMs: 500,
+                ...(input.context.signal ? { signal: input.context.signal } : {}),
+                onAttemptFailure: (error, attempt) => {
+                    this.logger?.warn(`[enrich] llm:retry table=${input.table.name} attempt=${attempt}: ${errorMessage(error)}`, { connectorId: input.connector.id, table: input.table.name, attempt });
+                },
+            });
+            this.logger?.info(`[enrich] llm:done table=${input.table.name} ms=${Date.now() - llmStartedAt}`, {
+                connectorId: input.connector.id,
+                table: input.table.name,
             });
             structuredGenerationSucceeded = true;
             tableDescription = generated.tableDescription.trim() || null;
@@ -533,16 +616,20 @@ export class KtxDescriptionGenerator {
             }
         }
         catch (error) {
-            this.logger?.warn(`Batched table description failed for ${input.table.name}: ${errorMessage(error)}`, {
-                connectorId: input.connector.id,
-                table: input.table.name,
-            });
+            // A genuine cancellation propagates so the stage fails and resumes; a
+            // per-table timeout (context.signal not aborted) still degrades to null.
+            if (input.context.signal?.aborted) {
+                throw error;
+            }
+            const elapsedMs = llmStartedAt ? Date.now() - llmStartedAt : 0;
+            const timedOut = lastTimedOut;
+            this.logger?.warn(`[enrich] llm:${timedOut ? 'TIMEOUT' : 'fail'} table=${input.table.name} cols=${input.table.columns.length} ms=${elapsedMs}: ${errorMessage(error)}`, { connectorId: input.connector.id, table: input.table.name, timedOut, elapsedMs });
             this.onWarning?.({
-                code: 'enrichment_failed',
-                message: `Failed to generate batched description for table ${input.table.name}: ${errorMessage(error)}`,
+                code: timedOut ? 'enrichment_timeout' : 'enrichment_failed',
+                message: `${timedOut ? `Timed out after ${elapsedMs}ms generating` : 'Failed to generate'} batched description for table ${input.table.name}: ${errorMessage(error)}`,
                 table: input.table.name,
                 recoverable: true,
-                metadata: { connectorId: input.connector.id },
+                metadata: { connectorId: input.connector.id, ...(timedOut ? { timeoutMs: enrichTimeoutMs } : {}) },
             });
         }
         if (!structuredGenerationSucceeded) {

package/dist/context/scan/enabled-tables.d.ts

@@ -9,6 +9,11 @@ import type { KtxTableRef } from './types.js';
  *   "catalog.db.name"  — fully qualified
  *   "db.name"          — schema-qualified (catalog = null)
  *   "name"             — bare (catalog = db = null; SQLite-shape)
+ *
+ * SQLite exposes a single schema named `main` but the connector emits objects
+ * with `db: null`, so the `"main.<name>"` form is normalized to the bare shape
+ * to match. Both `"main.customers"` and `"customers"` therefore select the same
+ * object.
  */
 export declare function resolveEnabledTables(connection: Record<string, unknown> | undefined): ReadonlySet<KtxTableRefKey> | null;
 /** @internal */

package/dist/context/scan/enabled-tables.js

@@ -8,21 +8,33 @@ import { tableRefSet } from './table-ref.js';
  *   "catalog.db.name"  — fully qualified
  *   "db.name"          — schema-qualified (catalog = null)
  *   "name"             — bare (catalog = db = null; SQLite-shape)
+ *
+ * SQLite exposes a single schema named `main` but the connector emits objects
+ * with `db: null`, so the `"main.<name>"` form is normalized to the bare shape
+ * to match. Both `"main.customers"` and `"customers"` therefore select the same
+ * object.
  */
 export function resolveEnabledTables(connection) {
     const raw = connection?.enabled_tables;
     if (!Array.isArray(raw) || raw.length === 0)
         return null;
+    const driver = typeof connection?.driver === 'string' ? connection.driver : undefined;
     const refs = [];
     for (const value of raw) {
         const parsed = parseEnabledTableEntry(value);
         if (parsed)
-            refs.push(parsed);
+            refs.push(normalizeRefForDriver(parsed, driver));
     }
     if (refs.length === 0)
         return null;
     return tableRefSet(refs);
 }
+function normalizeRefForDriver(ref, driver) {
+    if (driver === 'sqlite' && ref.catalog === null && ref.db === 'main') {
+        return { catalog: null, db: null, name: ref.name };
+    }
+    return ref;
+}
 function parseEnabledTableEntry(value) {
     if (typeof value === 'string') {
         return parseDottedTableEntry(value);

package/dist/context/scan/enrichment-state.d.ts

@@ -1,6 +1,12 @@
+import type { KtxScanRelationshipConfig } from '../project/config.js';
 import type { KtxScanEnrichmentStage, KtxScanEnrichmentStateSummary, KtxScanMode, KtxSchemaSnapshot } from './types.js';
+/**
+ * Canonical enrichment-stage registry. The `--stages` CLI parser validates
+ * against this list, and stage selection / iteration derives its order here.
+ */
+export declare const KTX_SCAN_ENRICHMENT_STAGES: readonly KtxScanEnrichmentStage[];
 export interface KtxScanEnrichmentStageLookup {
-    runId: string;
+    connectionId: string;
     stage: KtxScanEnrichmentStage;
     inputHash: string;
 }
@@ -31,17 +37,54 @@ export interface KtxScanEnrichmentFailedStage {
 export type KtxScanEnrichmentStageRecord<TOutput = unknown> = KtxScanEnrichmentCompletedStage<TOutput> | KtxScanEnrichmentFailedStage;
 export interface KtxScanEnrichmentStateStore {
     findCompletedStage<TOutput = unknown>(input: KtxScanEnrichmentStageLookup): Promise<KtxScanEnrichmentCompletedStage<TOutput> | null>;
+    /**
+     * The most recently completed row for a (connection, stage) pair regardless of
+     * input hash. Used by the staleness check to compare a stage's stored hash
+     * against its freshly recomputed one (D4).
+     */
+    findLatestCompletedStage(input: {
+        connectionId: string;
+        stage: KtxScanEnrichmentStage;
+    }): Promise<KtxScanEnrichmentCompletedStage | null>;
     saveCompletedStage<TOutput = unknown>(input: Omit<KtxScanEnrichmentCompletedStage<TOutput>, 'status' | 'errorMessage'>): Promise<void>;
     saveFailedStage(input: Omit<KtxScanEnrichmentFailedStage, 'status' | 'output'>): Promise<void>;
     listRunStages(runId: string): Promise<KtxScanEnrichmentStageRecord[]>;
 }
-export interface ComputeKtxScanEnrichmentInputHashInput {
+/** Description-LLM identity: the inputs that change a description's content. */
+export interface KtxScanLlmIdentity {
+    model: string | null;
+    baseUrlConfigured: boolean;
+}
+/** Embedding-model identity: the inputs that change an embedding vector. */
+export interface KtxScanEmbeddingIdentity {
+    model: string | null;
+    dimensions: number | null;
+    batchSize: number | null;
+}
+export interface KtxDescriptionsStageHashInput {
     snapshot: KtxSchemaSnapshot;
-    mode: KtxScanMode;
-    detectRelationships: boolean;
-    providerIdentity: Record<string, unknown>;
-    relationshipSettings?: unknown;
+    llmIdentity: KtxScanLlmIdentity;
+}
+export interface KtxEmbeddingsStageHashInput {
+    snapshot: KtxSchemaSnapshot;
+    embeddingIdentity: KtxScanEmbeddingIdentity;
+    /** Digest of the resolved description text the embeddings consume (see {@link computeKtxScanDescriptionDigest}). */
+    descriptionDigest: string;
+}
+export interface KtxRelationshipsStageHashInput {
+    snapshot: KtxSchemaSnapshot;
+    relationshipSettings: KtxScanRelationshipConfig;
+    llmIdentity: KtxScanLlmIdentity;
 }
-export declare function computeKtxScanEnrichmentInputHash(input: ComputeKtxScanEnrichmentInputHashInput): string;
+export declare function computeKtxDescriptionsStageHash(input: KtxDescriptionsStageHashInput): string;
+export declare function computeKtxEmbeddingsStageHash(input: KtxEmbeddingsStageHashInput): string;
+export declare function computeKtxRelationshipsStageHash(input: KtxRelationshipsStageHashInput): string;
+/**
+ * Content digest of the resolved per-column description text the embeddings
+ * stage consumes. Folding it into the embeddings hash content-addresses
+ * embeddings on their real upstream, so re-describing busts only the embeddings
+ * that depend on the changed text (D4 self-healing).
+ */
+export declare function computeKtxScanDescriptionDigest(texts: readonly string[]): string;
 export declare function completedKtxScanEnrichmentStateSummary(): KtxScanEnrichmentStateSummary;
 export declare function summarizeKtxScanEnrichmentState(input: KtxScanEnrichmentStateSummary): KtxScanEnrichmentStateSummary;

package/dist/context/scan/enrichment-state.js

@@ -1,21 +1,38 @@
-import { createHash } from 'node:crypto';
-const KTX_SCAN_ENRICHMENT_STAGES = [
+import { stableContentHash } from '../cache/content-result-cache.js';
+/**
+ * Canonical enrichment-stage registry. The `--stages` CLI parser validates
+ * against this list, and stage selection / iteration derives its order here.
+ */
+export const KTX_SCAN_ENRICHMENT_STAGES = [
     'descriptions',
     'embeddings',
     'relationships',
 ];
-function stableJson(value) {
-    if (Array.isArray(value)) {
-        return `[${value.map(stableJson).join(',')}]`;
-    }
-    if (value && typeof value === 'object') {
-        const entries = Object.entries(value).sort(([left], [right]) => left.localeCompare(right));
-        return `{${entries.map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`).join(',')}}`;
-    }
-    return JSON.stringify(value);
+export function computeKtxDescriptionsStageHash(input) {
+    return stableContentHash({ snapshot: input.snapshot, llmIdentity: input.llmIdentity });
+}
+export function computeKtxEmbeddingsStageHash(input) {
+    return stableContentHash({
+        snapshot: input.snapshot,
+        embeddingIdentity: input.embeddingIdentity,
+        descriptionDigest: input.descriptionDigest,
+    });
+}
+export function computeKtxRelationshipsStageHash(input) {
+    return stableContentHash({
+        snapshot: input.snapshot,
+        relationshipSettings: input.relationshipSettings,
+        llmIdentity: input.llmIdentity,
+    });
 }
-export function computeKtxScanEnrichmentInputHash(input) {
-    return createHash('sha256').update(stableJson(input)).digest('hex');
+/**
+ * Content digest of the resolved per-column description text the embeddings
+ * stage consumes. Folding it into the embeddings hash content-addresses
+ * embeddings on their real upstream, so re-describing busts only the embeddings
+ * that depend on the changed text (D4 self-healing).
+ */
+export function computeKtxScanDescriptionDigest(texts) {
+    return stableContentHash(texts);
 }
 function uniqueStages(stages) {
     const seen = new Set();

package/dist/context/scan/local-enrichment-artifacts.d.ts

@@ -11,6 +11,12 @@ export interface WriteLocalScanManifestShardsInput {
     dryRun: boolean;
     descriptionUpdates?: KtxLocalScanEnrichmentResult['descriptionUpdates'];
     relationshipUpdate?: KtxLocalScanEnrichmentResult['relationshipUpdate'];
+    /**
+     * When set, write only the shards that contain one of these tables. All shards
+     * are still built (so merging preserves prior content); the unlisted shards are
+     * left untouched on disk. Used by the incremental flush to bound git commits.
+     */
+    onlyChangedTableNames?: ReadonlySet<string>;
 }
 export interface WriteLocalScanManifestShardsResult {
     manifestShards: string[];
@@ -28,5 +34,40 @@ export interface WriteLocalScanEnrichmentArtifactsInput {
 export interface WriteLocalScanEnrichmentArtifactsResult extends WriteLocalScanManifestShardsResult {
     enrichmentArtifacts: string[];
 }
+type LocalDescriptionUpdates = KtxLocalScanEnrichmentResult['descriptionUpdates'];
+/**
+ * Reconstructs the descriptions already persisted in the on-disk `_schema` as
+ * the in-memory `descriptionUpdates` shape, so a stage-selective run that skips
+ * the descriptions stage (e.g. `--stages relationships`/`--stages embeddings`)
+ * can still feed embeddings + relationships the prior AI descriptions. Tables or
+ * columns with no AI description carry `null`.
+ */
+export declare function loadOnDiskDescriptionUpdates(project: KtxLocalProject, connectionId: string, snapshot: KtxSchemaSnapshot): Promise<LocalDescriptionUpdates>;
+export interface KtxScanDescriptionResumeStore {
+    /** Prior enriched descriptions when the durable record matches `inputHash`, else null. */
+    load(inputHash: string): Promise<LocalDescriptionUpdates | null>;
+    /** Persist the descriptions so far + the manifest shards that gained a table this batch. */
+    flush(input: {
+        inputHash: string;
+        snapshot: KtxSchemaSnapshot;
+        descriptionUpdates: LocalDescriptionUpdates;
+        changedTableNames: ReadonlySet<string>;
+    }): Promise<void>;
+}
+export declare function createKtxScanDescriptionResumeStore(deps: {
+    project: KtxLocalProject;
+    connectionId: string;
+    syncId: string;
+    driver: KtxConnectionDriver;
+}): KtxScanDescriptionResumeStore;
 export declare function writeLocalScanManifestShards(input: WriteLocalScanManifestShardsInput): Promise<WriteLocalScanManifestShardsResult>;
+/**
+ * Promote the descriptions + embeddings into the queryable `_schema` manifest
+ * (and the raw enrichment artifacts) before relationship detection runs. The
+ * generated joins and the relationship diagnostics are deliberately left to the
+ * final write, so an interrupted relationship stage never loses the paid LLM
+ * enrichment and never emits empty relationship diagnostics.
+ */
+export declare function writeLocalScanEnrichmentCheckpoint(input: WriteLocalScanEnrichmentArtifactsInput): Promise<WriteLocalScanEnrichmentArtifactsResult>;
 export declare function writeLocalScanEnrichmentArtifacts(input: WriteLocalScanEnrichmentArtifactsInput): Promise<WriteLocalScanEnrichmentArtifactsResult>;
+export {};

package/dist/context/scan/local-enrichment-artifacts.js

@@ -1,7 +1,8 @@
 import YAML from 'yaml';
-import { buildLiveDatabaseManifestShards } from '../../context/ingest/adapters/live-database/manifest.js';
+import { buildLiveDatabaseManifestShards, buildTableRef } from '../../context/ingest/adapters/live-database/manifest.js';
 import { isSlYamlPath } from '../../context/sl/source-files.js';
 import { deriveFederatedConnection } from '../connections/federation.js';
+import { tableRefKey } from './table-ref.js';
 import { buildKtxRelationshipArtifacts, buildKtxRelationshipDiagnostics, emptyKtxRelationshipProfileArtifact, } from './relationship-diagnostics.js';
 const LIVE_DATABASE_ADAPTER = 'live-database';
 const LOCAL_AUTHOR = 'ktx';
@@ -19,8 +20,7 @@ function artifactDir(connectionId, syncId) {
 function schemaDir(connectionId) {
     return `${SL_DIR_PREFIX}/${connectionId}/${SCHEMA_DIR}`;
 }
-function tableDescription(table, descriptionUpdates = []) {
-    const update = descriptionUpdates.find((candidate) => candidate.table.name === table.name);
+function tableDescription(table, update) {
     const descriptions = {};
     if (table.comment) {
         descriptions.db = table.comment;
@@ -30,8 +30,7 @@ function tableDescription(table, descriptionUpdates = []) {
     }
     return Object.keys(descriptions).length > 0 ? descriptions : undefined;
 }
-function columnDescription(table, column, descriptionUpdates = []) {
-    const update = descriptionUpdates.find((candidate) => candidate.table.name === table.name);
+function columnDescription(column, update) {
     const aiDescription = update?.columnDescriptions[column.name] ?? null;
     const descriptions = {};
     if (column.comment) {
@@ -43,19 +42,25 @@ function columnDescription(table, column, descriptionUpdates = []) {
     return Object.keys(descriptions).length > 0 ? descriptions : undefined;
 }
 function snapshotTablesToManifestData(snapshot, descriptionUpdates = []) {
-    return snapshot.tables.map((table) => ({
-        name: table.name,
-        catalog: table.catalog,
-        db: table.db,
-        descriptions: tableDescription(table, descriptionUpdates),
-        columns: table.columns.map((column) => ({
-            name: column.name,
-            type: column.dimensionType,
-            ...(column.primaryKey ? { pk: true } : {}),
-            ...(column.nullable === false ? { nullable: false } : {}),
-            descriptions: columnDescription(table, column, descriptionUpdates),
-        })),
-    }));
+    // Resolve a table's descriptions by full identity: two same-named tables in
+    // different schemas must not collapse onto one update.
+    const updateByRef = new Map(descriptionUpdates.map((update) => [tableRefKey(update.table), update]));
+    return snapshot.tables.map((table) => {
+        const update = updateByRef.get(tableRefKey({ catalog: table.catalog, db: table.db, name: table.name }));
+        return {
+            name: table.name,
+            catalog: table.catalog,
+            db: table.db,
+            descriptions: tableDescription(table, update),
+            columns: table.columns.map((column) => ({
+                name: column.name,
+                type: column.dimensionType,
+                ...(column.primaryKey ? { pk: true } : {}),
+                ...(column.nullable === false ? { nullable: false } : {}),
+                descriptions: columnDescription(column, update),
+            })),
+        };
+    });
 }
 function formalJoins(snapshot) {
     const joins = [];
@@ -163,12 +168,15 @@ async function loadExistingManifestState(project, connectionId, snapshot, siblin
                 if (!validTableNames.has(tableName)) {
                     continue;
                 }
-                descriptions.set(tableName, {
+                // Descriptions/usage key on the fully-qualified `entry.table` ref so two
+                // same-named tables across schemas stay distinct; joins remain keyed by
+                // bare name to match the bare-name join graph.
+                descriptions.set(entry.table, {
                     table: entry.descriptions ? { ...entry.descriptions } : undefined,
                     columns: new Map((entry.columns ?? []).flatMap((column) => column.descriptions ? [[column.name, { ...column.descriptions }]] : [])),
                 });
                 if (entry.usage) {
-                    usage.set(tableName, { ...entry.usage });
+                    usage.set(entry.table, { ...entry.usage });
                 }
                 const joins = (entry.joins ?? []).filter((join) => {
                     return ((join.source === 'manual' || join.source === 'inferred') &&
@@ -186,6 +194,76 @@ async function loadExistingManifestState(project, connectionId, snapshot, siblin
     }
     return { descriptions, preservedJoins, usage };
 }
+/**
+ * Reconstructs the descriptions already persisted in the on-disk `_schema` as
+ * the in-memory `descriptionUpdates` shape, so a stage-selective run that skips
+ * the descriptions stage (e.g. `--stages relationships`/`--stages embeddings`)
+ * can still feed embeddings + relationships the prior AI descriptions. Tables or
+ * columns with no AI description carry `null`.
+ */
+export async function loadOnDiskDescriptionUpdates(project, connectionId, snapshot) {
+    const siblingTargets = await federatedSiblingTargets(project, connectionId);
+    const existing = await loadExistingManifestState(project, connectionId, snapshot, siblingTargets);
+    return snapshot.tables.map((table) => {
+        const entry = existing.descriptions.get(buildTableRef(table.name, table.catalog, table.db));
+        const columnDescriptions = {};
+        for (const column of table.columns) {
+            columnDescriptions[column.name] = entry?.columns.get(column.name)?.ai ?? null;
+        }
+        return {
+            table: { catalog: table.catalog, db: table.db, name: table.name },
+            tableDescription: entry?.table?.ai ?? null,
+            columnDescriptions,
+        };
+    });
+}
+// The incremental descriptions resume record. It lives at a stable, NON-syncId
+// path: a from-scratch interruption gets a fresh syncId on the next run, so a
+// syncId-scoped record would be unreachable on resume. The manifest already lives
+// at the same stable per-connection scope.
+function descriptionsProgressPath(connectionId) {
+    return `raw-sources/${connectionId}/${LIVE_DATABASE_ADAPTER}/enrichment-progress/descriptions.json`;
+}
+export function createKtxScanDescriptionResumeStore(deps) {
+    const path = descriptionsProgressPath(deps.connectionId);
+    return {
+        async load(inputHash) {
+            let content;
+            try {
+                ({ content } = await deps.project.fileStore.readFile(path));
+            }
+            catch {
+                return null;
+            }
+            try {
+                const record = JSON.parse(content);
+                // A changed inputHash (schema or enrichment settings changed) ignores the
+                // prior record and recomputes — spec-19's inputHash-gated resume semantics.
+                if (!record || record.inputHash !== inputHash || !Array.isArray(record.descriptions)) {
+                    return null;
+                }
+                return record.descriptions;
+            }
+            catch {
+                return null;
+            }
+        },
+        async flush({ inputHash, snapshot, descriptionUpdates, changedTableNames }) {
+            const record = { inputHash, descriptions: descriptionUpdates };
+            await writeJsonArtifact(deps.project, path, record, `scan(${LIVE_DATABASE_ADAPTER}): flush enrichment descriptions progress syncId=${deps.syncId}`);
+            await writeLocalScanManifestShards({
+                project: deps.project,
+                connectionId: deps.connectionId,
+                syncId: deps.syncId,
+                driver: deps.driver,
+                snapshot,
+                descriptionUpdates,
+                dryRun: false,
+                onlyChangedTableNames: changedTableNames,
+            });
+        },
+    };
+}
 async function writeJsonArtifact(project, path, value, commitMessage) {
     await project.fileStore.writeFile(path, `${JSON.stringify(value, null, 2)}\n`, LOCAL_AUTHOR, LOCAL_AUTHOR_EMAIL, commitMessage);
 }
@@ -210,6 +288,9 @@ export async function writeLocalScanManifestShards(input) {
     });
     const manifestShards = [];
     for (const [shardKey, shard] of [...shards.entries()].sort(([left], [right]) => left.localeCompare(right))) {
+        if (input.onlyChangedTableNames && !Object.keys(shard.tables).some((table) => input.onlyChangedTableNames.has(table))) {
+            continue;
+        }
         const path = `${schemaDir(input.connectionId)}/${shardKey}.yaml`;
         await input.project.fileStore.writeFile(path, YAML.stringify(shard, { indent: 2, lineWidth: 0, version: '1.1' }), LOCAL_AUTHOR, LOCAL_AUTHOR_EMAIL, `scan(${LIVE_DATABASE_ADAPTER}): write manifest shard ${shardKey} syncId=${input.syncId}`);
         manifestShards.push(path);
@@ -219,6 +300,53 @@ export async function writeLocalScanManifestShards(input) {
         manifestShardsWritten: manifestShards.length,
     };
 }
+async function writeEnrichmentDescriptionArtifacts(input) {
+    const descriptionsArtifact = `${input.enrichmentRoot}/descriptions.json`;
+    const embeddingsArtifact = `${input.enrichmentRoot}/embeddings.json`;
+    const enrichmentArtifacts = [];
+    if (input.enrichment.summary.tableDescriptions === 'completed' ||
+        input.enrichment.summary.columnDescriptions === 'completed') {
+        enrichmentArtifacts.push(descriptionsArtifact);
+        await writeJsonArtifact(input.project, descriptionsArtifact, input.enrichment.descriptionUpdates, `scan(${LIVE_DATABASE_ADAPTER}): write enrichment descriptions syncId=${input.syncId}`);
+    }
+    if (input.enrichment.summary.embeddings === 'completed') {
+        enrichmentArtifacts.push(embeddingsArtifact);
+        await writeJsonArtifact(input.project, embeddingsArtifact, input.enrichment.embeddingUpdates, `scan(${LIVE_DATABASE_ADAPTER}): write enrichment embeddings syncId=${input.syncId}`);
+    }
+    return enrichmentArtifacts;
+}
+/**
+ * Promote the descriptions + embeddings into the queryable `_schema` manifest
+ * (and the raw enrichment artifacts) before relationship detection runs. The
+ * generated joins and the relationship diagnostics are deliberately left to the
+ * final write, so an interrupted relationship stage never loses the paid LLM
+ * enrichment and never emits empty relationship diagnostics.
+ */
+export async function writeLocalScanEnrichmentCheckpoint(input) {
+    if (input.dryRun) {
+        return { enrichmentArtifacts: [], manifestShards: [], manifestShardsWritten: 0 };
+    }
+    const enrichmentArtifacts = await writeEnrichmentDescriptionArtifacts({
+        project: input.project,
+        enrichmentRoot: artifactDir(input.connectionId, input.syncId),
+        syncId: input.syncId,
+        enrichment: input.enrichment,
+    });
+    const manifestResult = await writeLocalScanManifestShards({
+        project: input.project,
+        connectionId: input.connectionId,
+        syncId: input.syncId,
+        driver: input.driver,
+        snapshot: input.enrichment.snapshot,
+        descriptionUpdates: input.enrichment.descriptionUpdates,
+        dryRun: false,
+    });
+    return {
+        enrichmentArtifacts,
+        manifestShards: manifestResult.manifestShards,
+        manifestShardsWritten: manifestResult.manifestShardsWritten,
+    };
+}
 export async function writeLocalScanEnrichmentArtifacts(input) {
     if (input.dryRun) {
         return {
@@ -228,21 +356,15 @@ export async function writeLocalScanEnrichmentArtifacts(input) {
         };
     }
     const enrichmentRoot = artifactDir(input.connectionId, input.syncId);
-    const descriptionsArtifact = `${enrichmentRoot}/descriptions.json`;
-    const embeddingsArtifact = `${enrichmentRoot}/embeddings.json`;
     const relationshipsArtifact = `${enrichmentRoot}/relationships.json`;
     const relationshipProfileArtifact = `${enrichmentRoot}/relationship-profile.json`;
     const relationshipDiagnosticsArtifact = `${enrichmentRoot}/relationship-diagnostics.json`;
-    const enrichmentArtifacts = [];
-    if (input.enrichment.summary.tableDescriptions === 'completed' ||
-        input.enrichment.summary.columnDescriptions === 'completed') {
-        enrichmentArtifacts.push(descriptionsArtifact);
-        await writeJsonArtifact(input.project, descriptionsArtifact, input.enrichment.descriptionUpdates, `scan(${LIVE_DATABASE_ADAPTER}): write enrichment descriptions syncId=${input.syncId}`);
-    }
-    if (input.enrichment.summary.embeddings === 'completed') {
-        enrichmentArtifacts.push(embeddingsArtifact);
-        await writeJsonArtifact(input.project, embeddingsArtifact, input.enrichment.embeddingUpdates, `scan(${LIVE_DATABASE_ADAPTER}): write enrichment embeddings syncId=${input.syncId}`);
-    }
+    const enrichmentArtifacts = await writeEnrichmentDescriptionArtifacts({
+        project: input.project,
+        enrichmentRoot,
+        syncId: input.syncId,
+        enrichment: input.enrichment,
+    });
     enrichmentArtifacts.push(relationshipsArtifact, relationshipProfileArtifact, relationshipDiagnosticsArtifact);
     const hasResolvedRelationships = input.enrichment.resolvedRelationships !== null;
     const relationshipArtifacts = buildKtxRelationshipArtifacts({
@@ -267,6 +389,7 @@ export async function writeLocalScanEnrichmentArtifacts(input) {
         artifacts: relationshipArtifacts,
         profile: relationshipProfile,
         warnings: input.enrichment.warnings,
+        partial: input.enrichment.relationshipPartial,
         thresholds: input.relationshipSettings
             ? {
                 acceptThreshold: input.relationshipSettings.acceptThreshold,