@kaelio/ktx 0.13.1 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.13.1-py3-none-any.whl → kaelio_ktx-0.15.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli-program.js +1 -1
- package/dist/commands/ingest-commands.d.ts +9 -0
- package/dist/commands/ingest-commands.js +37 -1
- package/dist/commands/knowledge-commands.js +3 -0
- package/dist/commands/setup-commands.js +15 -1
- package/dist/connection-drivers.d.ts +2 -0
- package/dist/connection-drivers.js +7 -3
- package/dist/connection.d.ts +3 -0
- package/dist/connection.js +27 -0
- package/dist/connectors/bigquery/connector.d.ts +17 -6
- package/dist/connectors/bigquery/connector.js +152 -76
- package/dist/connectors/bigquery/dialect.d.ts +2 -2
- package/dist/connectors/clickhouse/connector.d.ts +1 -0
- package/dist/connectors/clickhouse/connector.js +45 -16
- package/dist/connectors/clickhouse/dialect.d.ts +2 -2
- package/dist/connectors/mongodb/connector.d.ts +69 -0
- package/dist/connectors/mongodb/connector.js +307 -0
- package/dist/connectors/mongodb/dialect.d.ts +17 -0
- package/dist/connectors/mongodb/dialect.js +50 -0
- package/dist/connectors/mongodb/live-database-introspection.d.ts +10 -0
- package/dist/connectors/mongodb/live-database-introspection.js +24 -0
- package/dist/connectors/mongodb/schema-inference.d.ts +20 -0
- package/dist/connectors/mongodb/schema-inference.js +110 -0
- package/dist/connectors/mysql/connector.d.ts +1 -0
- package/dist/connectors/mysql/connector.js +18 -2
- package/dist/connectors/mysql/dialect.d.ts +2 -2
- package/dist/connectors/postgres/connector.d.ts +1 -0
- package/dist/connectors/postgres/connector.js +20 -3
- package/dist/connectors/postgres/dialect.d.ts +2 -2
- package/dist/connectors/snowflake/connector.d.ts +1 -0
- package/dist/connectors/snowflake/connector.js +27 -3
- package/dist/connectors/snowflake/dialect.d.ts +2 -2
- package/dist/connectors/sqlite/connector.d.ts +11 -1
- package/dist/connectors/sqlite/connector.js +120 -17
- package/dist/connectors/sqlite/dialect.d.ts +2 -2
- package/dist/connectors/sqlite/read-query-child.d.ts +1 -0
- package/dist/connectors/sqlite/read-query-child.js +23 -0
- package/dist/connectors/sqlserver/connector.d.ts +2 -0
- package/dist/connectors/sqlserver/connector.js +20 -3
- package/dist/connectors/sqlserver/dialect.d.ts +2 -2
- package/dist/context/cache/content-result-cache.d.ts +36 -0
- package/dist/context/cache/content-result-cache.js +14 -0
- package/dist/context/cache/sqlite-content-result-cache.d.ts +18 -0
- package/dist/context/cache/sqlite-content-result-cache.js +223 -0
- package/dist/context/connections/bigquery-identifiers.d.ts +1 -0
- package/dist/context/connections/bigquery-identifiers.js +7 -0
- package/dist/context/connections/configured-connections.d.ts +9 -0
- package/dist/context/connections/configured-connections.js +18 -0
- package/dist/context/connections/dialects.d.ts +30 -4
- package/dist/context/connections/dialects.js +38 -11
- package/dist/context/connections/drivers.js +21 -0
- package/dist/context/connections/gdrive-config.d.ts +19 -0
- package/dist/context/connections/gdrive-config.js +57 -0
- package/dist/context/connections/query-deadline.d.ts +31 -0
- package/dist/context/connections/query-deadline.js +37 -0
- package/dist/context/ingest/adapters/gdrive/chunk.d.ts +3 -0
- package/dist/context/ingest/adapters/gdrive/chunk.js +74 -0
- package/dist/context/ingest/adapters/gdrive/detect.d.ts +1 -0
- package/dist/context/ingest/adapters/gdrive/detect.js +20 -0
- package/dist/context/ingest/adapters/gdrive/fetch.d.ts +6 -0
- package/dist/context/ingest/adapters/gdrive/fetch.js +95 -0
- package/dist/context/ingest/adapters/gdrive/gdrive-client.d.ts +30 -0
- package/dist/context/ingest/adapters/gdrive/gdrive-client.js +132 -0
- package/dist/context/ingest/adapters/gdrive/gdrive.adapter.d.ts +11 -0
- package/dist/context/ingest/adapters/gdrive/gdrive.adapter.js +27 -0
- package/dist/context/ingest/adapters/gdrive/normalize.d.ts +2 -0
- package/dist/context/ingest/adapters/gdrive/normalize.js +256 -0
- package/dist/context/ingest/adapters/gdrive/types.d.ts +153 -0
- package/dist/context/ingest/adapters/gdrive/types.js +36 -0
- package/dist/context/ingest/adapters/live-database/daemon-introspection.js +24 -0
- package/dist/context/ingest/adapters/live-database/fetch-report.d.ts +8 -0
- package/dist/context/ingest/adapters/live-database/fetch-report.js +37 -0
- package/dist/context/ingest/adapters/live-database/live-database.adapter.d.ts +2 -1
- package/dist/context/ingest/adapters/live-database/live-database.adapter.js +9 -2
- package/dist/context/ingest/adapters/live-database/manifest.d.ts +2 -0
- package/dist/context/ingest/adapters/live-database/manifest.js +8 -4
- package/dist/context/ingest/adapters/live-database/scan-outcome.d.ts +17 -0
- package/dist/context/ingest/adapters/live-database/scan-outcome.js +40 -0
- package/dist/context/ingest/adapters/live-database/stage.js +5 -5
- package/dist/context/ingest/adapters/metabase/types.d.ts +1 -1
- package/dist/context/ingest/adapters/metabase/types.js +1 -1
- package/dist/context/ingest/adapters/sigma/chunk.d.ts +6 -0
- package/dist/context/ingest/adapters/sigma/chunk.js +119 -0
- package/dist/context/ingest/adapters/sigma/client-port.d.ts +45 -0
- package/dist/context/ingest/adapters/sigma/client-port.js +1 -0
- package/dist/context/ingest/adapters/sigma/client.d.ts +33 -0
- package/dist/context/ingest/adapters/sigma/client.js +176 -0
- package/dist/context/ingest/adapters/sigma/detect.d.ts +1 -0
- package/dist/context/ingest/adapters/sigma/detect.js +23 -0
- package/dist/context/ingest/adapters/sigma/fetch.d.ts +14 -0
- package/dist/context/ingest/adapters/sigma/fetch.js +191 -0
- package/dist/context/ingest/adapters/sigma/local-sigma.adapter.d.ts +17 -0
- package/dist/context/ingest/adapters/sigma/local-sigma.adapter.js +41 -0
- package/dist/context/ingest/adapters/sigma/project.d.ts +8 -0
- package/dist/context/ingest/adapters/sigma/project.js +186 -0
- package/dist/context/ingest/adapters/sigma/sigma.adapter.d.ts +18 -0
- package/dist/context/ingest/adapters/sigma/sigma.adapter.js +43 -0
- package/dist/context/ingest/adapters/sigma/types.d.ts +80 -0
- package/dist/context/ingest/adapters/sigma/types.js +82 -0
- package/dist/context/ingest/artifact-gates.d.ts +32 -1
- package/dist/context/ingest/artifact-gates.js +85 -18
- package/dist/context/ingest/final-gate-prune.d.ts +35 -0
- package/dist/context/ingest/final-gate-prune.js +229 -0
- package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
- package/dist/context/ingest/ingest-bundle.runner.js +459 -240
- package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +0 -11
- package/dist/context/ingest/isolated-diff/patch-integrator.js +0 -44
- package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
- package/dist/context/ingest/isolated-diff/work-unit-executor.js +13 -4
- package/dist/context/ingest/local-adapters.d.ts +2 -1
- package/dist/context/ingest/local-adapters.js +28 -0
- package/dist/context/ingest/local-bundle-runtime.js +7 -2
- package/dist/context/ingest/local-ingest.js +1 -1
- package/dist/context/ingest/local-stage-ingest.d.ts +3 -1
- package/dist/context/ingest/local-stage-ingest.js +3 -1
- package/dist/context/ingest/ports.d.ts +3 -0
- package/dist/context/ingest/report-snapshot.js +13 -3
- package/dist/context/ingest/reports.d.ts +3 -3
- package/dist/context/ingest/reports.js +3 -1
- package/dist/context/ingest/stages/build-wu-context.d.ts +1 -0
- package/dist/context/ingest/stages/build-wu-context.js +2 -1
- package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -1
- package/dist/context/ingest/stages/stage-3-work-units.js +4 -10
- package/dist/context/ingest/stages/validate-wu-sources.d.ts +12 -0
- package/dist/context/ingest/stages/validate-wu-sources.js +23 -8
- package/dist/context/ingest/tools/read-raw-file.tool.js +10 -5
- package/dist/context/ingest/tools/read-raw-span.tool.js +10 -5
- package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/entity-details.tool.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +1 -1
- package/dist/context/ingest/types.d.ts +3 -0
- package/dist/context/ingest/wiki-body-refs.d.ts +28 -0
- package/dist/context/ingest/wiki-body-refs.js +37 -8
- package/dist/context/ingest/wiki-sl-ref-repair.d.ts +1 -0
- package/dist/context/ingest/wiki-sl-ref-repair.js +4 -0
- package/dist/context/ingest/work-unit-cache.d.ts +67 -0
- package/dist/context/ingest/work-unit-cache.js +230 -0
- package/dist/context/llm/ai-sdk-runtime.d.ts +1 -0
- package/dist/context/llm/ai-sdk-runtime.js +5 -0
- package/dist/context/llm/claude-code-runtime.d.ts +4 -1
- package/dist/context/llm/claude-code-runtime.js +38 -8
- package/dist/context/llm/codex-runtime.d.ts +4 -1
- package/dist/context/llm/codex-runtime.js +48 -35
- package/dist/context/llm/runtime-port.d.ts +26 -0
- package/dist/context/llm/subprocess-generate-object-child.d.ts +1 -0
- package/dist/context/llm/subprocess-generate-object-child.js +34 -0
- package/dist/context/llm/subprocess-generate-object.d.ts +42 -0
- package/dist/context/llm/subprocess-generate-object.js +122 -0
- package/dist/context/mcp/context-tools.d.ts +2 -0
- package/dist/context/mcp/context-tools.js +111 -17
- package/dist/context/mcp/local-project-ports.d.ts +7 -0
- package/dist/context/mcp/local-project-ports.js +29 -0
- package/dist/context/mcp/logger.d.ts +24 -0
- package/dist/context/mcp/logger.js +49 -0
- package/dist/context/mcp/server.js +2 -0
- package/dist/context/mcp/types.d.ts +17 -0
- package/dist/context/memory/memory-agent.service.js +1 -1
- package/dist/context/project/config.d.ts +72 -0
- package/dist/context/project/config.js +5 -0
- package/dist/context/project/driver-schemas.d.ts +35 -0
- package/dist/context/project/driver-schemas.js +89 -1
- package/dist/context/project/project.js +1 -1
- package/dist/context/project/setup-config.js +1 -0
- package/dist/context/scan/description-generation.d.ts +4 -0
- package/dist/context/scan/description-generation.js +100 -13
- package/dist/context/scan/enabled-tables.d.ts +5 -0
- package/dist/context/scan/enabled-tables.js +13 -1
- package/dist/context/scan/enrichment-state.d.ts +50 -7
- package/dist/context/scan/enrichment-state.js +30 -13
- package/dist/context/scan/local-enrichment-artifacts.d.ts +41 -0
- package/dist/context/scan/local-enrichment-artifacts.js +155 -32
- package/dist/context/scan/local-enrichment.d.ts +39 -4
- package/dist/context/scan/local-enrichment.js +345 -92
- package/dist/context/scan/local-scan.d.ts +4 -1
- package/dist/context/scan/local-scan.js +54 -13
- package/dist/context/scan/local-structural-artifacts.d.ts +3 -1
- package/dist/context/scan/local-structural-artifacts.js +5 -0
- package/dist/context/scan/object-introspection.d.ts +21 -0
- package/dist/context/scan/object-introspection.js +35 -0
- package/dist/context/scan/relationship-benchmarks.js +5 -3
- package/dist/context/scan/relationship-composite-candidates.d.ts +6 -3
- package/dist/context/scan/relationship-composite-candidates.js +13 -1
- package/dist/context/scan/relationship-detection-budget.d.ts +35 -0
- package/dist/context/scan/relationship-detection-budget.js +53 -0
- package/dist/context/scan/relationship-diagnostics.d.ts +5 -0
- package/dist/context/scan/relationship-diagnostics.js +2 -0
- package/dist/context/scan/relationship-discovery.d.ts +9 -3
- package/dist/context/scan/relationship-discovery.js +27 -4
- package/dist/context/scan/relationship-llm-proposal.js +36 -27
- package/dist/context/scan/relationship-profiling.d.ts +7 -3
- package/dist/context/scan/relationship-profiling.js +53 -41
- package/dist/context/scan/relationship-validation.d.ts +6 -4
- package/dist/context/scan/relationship-validation.js +46 -32
- package/dist/context/scan/sqlite-local-enrichment-state-store.d.ts +8 -1
- package/dist/context/scan/sqlite-local-enrichment-state-store.js +71 -159
- package/dist/context/scan/types.d.ts +5 -3
- package/dist/context/scan/types.js +2 -0
- package/dist/context/sl/local-query.js +5 -0
- package/dist/context/sl/pglite-sl-search-prototype.js +1 -1
- package/dist/context/sl/semantic-layer.service.js +1 -1
- package/dist/context/sl/source-files.d.ts +5 -0
- package/dist/context/sl/source-files.js +17 -11
- package/dist/context/sl/tools/connection-id-schema.js +1 -1
- package/dist/context/sql-analysis/dialect-notes.d.ts +9 -0
- package/dist/context/sql-analysis/dialect-notes.js +40 -0
- package/dist/context/sql-analysis/dialects/bigquery.md +13 -0
- package/dist/context/sql-analysis/dialects/clickhouse.md +9 -0
- package/dist/context/sql-analysis/dialects/mysql.md +9 -0
- package/dist/context/sql-analysis/dialects/postgres.md +10 -0
- package/dist/context/sql-analysis/dialects/snowflake.md +10 -0
- package/dist/context/sql-analysis/dialects/sqlite.md +11 -0
- package/dist/context/sql-analysis/dialects/tsql.md +10 -0
- package/dist/context/wiki/keys.js +1 -1
- package/dist/context/wiki/knowledge-wiki.service.d.ts +4 -4
- package/dist/context/wiki/knowledge-wiki.service.js +2 -1
- package/dist/context/wiki/local-knowledge.d.ts +13 -0
- package/dist/context/wiki/local-knowledge.js +50 -6
- package/dist/context/wiki/sqlite-knowledge-index.d.ts +2 -0
- package/dist/context/wiki/sqlite-knowledge-index.js +21 -5
- package/dist/context/wiki/tools/wiki-write.tool.d.ts +2 -0
- package/dist/context/wiki/tools/wiki-write.tool.js +27 -0
- package/dist/context/wiki/types.d.ts +6 -0
- package/dist/context-build-view.d.ts +3 -0
- package/dist/context-build-view.js +1 -0
- package/dist/knowledge.d.ts +2 -0
- package/dist/knowledge.js +12 -1
- package/dist/local-adapters.js +11 -0
- package/dist/mcp-http-server.js +15 -1
- package/dist/mcp-server-factory.d.ts +2 -0
- package/dist/mcp-server-factory.js +15 -1
- package/dist/mcp-stdio-server.js +10 -2
- package/dist/notion-page-picker.js +1 -1
- package/dist/prompts/memory_agent_external_ingest.md +2 -0
- package/dist/public-ingest.d.ts +3 -1
- package/dist/public-ingest.js +4 -0
- package/dist/scan.d.ts +3 -1
- package/dist/scan.js +7 -0
- package/dist/setup-databases.d.ts +1 -1
- package/dist/setup-databases.js +43 -1
- package/dist/setup-sources.d.ts +6 -1
- package/dist/setup-sources.js +208 -48
- package/dist/setup.d.ts +3 -0
- package/dist/setup.js +6 -1
- package/dist/skills/analytics/SKILL.md +200 -2
- package/dist/skills/gdrive_synthesize/SKILL.md +97 -0
- package/dist/skills/sigma_ingest/SKILL.md +189 -0
- package/dist/skills/wiki_capture/SKILL.md +24 -0
- package/dist/sql.js +4 -0
- package/dist/status-project.d.ts +4 -0
- package/dist/status-project.js +54 -2
- package/dist/telemetry/events.d.ts +2 -2
- package/dist/text-ingest.d.ts +4 -0
- package/dist/text-ingest.js +58 -29
- package/dist/verbatim-ingest.d.ts +46 -0
- package/dist/verbatim-ingest.js +193 -0
- package/package.json +5 -1
- package/dist/context/ingest/final-gate-repair.d.ts +0 -28
- package/dist/context/ingest/final-gate-repair.js +0 -91
|
@@ -5,10 +5,10 @@ import { KtxScanEmbeddingPortAdapter } from '../../context/llm/embedding-port.js
|
|
|
5
5
|
import { ktxLocalStateDbPath } from '../project/local-state-db.js';
|
|
6
6
|
import { redactKtxScanReport } from './credentials.js';
|
|
7
7
|
import { resolveEnabledTables } from './enabled-tables.js';
|
|
8
|
-
import { completedKtxScanEnrichmentStateSummary } from './enrichment-state.js';
|
|
8
|
+
import { completedKtxScanEnrichmentStateSummary, } from './enrichment-state.js';
|
|
9
9
|
import { failedKtxScanEnrichmentSummary, ktxScanErrorMessage } from './enrichment-summary.js';
|
|
10
10
|
import { createDeterministicLocalScanEnrichmentProviders, runLocalScanEnrichment, } from './local-enrichment.js';
|
|
11
|
-
import { writeLocalScanEnrichmentArtifacts, writeLocalScanManifestShards } from './local-enrichment-artifacts.js';
|
|
11
|
+
import { createKtxScanDescriptionResumeStore, loadOnDiskDescriptionUpdates, writeLocalScanEnrichmentArtifacts, writeLocalScanEnrichmentCheckpoint, writeLocalScanManifestShards, } from './local-enrichment-artifacts.js';
|
|
12
12
|
import { readLocalScanStructuralSnapshot } from './local-structural-artifacts.js';
|
|
13
13
|
import { SqliteLocalScanEnrichmentStateStore } from './sqlite-local-enrichment-state-store.js';
|
|
14
14
|
function enrichmentResolutionWarning(status) {
|
|
@@ -47,10 +47,11 @@ function normalizeDriver(driver) {
|
|
|
47
47
|
normalized === 'clickhouse' ||
|
|
48
48
|
normalized === 'sqlserver' ||
|
|
49
49
|
normalized === 'bigquery' ||
|
|
50
|
-
normalized === 'snowflake'
|
|
50
|
+
normalized === 'snowflake' ||
|
|
51
|
+
normalized === 'mongodb') {
|
|
51
52
|
return normalized;
|
|
52
53
|
}
|
|
53
|
-
throw new Error(`Standalone ktx scan supports postgres/sqlite/mysql/clickhouse/sqlserver/bigquery/snowflake in this phase, received "${driver ?? 'unknown'}"`);
|
|
54
|
+
throw new Error(`Standalone ktx scan supports postgres/sqlite/mysql/clickhouse/sqlserver/bigquery/snowflake/mongodb in this phase, received "${driver ?? 'unknown'}"`);
|
|
54
55
|
}
|
|
55
56
|
function tablePathCount(paths) {
|
|
56
57
|
return paths.filter((path) => path.startsWith('tables/') && path.endsWith('.json')).length;
|
|
@@ -116,15 +117,17 @@ function createLocalScanEnrichmentStateStore(options) {
|
|
|
116
117
|
}
|
|
117
118
|
return new SqliteLocalScanEnrichmentStateStore({ dbPath: ktxLocalStateDbPath(options.project) });
|
|
118
119
|
}
|
|
119
|
-
function
|
|
120
|
+
function localScanLlmIdentity(llmConfig) {
|
|
120
121
|
return {
|
|
121
|
-
|
|
122
|
-
embeddingDimensions: config.embeddings?.dimensions ?? null,
|
|
123
|
-
llmModel: llmConfig.models.default ?? null,
|
|
124
|
-
embeddingModel: config.embeddings?.model ?? null,
|
|
125
|
-
batchSize: config.embeddings?.batchSize ?? null,
|
|
122
|
+
model: llmConfig.models.default ?? null,
|
|
126
123
|
baseUrlConfigured: Boolean(llmConfig.provider.gateway?.base_url),
|
|
127
|
-
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
function localScanEmbeddingIdentity(config) {
|
|
127
|
+
return {
|
|
128
|
+
model: config.embeddings?.model ?? null,
|
|
129
|
+
dimensions: config.embeddings?.dimensions ?? null,
|
|
130
|
+
batchSize: config.embeddings?.batchSize ?? null,
|
|
128
131
|
};
|
|
129
132
|
}
|
|
130
133
|
function reportFromIngest(input) {
|
|
@@ -299,6 +302,13 @@ export async function runLocalScan(options) {
|
|
|
299
302
|
const enrichmentStateStore = connector ? createLocalScanEnrichmentStateStore(options) : null;
|
|
300
303
|
let enrichmentState = completedKtxScanEnrichmentStateSummary();
|
|
301
304
|
let enrichmentSnapshot = null;
|
|
305
|
+
// On a `--stages` subset run, the structural manifest write below (and the
|
|
306
|
+
// later enrichment write) merge with on-disk shards, but the merge treats ai/db
|
|
307
|
+
// descriptions as scan-managed and overwrites them with whatever the run emits.
|
|
308
|
+
// A subset that skips `descriptions` emits none, so without this the structural
|
|
309
|
+
// write would delete the prior descriptions before enrichment can preserve them.
|
|
310
|
+
// Capture them up front (only for subset runs) and feed them to both writes.
|
|
311
|
+
let priorDescriptionUpdates = null;
|
|
302
312
|
if (!reusedExistingScanArtifacts && !report.dryRun && report.artifactPaths.rawSourcesDir) {
|
|
303
313
|
await options.progress?.update(0.7, 'Writing schema artifacts');
|
|
304
314
|
const rawSnapshot = await readLocalScanStructuralSnapshot({
|
|
@@ -312,12 +322,16 @@ export async function runLocalScan(options) {
|
|
|
312
322
|
if (rawSnapshot.warnings?.length) {
|
|
313
323
|
report.warnings.push(...rawSnapshot.warnings);
|
|
314
324
|
}
|
|
325
|
+
if (options.stages !== undefined && connector) {
|
|
326
|
+
priorDescriptionUpdates = await loadOnDiskDescriptionUpdates(options.project, options.connectionId, rawSnapshot);
|
|
327
|
+
}
|
|
315
328
|
const manifestArtifacts = await writeLocalScanManifestShards({
|
|
316
329
|
project: options.project,
|
|
317
330
|
connectionId: options.connectionId,
|
|
318
331
|
syncId: record.syncId,
|
|
319
332
|
driver,
|
|
320
333
|
snapshot: rawSnapshot,
|
|
334
|
+
...(priorDescriptionUpdates ? { descriptionUpdates: priorDescriptionUpdates } : {}),
|
|
321
335
|
dryRun: false,
|
|
322
336
|
});
|
|
323
337
|
report.artifactPaths.manifestShards = manifestArtifacts.manifestShards;
|
|
@@ -332,15 +346,42 @@ export async function runLocalScan(options) {
|
|
|
332
346
|
connectionId: options.connectionId,
|
|
333
347
|
mode,
|
|
334
348
|
detectRelationships: options.detectRelationships,
|
|
349
|
+
...(options.stages ? { stages: options.stages } : {}),
|
|
335
350
|
connector,
|
|
336
351
|
...(enrichmentSnapshot ? { snapshot: enrichmentSnapshot } : {}),
|
|
337
|
-
context: {
|
|
352
|
+
context: {
|
|
353
|
+
runId: record.runId,
|
|
354
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
355
|
+
...(options.progress ? { progress: options.progress.startPhase(0.18) } : {}),
|
|
356
|
+
},
|
|
338
357
|
providers: enrichmentProviders,
|
|
339
358
|
stateStore: enrichmentStateStore,
|
|
359
|
+
descriptionResumeStore: options.dryRun
|
|
360
|
+
? null
|
|
361
|
+
: createKtxScanDescriptionResumeStore({
|
|
362
|
+
project: options.project,
|
|
363
|
+
connectionId: options.connectionId,
|
|
364
|
+
syncId: record.syncId,
|
|
365
|
+
driver,
|
|
366
|
+
}),
|
|
340
367
|
syncId: record.syncId,
|
|
341
|
-
|
|
368
|
+
loadPriorDescriptions: (enrichedSnapshot) => priorDescriptionUpdates
|
|
369
|
+
? Promise.resolve(priorDescriptionUpdates)
|
|
370
|
+
: loadOnDiskDescriptionUpdates(options.project, options.connectionId, enrichedSnapshot),
|
|
371
|
+
llmIdentity: localScanLlmIdentity(options.project.config.llm),
|
|
372
|
+
embeddingIdentity: localScanEmbeddingIdentity(options.project.config.scan.enrichment),
|
|
342
373
|
relationshipSettings: options.project.config.scan.relationships,
|
|
343
374
|
now: options.now,
|
|
375
|
+
onCheckpoint: async (checkpoint) => {
|
|
376
|
+
await writeLocalScanEnrichmentCheckpoint({
|
|
377
|
+
project: options.project,
|
|
378
|
+
connectionId: options.connectionId,
|
|
379
|
+
syncId: record.syncId,
|
|
380
|
+
driver,
|
|
381
|
+
enrichment: checkpoint,
|
|
382
|
+
dryRun: options.dryRun ?? false,
|
|
383
|
+
});
|
|
384
|
+
},
|
|
344
385
|
});
|
|
345
386
|
const artifacts = await writeLocalScanEnrichmentArtifacts({
|
|
346
387
|
project: options.project,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { KtxLocalProject } from '../../context/project/project.js';
|
|
2
|
-
import type { KtxConnectionDriver, KtxSchemaSnapshot } from './types.js';
|
|
2
|
+
import type { KtxConnectionDriver, KtxScanWarning, KtxSchemaSnapshot } from './types.js';
|
|
3
3
|
export interface ReadLocalScanStructuralSnapshotInput {
|
|
4
4
|
project: KtxLocalProject;
|
|
5
5
|
connectionId: string;
|
|
@@ -7,4 +7,6 @@ export interface ReadLocalScanStructuralSnapshotInput {
|
|
|
7
7
|
rawSourcesDir: string;
|
|
8
8
|
extractedAtFallback: string;
|
|
9
9
|
}
|
|
10
|
+
/** @internal */
|
|
11
|
+
export declare function isKtxScanWarningCode(code: string): code is KtxScanWarning['code'];
|
|
10
12
|
export declare function readLocalScanStructuralSnapshot(input: ReadLocalScanStructuralSnapshotInput): Promise<KtxSchemaSnapshot>;
|
|
@@ -18,7 +18,12 @@ const scanWarningCodes = new Set([
|
|
|
18
18
|
'enrichment_failed',
|
|
19
19
|
'description_fallback_used',
|
|
20
20
|
'constraint_discovery_unauthorized',
|
|
21
|
+
'object_introspection_failed',
|
|
21
22
|
]);
|
|
23
|
+
/** @internal */
|
|
24
|
+
export function isKtxScanWarningCode(code) {
|
|
25
|
+
return scanWarningCodes.has(code);
|
|
26
|
+
}
|
|
22
27
|
function parseWarning(rawWarning, path) {
|
|
23
28
|
if (!isRecord(rawWarning) ||
|
|
24
29
|
typeof rawWarning.code !== 'string' ||
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { KtxScanWarning } from './types.js';
|
|
2
|
+
export interface IntrospectObjectContext {
|
|
3
|
+
/** Bare object name (table or view). */
|
|
4
|
+
object: string;
|
|
5
|
+
catalog?: string | null;
|
|
6
|
+
db?: string | null;
|
|
7
|
+
}
|
|
8
|
+
export type IntrospectObjectOutcome<T> = {
|
|
9
|
+
ok: true;
|
|
10
|
+
table: T;
|
|
11
|
+
} | {
|
|
12
|
+
ok: false;
|
|
13
|
+
warning: KtxScanWarning;
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Runs a single-object metadata/profiling read and isolates its failure: a
|
|
17
|
+
* broken or inaccessible object becomes a recoverable warning instead of
|
|
18
|
+
* aborting the whole scan. Native programming faults (a ktx bug, not a broken
|
|
19
|
+
* object) still propagate so they are not masked as object skips.
|
|
20
|
+
*/
|
|
21
|
+
export declare function tryIntrospectObject<T>(ctx: IntrospectObjectContext, fn: () => T | Promise<T>): Promise<IntrospectObjectOutcome<T>>;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { isNativeProgrammingFault } from '../../errors.js';
|
|
2
|
+
function objectLabel(ctx) {
|
|
3
|
+
return [ctx.catalog, ctx.db, ctx.object].filter((part) => Boolean(part)).join('.');
|
|
4
|
+
}
|
|
5
|
+
function objectIntrospectionWarning(ctx, error) {
|
|
6
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
7
|
+
return {
|
|
8
|
+
code: 'object_introspection_failed',
|
|
9
|
+
message: reason,
|
|
10
|
+
table: ctx.object,
|
|
11
|
+
recoverable: true,
|
|
12
|
+
metadata: {
|
|
13
|
+
object: objectLabel(ctx),
|
|
14
|
+
...(ctx.db ? { db: ctx.db } : {}),
|
|
15
|
+
...(ctx.catalog ? { catalog: ctx.catalog } : {}),
|
|
16
|
+
},
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Runs a single-object metadata/profiling read and isolates its failure: a
|
|
21
|
+
* broken or inaccessible object becomes a recoverable warning instead of
|
|
22
|
+
* aborting the whole scan. Native programming faults (a ktx bug, not a broken
|
|
23
|
+
* object) still propagate so they are not masked as object skips.
|
|
24
|
+
*/
|
|
25
|
+
export async function tryIntrospectObject(ctx, fn) {
|
|
26
|
+
try {
|
|
27
|
+
return { ok: true, table: await fn() };
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
if (isNativeProgrammingFault(error)) {
|
|
31
|
+
throw error;
|
|
32
|
+
}
|
|
33
|
+
return { ok: false, warning: objectIntrospectionWarning(ctx, error) };
|
|
34
|
+
}
|
|
35
|
+
}
|
|
@@ -6,7 +6,7 @@ import { gunzipSync } from 'node:zlib';
|
|
|
6
6
|
import Database from 'better-sqlite3';
|
|
7
7
|
import YAML from 'yaml';
|
|
8
8
|
import { z } from 'zod';
|
|
9
|
-
import {
|
|
9
|
+
import { getSqlDialectForDriver } from '../connections/dialects.js';
|
|
10
10
|
import { snapshotToKtxEnrichedSchema } from './local-enrichment.js';
|
|
11
11
|
import { generateKtxRelationshipDiscoveryCandidates, mergeKtxRelationshipDiscoveryCandidates, } from './relationship-candidates.js';
|
|
12
12
|
import { proposeKtxRelationshipCandidatesWithLlm } from './relationship-llm-proposal.js';
|
|
@@ -319,7 +319,7 @@ export function ktxRelationshipBenchmarkDetectorWithLlm(llmRuntime) {
|
|
|
319
319
|
const formalLinks = formalMetadata.accepted.map((relationship) => relationshipToBenchmarkLink(relationship));
|
|
320
320
|
const acceptedKeys = new Set(formalLinks.map(fkKey));
|
|
321
321
|
const sqliteDataAvailable = Boolean(input.dataPath && input.snapshot.driver === 'sqlite');
|
|
322
|
-
const dialect =
|
|
322
|
+
const dialect = getSqlDialectForDriver(input.snapshot.driver);
|
|
323
323
|
const profilingExecutor = sqliteDataAvailable && input.mode !== 'profiling_disabled'
|
|
324
324
|
? new KtxRelationshipBenchmarkSqliteExecutor(input.dataPath)
|
|
325
325
|
: null;
|
|
@@ -332,6 +332,7 @@ export function ktxRelationshipBenchmarkDetectorWithLlm(llmRuntime) {
|
|
|
332
332
|
})
|
|
333
333
|
: await profileKtxRelationshipSchema({
|
|
334
334
|
connectionId: input.snapshot.connectionId,
|
|
335
|
+
driver: input.snapshot.driver,
|
|
335
336
|
dialect,
|
|
336
337
|
schema: input.schema,
|
|
337
338
|
executor: profilingExecutor,
|
|
@@ -447,7 +448,7 @@ export function currentKtxRelationshipBenchmarkDetector() {
|
|
|
447
448
|
const formalLinks = formalMetadata.accepted.map((relationship) => relationshipToBenchmarkLink(relationship));
|
|
448
449
|
const acceptedKeys = new Set(formalLinks.map(fkKey));
|
|
449
450
|
const sqliteDataAvailable = Boolean(input.dataPath && input.snapshot.driver === 'sqlite');
|
|
450
|
-
const dialect =
|
|
451
|
+
const dialect = getSqlDialectForDriver(input.snapshot.driver);
|
|
451
452
|
const profilingExecutor = sqliteDataAvailable && input.mode !== 'profiling_disabled'
|
|
452
453
|
? new KtxRelationshipBenchmarkSqliteExecutor(input.dataPath)
|
|
453
454
|
: null;
|
|
@@ -460,6 +461,7 @@ export function currentKtxRelationshipBenchmarkDetector() {
|
|
|
460
461
|
})
|
|
461
462
|
: await profileKtxRelationshipSchema({
|
|
462
463
|
connectionId: input.snapshot.connectionId,
|
|
464
|
+
driver: input.snapshot.driver,
|
|
463
465
|
dialect,
|
|
464
466
|
schema: input.schema,
|
|
465
467
|
executor: profilingExecutor,
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { KtxSqlDialect } from '../connections/dialects.js';
|
|
2
2
|
import type { KtxEnrichedSchema, KtxRelationshipType } from './enrichment-types.js';
|
|
3
|
+
import type { KtxRelationshipDetectionBudget } from './relationship-detection-budget.js';
|
|
3
4
|
import { type KtxRelationshipProfileArtifact, type KtxRelationshipReadOnlyExecutor } from './relationship-profiling.js';
|
|
4
|
-
import type { KtxScanContext, KtxTableRef } from './types.js';
|
|
5
|
+
import type { KtxProgressPort, KtxScanContext, KtxTableRef } from './types.js';
|
|
5
6
|
type KtxCompositeRelationshipStatus = 'accepted' | 'review' | 'rejected';
|
|
6
7
|
interface KtxCompositeRelationshipTupleEndpoint {
|
|
7
8
|
tableId: string;
|
|
@@ -47,7 +48,7 @@ export interface KtxCompositeRelationshipCandidate {
|
|
|
47
48
|
}
|
|
48
49
|
export interface DiscoverKtxCompositeRelationshipsInput {
|
|
49
50
|
connectionId: string;
|
|
50
|
-
dialect:
|
|
51
|
+
dialect: KtxSqlDialect;
|
|
51
52
|
schema: KtxEnrichedSchema;
|
|
52
53
|
profiles: KtxRelationshipProfileArtifact;
|
|
53
54
|
executor: KtxRelationshipReadOnlyExecutor | null;
|
|
@@ -57,6 +58,8 @@ export interface DiscoverKtxCompositeRelationshipsInput {
|
|
|
57
58
|
minPrimaryKeyUniqueness?: number;
|
|
58
59
|
minSourceCoverage?: number;
|
|
59
60
|
maxViolationRatio?: number;
|
|
61
|
+
budget?: KtxRelationshipDetectionBudget;
|
|
62
|
+
progress?: KtxProgressPort;
|
|
60
63
|
}
|
|
61
64
|
export interface DiscoverKtxCompositeRelationshipsResult {
|
|
62
65
|
primaryKeys: KtxCompositePrimaryKeyCandidate[];
|
|
@@ -341,7 +341,13 @@ export async function discoverKtxCompositeRelationships(input) {
|
|
|
341
341
|
const tableByName = new Map(tables.map((table) => [table.ref.name, table]));
|
|
342
342
|
const primaryKeys = [];
|
|
343
343
|
let queryCount = 0;
|
|
344
|
-
for (const table of tables) {
|
|
344
|
+
for (const [index, table] of tables.entries()) {
|
|
345
|
+
if (input.budget?.check()) {
|
|
346
|
+
break;
|
|
347
|
+
}
|
|
348
|
+
await input.progress?.update((index + 1) / tables.length, `Probing composite keys ${index + 1}/${tables.length}`, {
|
|
349
|
+
transient: true,
|
|
350
|
+
});
|
|
345
351
|
const result = await detectCompositePrimaryKeys({
|
|
346
352
|
connectionId: input.connectionId,
|
|
347
353
|
dialect: input.dialect,
|
|
@@ -358,6 +364,9 @@ export async function discoverKtxCompositeRelationships(input) {
|
|
|
358
364
|
}
|
|
359
365
|
const relationships = [];
|
|
360
366
|
for (const targetKey of primaryKeys) {
|
|
367
|
+
if (input.budget?.check()) {
|
|
368
|
+
break;
|
|
369
|
+
}
|
|
361
370
|
const targetTable = tableByName.get(targetKey.table.name);
|
|
362
371
|
if (!targetTable) {
|
|
363
372
|
continue;
|
|
@@ -371,6 +380,9 @@ export async function discoverKtxCompositeRelationships(input) {
|
|
|
371
380
|
continue;
|
|
372
381
|
}
|
|
373
382
|
for (const sourceTable of tables) {
|
|
383
|
+
if (input.budget?.check()) {
|
|
384
|
+
break;
|
|
385
|
+
}
|
|
374
386
|
if (sourceTable.id === targetTable.id) {
|
|
375
387
|
continue;
|
|
376
388
|
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
export type KtxRelationshipDetectionStopReason = 'budget' | 'aborted';
|
|
2
|
+
export interface KtxRelationshipDetectionBudget {
|
|
3
|
+
/**
|
|
4
|
+
* Returns a stop reason when the relationship stage must stop scheduling new
|
|
5
|
+
* work, else null. Calling it at a unit boundary records the first observed
|
|
6
|
+
* stop so the stage can be finalized as partial.
|
|
7
|
+
*/
|
|
8
|
+
check(): KtxRelationshipDetectionStopReason | null;
|
|
9
|
+
/** The first stop reason observed via check(), or null if the stage ran to completion. */
|
|
10
|
+
stopReason(): KtxRelationshipDetectionStopReason | null;
|
|
11
|
+
}
|
|
12
|
+
export interface CreateKtxRelationshipDetectionBudgetInput {
|
|
13
|
+
budgetMs: number;
|
|
14
|
+
signal?: AbortSignal;
|
|
15
|
+
now?: () => number;
|
|
16
|
+
}
|
|
17
|
+
export declare function createKtxRelationshipDetectionBudget(input: CreateKtxRelationshipDetectionBudgetInput): KtxRelationshipDetectionBudget;
|
|
18
|
+
export interface MapWithBudgetInput<TInput, TOutput> {
|
|
19
|
+
inputs: readonly TInput[];
|
|
20
|
+
concurrency: number;
|
|
21
|
+
budget?: KtxRelationshipDetectionBudget;
|
|
22
|
+
onStart?: (index: number, total: number, item: TInput) => Promise<void> | void;
|
|
23
|
+
mapOne: (item: TInput, index: number) => Promise<TOutput>;
|
|
24
|
+
}
|
|
25
|
+
export interface MapWithBudgetResult<TOutput> {
|
|
26
|
+
/** Output aligned with inputs; entries skipped on budget exhaustion are undefined. */
|
|
27
|
+
results: Array<TOutput | undefined>;
|
|
28
|
+
processedCount: number;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Concurrent map that stops claiming new items once the budget trips. In-flight
|
|
32
|
+
* items finish; pending items are left undefined. With no budget it is a plain
|
|
33
|
+
* bounded-concurrency map.
|
|
34
|
+
*/
|
|
35
|
+
export declare function mapWithBudget<TInput, TOutput>(input: MapWithBudgetInput<TInput, TOutput>): Promise<MapWithBudgetResult<TOutput>>;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
export function createKtxRelationshipDetectionBudget(input) {
|
|
2
|
+
const now = input.now ?? (() => Date.now());
|
|
3
|
+
const deadline = now() + Math.max(0, input.budgetMs);
|
|
4
|
+
let tripped = null;
|
|
5
|
+
return {
|
|
6
|
+
check() {
|
|
7
|
+
if (input.signal?.aborted) {
|
|
8
|
+
tripped = 'aborted';
|
|
9
|
+
return 'aborted';
|
|
10
|
+
}
|
|
11
|
+
if (now() >= deadline) {
|
|
12
|
+
tripped ??= 'budget';
|
|
13
|
+
return 'budget';
|
|
14
|
+
}
|
|
15
|
+
return null;
|
|
16
|
+
},
|
|
17
|
+
stopReason() {
|
|
18
|
+
return tripped;
|
|
19
|
+
},
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Concurrent map that stops claiming new items once the budget trips. In-flight
|
|
24
|
+
* items finish; pending items are left undefined. With no budget it is a plain
|
|
25
|
+
* bounded-concurrency map.
|
|
26
|
+
*/
|
|
27
|
+
export async function mapWithBudget(input) {
|
|
28
|
+
const total = input.inputs.length;
|
|
29
|
+
const results = new Array(total);
|
|
30
|
+
const safeConcurrency = Math.max(1, Math.floor(input.concurrency));
|
|
31
|
+
let nextIndex = 0;
|
|
32
|
+
let processedCount = 0;
|
|
33
|
+
async function worker() {
|
|
34
|
+
while (true) {
|
|
35
|
+
const index = nextIndex;
|
|
36
|
+
if (index >= total) {
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
// Check the budget only when work remains, so a deadline that elapses
|
|
40
|
+
// after the last item is claimed never marks a fully-processed stage partial.
|
|
41
|
+
if (input.budget?.check()) {
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
nextIndex += 1;
|
|
45
|
+
const item = input.inputs[index];
|
|
46
|
+
await input.onStart?.(index, total, item);
|
|
47
|
+
results[index] = await input.mapOne(item, index);
|
|
48
|
+
processedCount += 1;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
await Promise.all(Array.from({ length: Math.min(safeConcurrency, total) }, () => worker()));
|
|
52
|
+
return { results, processedCount };
|
|
53
|
+
}
|
|
@@ -63,6 +63,8 @@ export interface KtxRelationshipDiagnosticsArtifact {
|
|
|
63
63
|
generatedAt: string;
|
|
64
64
|
summary: KtxRelationshipDiagnosticsSummary;
|
|
65
65
|
noAcceptedReason: string | null;
|
|
66
|
+
partial: boolean;
|
|
67
|
+
partialReason: string | null;
|
|
66
68
|
candidateCountsBySource: Record<string, number>;
|
|
67
69
|
validation: KtxRelationshipDiagnosticsValidation;
|
|
68
70
|
thresholds: KtxRelationshipDiagnosticsThresholds;
|
|
@@ -83,6 +85,9 @@ export interface BuildKtxRelationshipDiagnosticsInput {
|
|
|
83
85
|
warnings?: readonly KtxScanWarning[];
|
|
84
86
|
thresholds?: Partial<KtxRelationshipDiagnosticsThresholds>;
|
|
85
87
|
policy?: Partial<KtxRelationshipDiagnosticsPolicy>;
|
|
88
|
+
partial?: {
|
|
89
|
+
reason: string;
|
|
90
|
+
} | null;
|
|
86
91
|
generatedAt?: string;
|
|
87
92
|
}
|
|
88
93
|
export interface EmptyKtxRelationshipProfileArtifactInput {
|
|
@@ -211,6 +211,8 @@ export function buildKtxRelationshipDiagnostics(input) {
|
|
|
211
211
|
generatedAt: input.generatedAt ?? new Date().toISOString(),
|
|
212
212
|
summary,
|
|
213
213
|
noAcceptedReason: noAcceptedReason({ artifacts: input.artifacts, profile: input.profile }),
|
|
214
|
+
partial: Boolean(input.partial),
|
|
215
|
+
partialReason: input.partial?.reason ?? null,
|
|
214
216
|
candidateCountsBySource: candidateCountsBySource(input.artifacts),
|
|
215
217
|
validation: {
|
|
216
218
|
available: input.profile.sqlAvailable,
|
|
@@ -1,19 +1,22 @@
|
|
|
1
1
|
import type { KtxLlmRuntimePort } from '../../context/llm/runtime-port.js';
|
|
2
|
-
import type {
|
|
2
|
+
import type { KtxSqlDialect } from '../connections/dialects.js';
|
|
3
3
|
import type { KtxScanRelationshipConfig } from '../project/config.js';
|
|
4
4
|
import type { KtxEnrichedSchema, KtxRelationshipUpdate } from './enrichment-types.js';
|
|
5
5
|
import { type KtxCompositeRelationshipCandidate } from './relationship-composite-candidates.js';
|
|
6
|
+
import { type KtxRelationshipDetectionStopReason } from './relationship-detection-budget.js';
|
|
6
7
|
import { type KtxResolvedRelationshipDiscoveryCandidate } from './relationship-graph-resolver.js';
|
|
7
8
|
import { type KtxRelationshipProfileArtifact } from './relationship-profiling.js';
|
|
8
|
-
import type { KtxScanConnector, KtxScanContext, KtxScanEnrichmentSummary, KtxScanRelationshipSummary, KtxScanWarning } from './types.js';
|
|
9
|
+
import type { KtxProgressPort, KtxScanConnector, KtxScanContext, KtxScanEnrichmentSummary, KtxScanRelationshipSummary, KtxScanWarning } from './types.js';
|
|
9
10
|
export interface DiscoverKtxRelationshipsInput {
|
|
10
11
|
connectionId: string;
|
|
11
|
-
dialect:
|
|
12
|
+
dialect: KtxSqlDialect | null;
|
|
12
13
|
connector: KtxScanConnector;
|
|
13
14
|
schema: KtxEnrichedSchema;
|
|
14
15
|
context: KtxScanContext;
|
|
15
16
|
settings: KtxScanRelationshipConfig;
|
|
16
17
|
llmRuntime?: KtxLlmRuntimePort | null;
|
|
18
|
+
progress?: KtxProgressPort;
|
|
19
|
+
now?: () => number;
|
|
17
20
|
}
|
|
18
21
|
export interface DiscoverKtxRelationshipsResult {
|
|
19
22
|
relationshipUpdate: KtxRelationshipUpdate;
|
|
@@ -24,5 +27,8 @@ export interface DiscoverKtxRelationshipsResult {
|
|
|
24
27
|
statisticalValidation: KtxScanEnrichmentSummary['statisticalValidation'];
|
|
25
28
|
llmRelationshipValidation: KtxScanEnrichmentSummary['llmRelationshipValidation'];
|
|
26
29
|
warnings: KtxScanWarning[];
|
|
30
|
+
partial: {
|
|
31
|
+
reason: KtxRelationshipDetectionStopReason;
|
|
32
|
+
} | null;
|
|
27
33
|
}
|
|
28
34
|
export declare function discoverKtxRelationships(input: DiscoverKtxRelationshipsInput): Promise<DiscoverKtxRelationshipsResult>;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { generateKtxRelationshipDiscoveryCandidates, mergeKtxRelationshipDiscoveryCandidates, } from './relationship-candidates.js';
|
|
2
2
|
import { discoverKtxCompositeRelationships, } from './relationship-composite-candidates.js';
|
|
3
|
+
import { createKtxRelationshipDetectionBudget, } from './relationship-detection-budget.js';
|
|
3
4
|
import { collectKtxFormalMetadataRelationships } from './relationship-formal-metadata.js';
|
|
4
5
|
import { resolveKtxRelationshipGraph, } from './relationship-graph-resolver.js';
|
|
5
6
|
import { proposeKtxRelationshipCandidatesWithLlm } from './relationship-llm-proposal.js';
|
|
@@ -62,17 +63,20 @@ function compositeSummary(relationships) {
|
|
|
62
63
|
};
|
|
63
64
|
}
|
|
64
65
|
async function detectCompositeRelationships(input) {
|
|
65
|
-
if (!input.executor || !input.profile.sqlAvailable) {
|
|
66
|
+
if (!input.executor || !input.profile.sqlAvailable || !input.dialect) {
|
|
66
67
|
return [];
|
|
67
68
|
}
|
|
69
|
+
const dialect = input.dialect;
|
|
68
70
|
try {
|
|
69
71
|
const compositeDetection = await discoverKtxCompositeRelationships({
|
|
70
72
|
connectionId: input.connectionId,
|
|
71
|
-
dialect
|
|
73
|
+
dialect,
|
|
72
74
|
schema: input.schema,
|
|
73
75
|
profiles: input.profile,
|
|
74
76
|
executor: input.executor,
|
|
75
77
|
ctx: input.context,
|
|
78
|
+
budget: input.budget,
|
|
79
|
+
...(input.progress ? { progress: input.progress } : {}),
|
|
76
80
|
});
|
|
77
81
|
for (const warning of compositeDetection.warnings) {
|
|
78
82
|
input.warnings.push({
|
|
@@ -139,10 +143,16 @@ function sqlExecutor(input) {
|
|
|
139
143
|
}
|
|
140
144
|
export async function discoverKtxRelationships(input) {
|
|
141
145
|
const { executor, warnings } = sqlExecutor(input);
|
|
146
|
+
const budget = createKtxRelationshipDetectionBudget({
|
|
147
|
+
budgetMs: input.settings.detectionBudgetMs,
|
|
148
|
+
...(input.context.signal ? { signal: input.context.signal } : {}),
|
|
149
|
+
...(input.now ? { now: input.now } : {}),
|
|
150
|
+
});
|
|
142
151
|
const formalMetadata = collectKtxFormalMetadataRelationships(input.schema);
|
|
143
152
|
const profileCache = createKtxRelationshipProfileCache();
|
|
144
153
|
const profile = await profileKtxRelationshipSchema({
|
|
145
154
|
connectionId: input.connectionId,
|
|
155
|
+
driver: input.connector.driver,
|
|
146
156
|
dialect: input.dialect,
|
|
147
157
|
schema: input.schema,
|
|
148
158
|
executor,
|
|
@@ -150,12 +160,17 @@ export async function discoverKtxRelationships(input) {
|
|
|
150
160
|
profileSampleRows: input.settings.profileSampleRows,
|
|
151
161
|
profileConcurrency: input.settings.profileConcurrency,
|
|
152
162
|
cache: profileCache,
|
|
163
|
+
budget,
|
|
164
|
+
...(input.progress ? { progress: input.progress } : {}),
|
|
153
165
|
});
|
|
154
166
|
const deterministicCandidates = generateKtxRelationshipDiscoveryCandidates(input.schema, {
|
|
155
167
|
maxCandidatesPerColumn: input.settings.maxCandidatesPerColumn,
|
|
156
168
|
profiles: profile,
|
|
157
169
|
});
|
|
158
|
-
|
|
170
|
+
// The LLM proposal is one more unit of relationship work, so it honors the same
|
|
171
|
+
// budget/abort gate as profiling, validation, and composite probing — a stage
|
|
172
|
+
// that already exhausted its budget (or was aborted) must not start a fresh call.
|
|
173
|
+
const llmProposalResult = input.settings.llmProposals && !budget.check()
|
|
159
174
|
? await proposeKtxRelationshipCandidatesWithLlm({
|
|
160
175
|
connectionId: input.connectionId,
|
|
161
176
|
schema: input.schema,
|
|
@@ -186,6 +201,8 @@ export async function discoverKtxRelationships(input) {
|
|
|
186
201
|
concurrency: input.settings.validationConcurrency,
|
|
187
202
|
validationBudget: input.settings.validationBudget,
|
|
188
203
|
},
|
|
204
|
+
budget,
|
|
205
|
+
...(input.progress ? { progress: input.progress } : {}),
|
|
189
206
|
});
|
|
190
207
|
const graph = resolveKtxRelationshipGraph({
|
|
191
208
|
schema: input.schema,
|
|
@@ -205,6 +222,8 @@ export async function discoverKtxRelationships(input) {
|
|
|
205
222
|
executor,
|
|
206
223
|
context: input.context,
|
|
207
224
|
warnings,
|
|
225
|
+
budget,
|
|
226
|
+
...(input.progress ? { progress: input.progress } : {}),
|
|
208
227
|
});
|
|
209
228
|
const inferredAccepted = nonFormalAcceptedRelationships({
|
|
210
229
|
formalIds: formalMetadata.acceptedIds,
|
|
@@ -225,6 +244,7 @@ export async function discoverKtxRelationships(input) {
|
|
|
225
244
|
resolvedRelationships: graph.relationships,
|
|
226
245
|
});
|
|
227
246
|
const compositeCounts = compositeSummary(compositeRelationships);
|
|
247
|
+
const stopReason = budget.stopReason();
|
|
228
248
|
return {
|
|
229
249
|
relationshipUpdate: {
|
|
230
250
|
connectionId: input.connectionId,
|
|
@@ -241,8 +261,11 @@ export async function discoverKtxRelationships(input) {
|
|
|
241
261
|
profile,
|
|
242
262
|
resolvedRelationships: graph.relationships,
|
|
243
263
|
compositeRelationships,
|
|
244
|
-
|
|
264
|
+
// A budget/abort stop means profiling did not finish, so report it as not
|
|
265
|
+
// completed even though the SQL capability was available.
|
|
266
|
+
statisticalValidation: profile.sqlAvailable && !stopReason ? 'completed' : 'skipped',
|
|
245
267
|
llmRelationshipValidation: llmProposalResult.summary,
|
|
246
268
|
warnings,
|
|
269
|
+
partial: stopReason ? { reason: stopReason } : null,
|
|
247
270
|
};
|
|
248
271
|
}
|