@kaelio/ktx 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/kaelio_ktx-0.13.0-py3-none-any.whl +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/admin.js +1 -1
- package/dist/clack.d.ts +16 -0
- package/dist/clack.js +37 -6
- package/dist/claude-code-prompt-caching.js +1 -1
- package/dist/cli-program.js +3 -3
- package/dist/cli-runtime.js +2 -2
- package/dist/commands/connection-commands.js +1 -1
- package/dist/commands/ingest-commands.js +4 -4
- package/dist/commands/mcp-commands.js +12 -12
- package/dist/commands/runtime-commands.js +4 -4
- package/dist/commands/setup-commands.js +19 -5
- package/dist/commands/sl-commands.js +1 -1
- package/dist/commands/sql-commands.js +1 -1
- package/dist/commands/status-commands.js +1 -1
- package/dist/connection.js +15 -3
- package/dist/connectors/bigquery/connector.js +1 -14
- package/dist/connectors/clickhouse/connector.js +2 -16
- package/dist/connectors/duckdb/federated-attach.d.ts +7 -0
- package/dist/connectors/duckdb/federated-attach.js +86 -0
- package/dist/connectors/duckdb/federated-executor.d.ts +5 -0
- package/dist/connectors/duckdb/federated-executor.js +59 -0
- package/dist/connectors/mysql/connector.js +2 -16
- package/dist/connectors/postgres/connector.js +1 -14
- package/dist/connectors/shared/string-reference.d.ts +6 -0
- package/dist/connectors/shared/string-reference.js +19 -0
- package/dist/connectors/snowflake/connector.d.ts +1 -1
- package/dist/connectors/snowflake/connector.js +1 -14
- package/dist/connectors/sqlite/connector.js +2 -25
- package/dist/connectors/sqlserver/connector.js +4 -17
- package/dist/context/connections/connection-type.d.ts +1 -1
- package/dist/context/connections/federation.d.ts +33 -0
- package/dist/context/connections/federation.js +51 -0
- package/dist/context/connections/local-warehouse-descriptor.d.ts +2 -0
- package/dist/context/connections/project-sql-executor.d.ts +18 -0
- package/dist/context/connections/project-sql-executor.js +39 -0
- package/dist/context/connections/query-executor.d.ts +2 -2
- package/dist/context/connections/read-only-sql.d.ts +1 -0
- package/dist/context/connections/read-only-sql.js +119 -4
- package/dist/context/connections/resolve-connection.d.ts +12 -0
- package/dist/context/connections/resolve-connection.js +37 -0
- package/dist/context/core/git-env.d.ts +4 -0
- package/dist/context/core/git-env.js +5 -1
- package/dist/context/core/git.service.d.ts +23 -0
- package/dist/context/core/git.service.js +71 -8
- package/dist/context/ingest/adapters/historic-sql/projection.js +2 -1
- package/dist/context/ingest/adapters/live-database/manifest.d.ts +3 -0
- package/dist/context/ingest/adapters/live-database/manifest.js +19 -11
- package/dist/context/ingest/adapters/looker/client.js +7 -2
- package/dist/context/ingest/adapters/looker/factory.d.ts +8 -1
- package/dist/context/ingest/adapters/looker/factory.js +9 -0
- package/dist/context/ingest/adapters/looker/mapping.js +1 -1
- package/dist/context/ingest/adapters/looker/types.d.ts +1 -1
- package/dist/context/ingest/adapters/metabase/client.d.ts +1 -1
- package/dist/context/ingest/adapters/metabase/client.js +1 -1
- package/dist/context/ingest/adapters/metabase/local-metabase.adapter.js +1 -1
- package/dist/context/ingest/adapters/metabase/mapping.js +6 -6
- package/dist/context/ingest/artifact-gates.d.ts +2 -6
- package/dist/context/ingest/artifact-gates.js +5 -47
- package/dist/context/ingest/constrained-repair.d.ts +55 -0
- package/dist/context/ingest/constrained-repair.js +167 -0
- package/dist/context/ingest/final-gate-repair.d.ts +9 -11
- package/dist/context/ingest/final-gate-repair.js +40 -128
- package/dist/context/ingest/finalization-scope.d.ts +1 -1
- package/dist/context/ingest/finalization-scope.js +15 -15
- package/dist/context/ingest/ingest-bundle.runner.d.ts +1 -0
- package/dist/context/ingest/ingest-bundle.runner.js +101 -67
- package/dist/context/ingest/isolated-diff/patch-integrator.d.ts +6 -13
- package/dist/context/ingest/isolated-diff/patch-integrator.js +32 -109
- package/dist/context/ingest/isolated-diff/textual-conflict-resolver.d.ts +8 -9
- package/dist/context/ingest/isolated-diff/textual-conflict-resolver.js +63 -141
- package/dist/context/ingest/local-bundle-runtime.d.ts +2 -0
- package/dist/context/ingest/local-bundle-runtime.js +9 -10
- package/dist/context/ingest/local-ingest.d.ts +2 -0
- package/dist/context/ingest/local-ingest.js +2 -0
- package/dist/context/ingest/memory-flow/view-model.js +1 -1
- package/dist/context/ingest/stages/stage-3-work-units.d.ts +2 -6
- package/dist/context/ingest/stages/stage-3-work-units.js +2 -1
- package/dist/context/ingest/stages/validate-wu-sources.d.ts +7 -1
- package/dist/context/ingest/stages/validate-wu-sources.js +109 -4
- package/dist/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.d.ts +2 -0
- package/dist/context/ingest/tools/warehouse-verification/create-warehouse-verification-tools.js +1 -1
- package/dist/context/ingest/tools/warehouse-verification/discover-data.tool.js +3 -3
- package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.d.ts +3 -1
- package/dist/context/ingest/tools/warehouse-verification/sql-execution.tool.js +15 -1
- package/dist/context/llm/ai-sdk-runtime.js +2 -2
- package/dist/context/llm/claude-code-runtime.js +19 -3
- package/dist/context/llm/local-config.js +1 -1
- package/dist/context/llm/runtime-tools.js +2 -2
- package/dist/context/mcp/context-tools.js +33 -8
- package/dist/context/mcp/local-project-ports.js +63 -89
- package/dist/context/mcp/types.d.ts +2 -0
- package/dist/context/memory/local-memory.js +4 -1
- package/dist/context/memory/memory-agent.service.js +1 -1
- package/dist/context/project/config.d.ts +11 -4
- package/dist/context/project/config.js +85 -30
- package/dist/context/project/driver-schemas.js +1 -1
- package/dist/context/project/mappings-yaml-schema.js +2 -2
- package/dist/context/project/project.js +12 -4
- package/dist/context/scan/description-generation.js +4 -4
- package/dist/context/scan/local-enrichment-artifacts.js +33 -4
- package/dist/context/scan/local-scan.js +2 -2
- package/dist/context/scan/local-structural-artifacts.js +5 -5
- package/dist/context/scan/relationship-benchmark-report.js +1 -1
- package/dist/context/scan/relationship-discovery.js +3 -3
- package/dist/context/scan/relationship-llm-proposal.js +3 -3
- package/dist/context/sl/local-query.js +31 -44
- package/dist/context/sl/local-sl.d.ts +0 -8
- package/dist/context/sl/local-sl.js +71 -70
- package/dist/context/sl/semantic-layer.service.d.ts +25 -8
- package/dist/context/sl/semantic-layer.service.js +109 -56
- package/dist/context/sl/source-files.d.ts +48 -0
- package/dist/context/sl/source-files.js +138 -0
- package/dist/context/sl/tools/base-semantic-layer.tool.d.ts +2 -2
- package/dist/context/sl/tools/base-semantic-layer.tool.js +2 -7
- package/dist/context/sl/tools/sl-edit-source.tool.js +10 -8
- package/dist/context/sl/tools/sl-warehouse-validation.js +55 -27
- package/dist/context/sl/tools/sl-write-source.tool.js +12 -9
- package/dist/context/sql-analysis/dialect.d.ts +2 -0
- package/dist/context/sql-analysis/dialect.js +20 -0
- package/dist/context/tools/base-tool.d.ts +6 -19
- package/dist/context/tools/base-tool.js +0 -14
- package/dist/context-build-view.js +5 -5
- package/dist/database-tree-picker.js +18 -3
- package/dist/demo-assets.js +0 -1
- package/dist/doctor.d.ts +1 -1
- package/dist/doctor.js +31 -23
- package/dist/errors.d.ts +31 -0
- package/dist/errors.js +44 -0
- package/dist/ingest-query-executor.d.ts +2 -0
- package/dist/ingest-query-executor.js +8 -22
- package/dist/ingest.d.ts +1 -1
- package/dist/ingest.js +8 -2
- package/dist/io/symbols.d.ts +2 -0
- package/dist/io/symbols.js +2 -0
- package/dist/io/tty.d.ts +8 -0
- package/dist/io/tty.js +16 -0
- package/dist/llm/embedding-health.js +1 -1
- package/dist/llm/embedding-provider.js +3 -3
- package/dist/llm/model-provider.js +1 -1
- package/dist/local-adapters.d.ts +1 -0
- package/dist/local-adapters.js +2 -2
- package/dist/local-scan-connectors.js +1 -1
- package/dist/managed-local-embeddings.js +17 -8
- package/dist/managed-mcp-daemon.js +3 -3
- package/dist/managed-python-command.d.ts +7 -0
- package/dist/managed-python-command.js +34 -8
- package/dist/managed-python-daemon.js +2 -2
- package/dist/managed-python-http.js +3 -3
- package/dist/managed-python-runtime.d.ts +30 -1
- package/dist/managed-python-runtime.js +134 -18
- package/dist/managed-uv-release.d.ts +7 -0
- package/dist/managed-uv-release.js +11 -0
- package/dist/mcp-http-server.js +4 -4
- package/dist/mcp-server-factory.js +3 -3
- package/dist/mcp-stdio-server.js +1 -1
- package/dist/memory-flow-hud.js +2 -2
- package/dist/next-steps.js +2 -2
- package/dist/prompt-navigation.d.ts +17 -0
- package/dist/prompt-navigation.js +49 -3
- package/dist/prompts/memory_agent_bundle_ingest_work_unit.md +2 -2
- package/dist/prompts/memory_agent_external_ingest.md +2 -2
- package/dist/public-ingest-copy.js +1 -1
- package/dist/public-ingest.js +3 -3
- package/dist/release-version.js +1 -1
- package/dist/runtime-requirements.js +1 -1
- package/dist/runtime.js +9 -9
- package/dist/scan.js +1 -1
- package/dist/setup-agents.d.ts +21 -15
- package/dist/setup-agents.js +143 -66
- package/dist/setup-banner.d.ts +20 -0
- package/dist/setup-banner.js +39 -0
- package/dist/setup-context.js +24 -15
- package/dist/setup-databases.d.ts +3 -0
- package/dist/setup-databases.js +47 -59
- package/dist/setup-demo-tour.js +12 -8
- package/dist/setup-embeddings.js +9 -9
- package/dist/setup-interrupt.js +1 -1
- package/dist/setup-models.d.ts +4 -1
- package/dist/setup-models.js +54 -28
- package/dist/setup-project.js +29 -5
- package/dist/setup-prompts.js +16 -1
- package/dist/setup-ready-menu.js +1 -1
- package/dist/setup-sources.js +28 -12
- package/dist/setup.d.ts +1 -0
- package/dist/setup.js +14 -13
- package/dist/skills/analytics/SKILL.md +3 -3
- package/dist/skills/dbt_ingest/SKILL.md +3 -3
- package/dist/skills/looker_ingest/SKILL.md +3 -3
- package/dist/skills/lookml_ingest/SKILL.md +7 -7
- package/dist/skills/metabase_ingest/SKILL.md +4 -4
- package/dist/skills/metricflow_ingest/SKILL.md +15 -15
- package/dist/skills/notion_synthesize/SKILL.md +1 -1
- package/dist/skills/sl/SKILL.md +3 -3
- package/dist/skills/sl_capture/SKILL.md +1 -1
- package/dist/skills/wiki_capture/SKILL.md +1 -1
- package/dist/source-mapping.js +1 -1
- package/dist/sql.d.ts +2 -0
- package/dist/sql.js +35 -53
- package/dist/startup-profile.js +1 -1
- package/dist/status-project.d.ts +0 -2
- package/dist/status-project.js +4 -6
- package/dist/telemetry/events.d.ts +3 -2
- package/dist/telemetry/events.js +11 -1
- package/dist/telemetry/exception.js +14 -0
- package/dist/text-ingest.js +1 -1
- package/dist/tree-picker-tui.d.ts +0 -1
- package/dist/tree-picker-tui.js +2 -3
- package/package.json +2 -1
- package/assets/python/kaelio_ktx-0.11.0-py3-none-any.whl +0 -0
|
@@ -53,7 +53,7 @@ const llmSchema = z
|
|
|
53
53
|
models: z
|
|
54
54
|
.partialRecord(z.enum(KTX_MODEL_ROLES), z.string().min(1))
|
|
55
55
|
.default({})
|
|
56
|
-
.describe('Per-role model overrides keyed by
|
|
56
|
+
.describe('Per-role model overrides keyed by ktx model role (e.g. "default", "triage"). Values are provider-specific model identifiers.'),
|
|
57
57
|
promptCaching: promptCachingSchema.optional().describe('Optional prompt-caching tunables.'),
|
|
58
58
|
})
|
|
59
59
|
.describe('LLM provider, per-role model overrides, and prompt-caching tunables.');
|
|
@@ -205,27 +205,26 @@ const setupSchema = z
|
|
|
205
205
|
.describe('Setup-wizard state captured during `ktx setup`.');
|
|
206
206
|
const storageGitSchema = z
|
|
207
207
|
.strictObject({
|
|
208
|
-
auto_commit: z.boolean().default(true).describe('When true, KTX automatically commits state changes to the local Git-backed store.'),
|
|
209
208
|
author: z
|
|
210
209
|
.string()
|
|
211
210
|
.min(1)
|
|
212
211
|
.default('ktx <ktx@example.com>')
|
|
213
|
-
.describe('Git author identity used for
|
|
212
|
+
.describe('Git author identity used for commits, in standard "Name <email>" form.'),
|
|
214
213
|
})
|
|
215
|
-
.describe('Git-backed storage
|
|
214
|
+
.describe('Git-backed storage author policy.');
|
|
216
215
|
const storageSchema = z
|
|
217
216
|
.strictObject({
|
|
218
217
|
state: z
|
|
219
218
|
.enum(KTX_STORAGE_STATES)
|
|
220
219
|
.default('sqlite')
|
|
221
|
-
.describe('Backend for
|
|
220
|
+
.describe('Backend for ktx state storage. "sqlite" uses .ktx/db.sqlite; "postgres" expects a configured Postgres connection.'),
|
|
222
221
|
search: z
|
|
223
222
|
.enum(KTX_SEARCH_BACKENDS)
|
|
224
223
|
.default('sqlite-fts5')
|
|
225
224
|
.describe('Backend for search indexes. "sqlite-fts5" uses SQLite FTS5; "postgres-hybrid" uses Postgres lexical + vector hybrid search.'),
|
|
226
225
|
git: storageGitSchema.prefault({}).describe('Git-backed storage commit policy.'),
|
|
227
226
|
})
|
|
228
|
-
.describe('Storage backends and commit policy for
|
|
227
|
+
.describe('Storage backends and commit policy for ktx state and search indexes.');
|
|
229
228
|
const connectionSchema = connectionConfigSchema;
|
|
230
229
|
const agentSchema = z
|
|
231
230
|
.strictObject({
|
|
@@ -247,11 +246,6 @@ const agentSchema = z
|
|
|
247
246
|
.describe('Research-agent configuration.'),
|
|
248
247
|
})
|
|
249
248
|
.describe('Agent feature configuration.');
|
|
250
|
-
const memorySchema = z
|
|
251
|
-
.strictObject({
|
|
252
|
-
auto_commit: z.boolean().default(true).describe('When true, KTX automatically commits memory updates to the Git-backed store.'),
|
|
253
|
-
})
|
|
254
|
-
.describe('Memory subsystem configuration.');
|
|
255
249
|
const ktxProjectConfigSchema = z
|
|
256
250
|
.strictObject({
|
|
257
251
|
setup: setupSchema.optional().describe('Setup-wizard state. Written by `ktx setup`; may be omitted.'),
|
|
@@ -259,14 +253,13 @@ const ktxProjectConfigSchema = z
|
|
|
259
253
|
.record(z.string(), connectionSchema)
|
|
260
254
|
.default({})
|
|
261
255
|
.describe('Map of connection ID to connector configuration. Keys are user-chosen names referenced elsewhere in the config.'),
|
|
262
|
-
storage: storageSchema.prefault({}).describe('Storage backends and commit policy for
|
|
256
|
+
storage: storageSchema.prefault({}).describe('Storage backends and commit policy for ktx state and search indexes.'),
|
|
263
257
|
llm: llmSchema.prefault({}).describe('LLM provider, per-role model overrides, and prompt-caching tunables.'),
|
|
264
258
|
ingest: ingestSchema.prefault({}).describe('Ingest pipeline configuration.'),
|
|
265
259
|
agent: agentSchema.prefault({}).describe('Agent feature configuration.'),
|
|
266
|
-
memory: memorySchema.prefault({}).describe('Memory subsystem configuration.'),
|
|
267
260
|
scan: scanSchema.prefault({}).describe('Schema-scan configuration: enrichment and relationship discovery.'),
|
|
268
261
|
})
|
|
269
|
-
.describe('Configuration schema for
|
|
262
|
+
.describe('Configuration schema for ktx project files (ktx.yaml).');
|
|
270
263
|
function isRecord(value) {
|
|
271
264
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
272
265
|
}
|
|
@@ -282,21 +275,53 @@ function valueAtPath(root, path) {
|
|
|
282
275
|
}
|
|
283
276
|
return cursor;
|
|
284
277
|
}
|
|
285
|
-
|
|
286
|
-
|
|
278
|
+
/**
|
|
279
|
+
* Zod reports unknown keys in two shapes: strict objects emit
|
|
280
|
+
* `unrecognized_keys` (path → container, `keys` → offenders), enum-keyed
|
|
281
|
+
* records (`llm.models`) emit one `invalid_key` per offender (path ends with
|
|
282
|
+
* the key). Normalize both so the warning report and the strip always agree.
|
|
283
|
+
*/
|
|
284
|
+
function unknownKeyLocations(issue) {
|
|
287
285
|
if (issue.code === 'unrecognized_keys') {
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
286
|
+
return issue.keys.map((key) => ({ containerPath: issue.path, key }));
|
|
287
|
+
}
|
|
288
|
+
if (issue.code === 'invalid_key' && issue.path.length > 0) {
|
|
289
|
+
return [
|
|
290
|
+
{
|
|
291
|
+
containerPath: issue.path.slice(0, -1),
|
|
292
|
+
key: String(issue.path[issue.path.length - 1]),
|
|
293
|
+
},
|
|
294
|
+
];
|
|
295
|
+
}
|
|
296
|
+
return [];
|
|
297
|
+
}
|
|
298
|
+
function formatIssue(issue, input) {
|
|
299
|
+
const unknownKeys = unknownKeyLocations(issue);
|
|
300
|
+
if (unknownKeys.length > 0) {
|
|
301
|
+
return unknownKeys.map(({ containerPath, key }) => {
|
|
302
|
+
const base = dottedPath(containerPath);
|
|
303
|
+
const fullPath = base.length > 0 ? `${base}.${key}` : key;
|
|
304
|
+
return {
|
|
305
|
+
path: fullPath,
|
|
306
|
+
message: `Unsupported ${fullPath}: unknown field (ignored)`,
|
|
307
|
+
fix: 'Unknown to this ktx version; it is ignored. Delete it from ktx.yaml when convenient.',
|
|
308
|
+
severity: 'warning',
|
|
309
|
+
};
|
|
292
310
|
});
|
|
293
311
|
}
|
|
312
|
+
const basePath = dottedPath(issue.path);
|
|
294
313
|
const lastSegment = issue.path[issue.path.length - 1];
|
|
295
314
|
if (lastSegment === 'backend' && (issue.code === 'invalid_value' || issue.code === 'invalid_type')) {
|
|
296
315
|
const value = valueAtPath(input, issue.path);
|
|
297
|
-
return [{ path: basePath, message: `Unsupported ${basePath}: ${String(value)}
|
|
316
|
+
return [{ path: basePath, message: `Unsupported ${basePath}: ${String(value)}`, severity: 'error' }];
|
|
298
317
|
}
|
|
299
|
-
return [
|
|
318
|
+
return [
|
|
319
|
+
{
|
|
320
|
+
path: basePath,
|
|
321
|
+
message: basePath.length > 0 ? `${basePath}: ${issue.message}` : issue.message,
|
|
322
|
+
severity: 'error',
|
|
323
|
+
},
|
|
324
|
+
];
|
|
300
325
|
}
|
|
301
326
|
function collectIssues(error, input) {
|
|
302
327
|
return error.issues.flatMap((issue) => formatIssue(issue, input));
|
|
@@ -309,16 +334,44 @@ function formatZodError(error, input) {
|
|
|
309
334
|
export function buildDefaultKtxProjectConfig() {
|
|
310
335
|
return ktxProjectConfigSchema.parse({});
|
|
311
336
|
}
|
|
337
|
+
function stripUnrecognizedKeys(input) {
|
|
338
|
+
const result = ktxProjectConfigSchema.safeParse(input);
|
|
339
|
+
if (result.success) {
|
|
340
|
+
return input;
|
|
341
|
+
}
|
|
342
|
+
const unknownKeys = result.error.issues.flatMap(unknownKeyLocations);
|
|
343
|
+
if (unknownKeys.length === 0) {
|
|
344
|
+
return input;
|
|
345
|
+
}
|
|
346
|
+
const value = structuredClone(input);
|
|
347
|
+
for (const { containerPath, key } of unknownKeys) {
|
|
348
|
+
const container = valueAtPath(value, containerPath);
|
|
349
|
+
if (container === null || typeof container !== 'object')
|
|
350
|
+
continue;
|
|
351
|
+
delete container[key];
|
|
352
|
+
}
|
|
353
|
+
return value;
|
|
354
|
+
}
|
|
355
|
+
function parseTolerant(input) {
|
|
356
|
+
const value = stripUnrecognizedKeys(input);
|
|
357
|
+
const result = ktxProjectConfigSchema.safeParse(value);
|
|
358
|
+
if (!result.success) {
|
|
359
|
+
throw new Error(formatZodError(result.error, value));
|
|
360
|
+
}
|
|
361
|
+
return result.data;
|
|
362
|
+
}
|
|
363
|
+
/**
|
|
364
|
+
* Parse and validate a ktx.yaml document. Keys this ktx version does not
|
|
365
|
+
* recognize are stripped from the returned config — never from the file, which
|
|
366
|
+
* a load must not rewrite — so a config written by a different ktx version
|
|
367
|
+
* still loads. Malformed values on recognized fields still throw.
|
|
368
|
+
*/
|
|
312
369
|
export function parseKtxProjectConfig(raw) {
|
|
313
370
|
const parsed = YAML.parse(raw);
|
|
314
371
|
if (!isRecord(parsed)) {
|
|
315
372
|
throw new Error('ktx.yaml must contain a YAML object');
|
|
316
373
|
}
|
|
317
|
-
|
|
318
|
-
if (!result.success) {
|
|
319
|
-
throw new Error(formatZodError(result.error, parsed));
|
|
320
|
-
}
|
|
321
|
-
return result.data;
|
|
374
|
+
return parseTolerant(parsed);
|
|
322
375
|
}
|
|
323
376
|
export function validateKtxProjectConfig(raw) {
|
|
324
377
|
let parsed;
|
|
@@ -327,16 +380,18 @@ export function validateKtxProjectConfig(raw) {
|
|
|
327
380
|
}
|
|
328
381
|
catch (error) {
|
|
329
382
|
const message = error instanceof Error ? error.message : String(error);
|
|
330
|
-
return { ok: false, issues: [{ path: '', message: `ktx.yaml parse error: ${message}
|
|
383
|
+
return { ok: false, issues: [{ path: '', message: `ktx.yaml parse error: ${message}`, severity: 'error' }] };
|
|
331
384
|
}
|
|
332
385
|
if (!isRecord(parsed)) {
|
|
333
|
-
return { ok: false, issues: [{ path: '', message: 'ktx.yaml must contain a YAML object' }] };
|
|
386
|
+
return { ok: false, issues: [{ path: '', message: 'ktx.yaml must contain a YAML object', severity: 'error' }] };
|
|
334
387
|
}
|
|
335
388
|
const result = ktxProjectConfigSchema.safeParse(parsed);
|
|
336
389
|
if (result.success) {
|
|
337
390
|
return { ok: true, issues: [] };
|
|
338
391
|
}
|
|
339
|
-
|
|
392
|
+
const issues = collectIssues(result.error, parsed);
|
|
393
|
+
const ok = !issues.some((issue) => issue.severity === 'error');
|
|
394
|
+
return { ok, issues };
|
|
340
395
|
}
|
|
341
396
|
export function generateKtxProjectConfigJsonSchema() {
|
|
342
397
|
const schema = z.toJSONSchema(ktxProjectConfigSchema, {
|
|
@@ -84,7 +84,7 @@ const lookerConnectionSchema = z
|
|
|
84
84
|
.min(1)
|
|
85
85
|
.optional()
|
|
86
86
|
.describe('Reference to Looker OAuth client secret (e.g. env:LOOKER_CLIENT_SECRET).'),
|
|
87
|
-
mappings: lookerMappingsSchema.optional().describe('Looker connection-name to
|
|
87
|
+
mappings: lookerMappingsSchema.optional().describe('Looker connection-name to ktx warehouse mappings.'),
|
|
88
88
|
})
|
|
89
89
|
.describe('Looker context-source connection.');
|
|
90
90
|
const lookmlConnectionSchema = z
|
|
@@ -12,7 +12,7 @@ export const metabaseMappingsSchema = z
|
|
|
12
12
|
databaseMappings: z
|
|
13
13
|
.record(z.string(), stringTargetSchema)
|
|
14
14
|
.default({})
|
|
15
|
-
.describe('Map of Metabase database ID (positive integer string) to
|
|
15
|
+
.describe('Map of Metabase database ID (positive integer string) to ktx connection ID. Use null to explicitly unmap.'),
|
|
16
16
|
syncEnabled: z
|
|
17
17
|
.record(z.string(), z.boolean())
|
|
18
18
|
.default({})
|
|
@@ -34,7 +34,7 @@ export const lookerMappingsSchema = z
|
|
|
34
34
|
connectionMappings: z
|
|
35
35
|
.record(z.string().min(1), stringTargetSchema)
|
|
36
36
|
.default({})
|
|
37
|
-
.describe('Map of Looker connection name to
|
|
37
|
+
.describe('Map of Looker connection name to ktx connection ID. Use null to explicitly unmap.'),
|
|
38
38
|
})
|
|
39
39
|
.describe('Looker connection-to-warehouse mapping configuration.');
|
|
40
40
|
export const lookmlMappingsSchema = z
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { promises as fs } from 'node:fs';
|
|
2
2
|
import { basename, dirname, join, resolve } from 'node:path';
|
|
3
|
-
import { GitService } from '../../context/core/git.service.js';
|
|
3
|
+
import { classifyKtxRepoOwnership, GitService, KtxForeignGitRepositoryError } from '../../context/core/git.service.js';
|
|
4
4
|
import { noopLogger } from '../../context/core/config.js';
|
|
5
5
|
import { buildDefaultKtxProjectConfig, parseKtxProjectConfig, serializeKtxProjectConfig } from './config.js';
|
|
6
6
|
import { LocalGitFileStore } from './local-git-file-store.js';
|
|
@@ -69,14 +69,22 @@ export async function initKtxProject(options) {
|
|
|
69
69
|
if (!options.force && (await fileExists(configPath))) {
|
|
70
70
|
throw new Error(`Project already contains ktx.yaml: ${configPath}`);
|
|
71
71
|
}
|
|
72
|
+
// Must run before ktx.yaml is written: once that file exists the directory
|
|
73
|
+
// classifies as ktx-managed, so a foreign repo would be silently adopted.
|
|
74
|
+
if ((await classifyKtxRepoOwnership(projectDir)) === 'foreign') {
|
|
75
|
+
throw new KtxForeignGitRepositoryError(projectDir);
|
|
76
|
+
}
|
|
72
77
|
const config = buildDefaultKtxProjectConfig();
|
|
73
|
-
|
|
74
|
-
|
|
78
|
+
// ktx.yaml (the ownership signal) is written before git init, so an
|
|
79
|
+
// interrupted init can never leave a bare `.git` without it — residue that
|
|
80
|
+
// would classify as a foreign repo and be unrecoverable.
|
|
75
81
|
await fs.mkdir(join(projectDir, '.ktx/cache'), { recursive: true });
|
|
76
82
|
for (const file of TRACKED_SCAFFOLD_FILES) {
|
|
77
83
|
await writeProjectFile(projectDir, file.path, file.content);
|
|
78
84
|
}
|
|
79
|
-
|
|
85
|
+
await writeProjectFile(projectDir, 'ktx.yaml', serializeKtxProjectConfig(config));
|
|
86
|
+
const runtime = await createRuntime(projectDir, config, authorName, authorEmail, logger);
|
|
87
|
+
const commit = await runtime.git.commitFiles(['ktx.yaml', ...TRACKED_SCAFFOLD_FILES.map((file) => file.path)], `Initialize ktx project: ${projectName}`, authorName, authorEmail);
|
|
80
88
|
return {
|
|
81
89
|
...runtime,
|
|
82
90
|
commitHash: commit.commitHash,
|
|
@@ -330,7 +330,7 @@ export class KtxDescriptionGenerator {
|
|
|
330
330
|
let fallbackReason = null;
|
|
331
331
|
if (!connector.sampleTable) {
|
|
332
332
|
fallbackReason = 'capability_missing';
|
|
333
|
-
this.logger?.warn('
|
|
333
|
+
this.logger?.warn('ktx scan connector does not support table sampling; falling back to metadata-only prompt', {
|
|
334
334
|
connectorId: input.connector.id,
|
|
335
335
|
table: input.table.name,
|
|
336
336
|
});
|
|
@@ -440,7 +440,7 @@ export class KtxDescriptionGenerator {
|
|
|
440
440
|
let fallbackReason = null;
|
|
441
441
|
if (!input.connector.sampleTable) {
|
|
442
442
|
fallbackReason = 'capability_missing';
|
|
443
|
-
this.logger?.warn('
|
|
443
|
+
this.logger?.warn('ktx scan connector does not support table sampling; falling back to metadata-only prompt', {
|
|
444
444
|
connectorId: input.connector.id,
|
|
445
445
|
table: input.table.name,
|
|
446
446
|
});
|
|
@@ -579,7 +579,7 @@ export class KtxDescriptionGenerator {
|
|
|
579
579
|
}
|
|
580
580
|
}
|
|
581
581
|
if (!input.connector.sampleTable) {
|
|
582
|
-
this.logger?.warn('
|
|
582
|
+
this.logger?.warn('ktx scan connector does not support table sampling for data-source description generation', {
|
|
583
583
|
connectorId: input.connector.id,
|
|
584
584
|
});
|
|
585
585
|
return 'No accessible tables found in database';
|
|
@@ -647,7 +647,7 @@ export class KtxDescriptionGenerator {
|
|
|
647
647
|
let columnValues = column.sampleValues;
|
|
648
648
|
if (!columnValues || columnValues.length === 0) {
|
|
649
649
|
if (!input.connector.sampleColumn) {
|
|
650
|
-
this.logger?.warn('
|
|
650
|
+
this.logger?.warn('ktx scan connector does not support column sampling; using available metadata only', {
|
|
651
651
|
connectorId: input.connector.id,
|
|
652
652
|
table: input.table.name,
|
|
653
653
|
column: column.name,
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import YAML from 'yaml';
|
|
2
2
|
import { buildLiveDatabaseManifestShards } from '../../context/ingest/adapters/live-database/manifest.js';
|
|
3
|
+
import { isSlYamlPath } from '../../context/sl/source-files.js';
|
|
4
|
+
import { deriveFederatedConnection } from '../connections/federation.js';
|
|
3
5
|
import { buildKtxRelationshipArtifacts, buildKtxRelationshipDiagnostics, emptyKtxRelationshipProfileArtifact, } from './relationship-diagnostics.js';
|
|
4
6
|
const LIVE_DATABASE_ADAPTER = 'live-database';
|
|
5
7
|
const LOCAL_AUTHOR = 'ktx';
|
|
@@ -112,7 +114,32 @@ function joinReferencesExistingColumns(join, columnsByTable) {
|
|
|
112
114
|
}
|
|
113
115
|
return true;
|
|
114
116
|
}
|
|
115
|
-
async function
|
|
117
|
+
async function federatedSiblingTargets(project, connectionId) {
|
|
118
|
+
const descriptor = deriveFederatedConnection(project.config.connections, project.projectDir);
|
|
119
|
+
if (!descriptor) {
|
|
120
|
+
return new Set();
|
|
121
|
+
}
|
|
122
|
+
const siblings = descriptor.members.filter((member) => member.connectionId !== connectionId);
|
|
123
|
+
const perSibling = await Promise.all(siblings.map((sibling) => siblingJoinTargets(project, sibling.connectionId)));
|
|
124
|
+
return new Set(perSibling.flat());
|
|
125
|
+
}
|
|
126
|
+
async function siblingJoinTargets(project, connectionId) {
|
|
127
|
+
const listed = await project.fileStore.listFiles(schemaDir(connectionId)).catch(() => ({ files: [] }));
|
|
128
|
+
const files = listed.files.filter(isSlYamlPath);
|
|
129
|
+
const perFile = await Promise.all(files.map(async (file) => {
|
|
130
|
+
const shard = await project.fileStore
|
|
131
|
+
.readFile(file)
|
|
132
|
+
.then(({ content }) => YAML.parse(content))
|
|
133
|
+
.catch(() => null);
|
|
134
|
+
// entry.table is buildTableRef's member-local ref (1-3 parts:
|
|
135
|
+
// table / schema.table / catalog.schema.table), never connectionId-
|
|
136
|
+
// prefixed — so prefixing with the member id yields the fully-qualified
|
|
137
|
+
// `to:` form authored in cross-DB joins.
|
|
138
|
+
return Object.values(shard?.tables ?? {}).map((entry) => `${connectionId}.${entry.table}`);
|
|
139
|
+
}));
|
|
140
|
+
return perFile.flat();
|
|
141
|
+
}
|
|
142
|
+
async function loadExistingManifestState(project, connectionId, snapshot, siblingTargets) {
|
|
116
143
|
const descriptions = new Map();
|
|
117
144
|
const preservedJoins = new Map();
|
|
118
145
|
const usage = new Map();
|
|
@@ -120,7 +147,7 @@ async function loadExistingManifestState(project, connectionId, snapshot) {
|
|
|
120
147
|
const columnsByTable = validColumns(snapshot);
|
|
121
148
|
let files;
|
|
122
149
|
try {
|
|
123
|
-
files = (await project.fileStore.listFiles(schemaDir(connectionId))).files.filter(
|
|
150
|
+
files = (await project.fileStore.listFiles(schemaDir(connectionId))).files.filter(isSlYamlPath);
|
|
124
151
|
}
|
|
125
152
|
catch {
|
|
126
153
|
return { descriptions, preservedJoins, usage };
|
|
@@ -145,7 +172,7 @@ async function loadExistingManifestState(project, connectionId, snapshot) {
|
|
|
145
172
|
}
|
|
146
173
|
const joins = (entry.joins ?? []).filter((join) => {
|
|
147
174
|
return ((join.source === 'manual' || join.source === 'inferred') &&
|
|
148
|
-
validTableNames.has(join.to) &&
|
|
175
|
+
(validTableNames.has(join.to) || siblingTargets.has(join.to)) &&
|
|
149
176
|
joinReferencesExistingColumns(join, columnsByTable));
|
|
150
177
|
});
|
|
151
178
|
if (joins.length > 0) {
|
|
@@ -169,7 +196,8 @@ export async function writeLocalScanManifestShards(input) {
|
|
|
169
196
|
manifestShardsWritten: 0,
|
|
170
197
|
};
|
|
171
198
|
}
|
|
172
|
-
const
|
|
199
|
+
const siblingTargets = await federatedSiblingTargets(input.project, input.connectionId);
|
|
200
|
+
const existing = await loadExistingManifestState(input.project, input.connectionId, input.snapshot, siblingTargets);
|
|
173
201
|
const { shards } = buildLiveDatabaseManifestShards({
|
|
174
202
|
connectionType: input.driver.toUpperCase(),
|
|
175
203
|
tables: snapshotTablesToManifestData(input.snapshot, input.descriptionUpdates),
|
|
@@ -177,6 +205,7 @@ export async function writeLocalScanManifestShards(input) {
|
|
|
177
205
|
existingDescriptions: existing.descriptions,
|
|
178
206
|
existingPreservedJoins: existing.preservedJoins,
|
|
179
207
|
existingUsage: existing.usage,
|
|
208
|
+
federatedSiblingTargets: siblingTargets,
|
|
180
209
|
mapColumnType: (dimensionType) => dimensionType,
|
|
181
210
|
});
|
|
182
211
|
const manifestShards = [];
|
|
@@ -63,7 +63,7 @@ function scanReportPath(connectionId, syncId) {
|
|
|
63
63
|
}
|
|
64
64
|
function assertSupportedMode(mode) {
|
|
65
65
|
if (mode !== 'structural' && mode !== 'relationships' && mode !== 'enriched') {
|
|
66
|
-
throw new Error(`Unsupported
|
|
66
|
+
throw new Error(`Unsupported ktx scan mode: ${mode}`);
|
|
67
67
|
}
|
|
68
68
|
}
|
|
69
69
|
async function resolveScanConnector(options, mode) {
|
|
@@ -381,7 +381,7 @@ export async function runLocalScan(options) {
|
|
|
381
381
|
}
|
|
382
382
|
report.warnings.push({
|
|
383
383
|
code: 'enrichment_failed',
|
|
384
|
-
message: `
|
|
384
|
+
message: `ktx scan enrichment failed after structural scan completed: ${message}`,
|
|
385
385
|
recoverable: true,
|
|
386
386
|
metadata: { mode, detectRelationships: options.detectRelationships ?? false },
|
|
387
387
|
});
|
|
@@ -25,7 +25,7 @@ function parseWarning(rawWarning, path) {
|
|
|
25
25
|
!scanWarningCodes.has(rawWarning.code) ||
|
|
26
26
|
typeof rawWarning.message !== 'string' ||
|
|
27
27
|
typeof rawWarning.recoverable !== 'boolean') {
|
|
28
|
-
throw new Error(`Invalid
|
|
28
|
+
throw new Error(`Invalid ktx schema warning artifact: ${path}`);
|
|
29
29
|
}
|
|
30
30
|
return {
|
|
31
31
|
code: rawWarning.code,
|
|
@@ -42,7 +42,7 @@ async function readWarnings(input) {
|
|
|
42
42
|
const warningRaw = await input.project.fileStore.readFile(path);
|
|
43
43
|
const parsed = JSON.parse(warningRaw.content);
|
|
44
44
|
if (!isRecord(parsed) || !Array.isArray(parsed.warnings)) {
|
|
45
|
-
throw new Error(`Invalid
|
|
45
|
+
throw new Error(`Invalid ktx schema warnings artifact: ${path}`);
|
|
46
46
|
}
|
|
47
47
|
return parsed.warnings.map((warning) => parseWarning(warning, path));
|
|
48
48
|
}
|
|
@@ -68,7 +68,7 @@ function parseColumn(rawColumn, path) {
|
|
|
68
68
|
rawColumn.dimensionType !== 'string' &&
|
|
69
69
|
rawColumn.dimensionType !== 'number' &&
|
|
70
70
|
rawColumn.dimensionType !== 'boolean')) {
|
|
71
|
-
throw new Error(`Invalid
|
|
71
|
+
throw new Error(`Invalid ktx schema column artifact: ${path}`);
|
|
72
72
|
}
|
|
73
73
|
return {
|
|
74
74
|
name: rawColumn.name,
|
|
@@ -85,7 +85,7 @@ function parseForeignKey(rawForeignKey, path) {
|
|
|
85
85
|
typeof rawForeignKey.fromColumn !== 'string' ||
|
|
86
86
|
typeof rawForeignKey.toTable !== 'string' ||
|
|
87
87
|
typeof rawForeignKey.toColumn !== 'string') {
|
|
88
|
-
throw new Error(`Invalid
|
|
88
|
+
throw new Error(`Invalid ktx schema foreign key artifact: ${path}`);
|
|
89
89
|
}
|
|
90
90
|
return {
|
|
91
91
|
fromColumn: rawForeignKey.fromColumn,
|
|
@@ -99,7 +99,7 @@ function parseForeignKey(rawForeignKey, path) {
|
|
|
99
99
|
function parseTable(raw, path) {
|
|
100
100
|
const parsed = JSON.parse(raw);
|
|
101
101
|
if (!isRecord(parsed) || typeof parsed.name !== 'string' || !Array.isArray(parsed.columns)) {
|
|
102
|
-
throw new Error(`Invalid
|
|
102
|
+
throw new Error(`Invalid ktx schema table artifact: ${path}`);
|
|
103
103
|
}
|
|
104
104
|
return {
|
|
105
105
|
catalog: optionalStringOrNull(parsed.catalog) ?? null,
|
|
@@ -211,7 +211,7 @@ function compositeSkipBlocks(report) {
|
|
|
211
211
|
}
|
|
212
212
|
export function formatKtxRelationshipBenchmarkReportMarkdown(report) {
|
|
213
213
|
const lines = [
|
|
214
|
-
'#
|
|
214
|
+
'# ktx Relationship Discovery Benchmark Evidence',
|
|
215
215
|
'',
|
|
216
216
|
`Generated: ${report.generatedAt}`,
|
|
217
217
|
'',
|
|
@@ -87,7 +87,7 @@ async function detectCompositeRelationships(input) {
|
|
|
87
87
|
catch (error) {
|
|
88
88
|
input.warnings.push({
|
|
89
89
|
code: 'relationship_validation_failed',
|
|
90
|
-
message: `
|
|
90
|
+
message: `ktx composite relationship detection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
91
91
|
recoverable: true,
|
|
92
92
|
metadata: { source: 'composite_relationship_detection' },
|
|
93
93
|
});
|
|
@@ -110,7 +110,7 @@ function sqlExecutor(input) {
|
|
|
110
110
|
warnings: [
|
|
111
111
|
{
|
|
112
112
|
code: 'connector_capability_missing',
|
|
113
|
-
message: '
|
|
113
|
+
message: 'ktx scan connector cannot run read-only SQL relationship validation',
|
|
114
114
|
recoverable: true,
|
|
115
115
|
metadata: { capability: 'readOnlySql' },
|
|
116
116
|
},
|
|
@@ -123,7 +123,7 @@ function sqlExecutor(input) {
|
|
|
123
123
|
warnings: [
|
|
124
124
|
{
|
|
125
125
|
code: 'relationship_validation_failed',
|
|
126
|
-
message: '
|
|
126
|
+
message: 'ktx scan connector advertises readOnlySql but does not expose executeReadOnly',
|
|
127
127
|
recoverable: true,
|
|
128
128
|
metadata: { capability: 'readOnlySql' },
|
|
129
129
|
},
|
|
@@ -116,7 +116,7 @@ function mapValidProposals(schema, output, settings) {
|
|
|
116
116
|
const fromColumn = fromTable ? findColumn(fromTable, item.fromColumn) : null;
|
|
117
117
|
const toColumn = toTable ? findColumn(toTable, item.toColumn) : null;
|
|
118
118
|
if (!fromTable || !toTable || !fromColumn || !toColumn) {
|
|
119
|
-
warnings.push(invalidReferenceWarning('
|
|
119
|
+
warnings.push(invalidReferenceWarning('ktx relationship LLM proposal referenced a table or column that is not in the schema.', {
|
|
120
120
|
proposal: item,
|
|
121
121
|
}));
|
|
122
122
|
continue;
|
|
@@ -148,7 +148,7 @@ function generationFailureWarning(error) {
|
|
|
148
148
|
const message = error instanceof Error ? error.message : String(error);
|
|
149
149
|
return {
|
|
150
150
|
code: 'relationship_llm_proposal_failed',
|
|
151
|
-
message: `
|
|
151
|
+
message: `ktx relationship LLM proposal failed: ${message}`,
|
|
152
152
|
recoverable: true,
|
|
153
153
|
};
|
|
154
154
|
}
|
|
@@ -159,7 +159,7 @@ export async function proposeKtxRelationshipCandidatesWithLlm(input) {
|
|
|
159
159
|
const settings = mergeSettings(input.settings);
|
|
160
160
|
const evidence = buildEvidencePacket(input.schema, input.profile, settings);
|
|
161
161
|
const system = [
|
|
162
|
-
'You are helping
|
|
162
|
+
'You are helping ktx review possible SQL relationships before validation.',
|
|
163
163
|
'Use only the compact schema evidence. Propose likely primary keys and foreign keys for later SQL validation.',
|
|
164
164
|
'Return structured output only; never assume a join is accepted.',
|
|
165
165
|
].join('\n');
|
|
@@ -1,52 +1,35 @@
|
|
|
1
|
+
import { FEDERATED_CONNECTION_ID } from '../connections/federation.js';
|
|
2
|
+
import { resolveRequiredConnectionId } from '../connections/resolve-connection.js';
|
|
3
|
+
import { sqlAnalysisDialectForDriver } from '../sql-analysis/dialect.js';
|
|
1
4
|
import { loadLocalSlSourceRecords } from './local-sl.js';
|
|
2
5
|
import { toResolvedWire } from './semantic-layer.service.js';
|
|
6
|
+
import { assertSafeConnectionId } from './source-files.js';
|
|
3
7
|
const COMPILE_ONLY_REASON = 'Local semantic-layer query compiled SQL but no data-source execution adapter is configured.';
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
value.startsWith('/') ||
|
|
9
|
-
value.startsWith('.') ||
|
|
10
|
-
value.includes('//')) {
|
|
11
|
-
throw new Error(`Unsafe ${kind}: ${value}`);
|
|
12
|
-
}
|
|
13
|
-
return value;
|
|
14
|
-
}
|
|
15
|
-
function assertSafeConnectionId(connectionId) {
|
|
16
|
-
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
|
|
17
|
-
throw new Error(`Unsafe connection id: ${connectionId}`);
|
|
18
|
-
}
|
|
19
|
-
return assertSafePathToken('connection id', connectionId);
|
|
20
|
-
}
|
|
21
|
-
function dialectForDriver(driver) {
|
|
22
|
-
const normalized = (driver ?? 'postgres').toUpperCase();
|
|
23
|
-
const map = {
|
|
24
|
-
POSTGRES: 'postgres',
|
|
25
|
-
BIGQUERY: 'bigquery',
|
|
26
|
-
SNOWFLAKE: 'snowflake',
|
|
27
|
-
MYSQL: 'mysql',
|
|
28
|
-
SQLSERVER: 'tsql',
|
|
29
|
-
SQLITE: 'sqlite',
|
|
30
|
-
DUCKDB: 'duckdb',
|
|
31
|
-
CLICKHOUSE: 'clickhouse',
|
|
32
|
-
DATABRICKS: 'databricks',
|
|
33
|
-
};
|
|
34
|
-
return map[normalized] ?? 'postgres';
|
|
35
|
-
}
|
|
8
|
+
const FEDERATED_SL_QUERY_UNSUPPORTED = `Semantic-layer queries are per-connection and cannot target the federated connection '${FEDERATED_CONNECTION_ID}'. ` +
|
|
9
|
+
`Run a cross-database query as read-only SQL instead — ktx sql -c ${FEDERATED_CONNECTION_ID} "SELECT ..." or the sql_execution tool — ` +
|
|
10
|
+
'using catalog-qualified table names (connectionId.schema.table, or connectionId.table for sqlite; ' +
|
|
11
|
+
'double-quote ids that are not bare identifiers, e.g. "books-db".public.books).';
|
|
36
12
|
function resolveLocalConnectionId(project, requested) {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
13
|
+
return assertSafeConnectionId(resolveRequiredConnectionId(project.config, requested));
|
|
14
|
+
}
|
|
15
|
+
// The planner rejects a source set carrying a join whose `to` names a source
|
|
16
|
+
// outside that set, which would break every query for this connection. Keep only
|
|
17
|
+
// joins resolvable within the connection's own sources; a cross-database join
|
|
18
|
+
// (its `to` qualified by a sibling connection id) is just one such unresolvable
|
|
19
|
+
// target and runs as raw SQL instead. Membership is the test, not a connection-id
|
|
20
|
+
// prefix match, so a same-connection target whose name collides with a sibling
|
|
21
|
+
// connection id is preserved.
|
|
22
|
+
function withResolvableJoinsOnly(source, knownSourceNames) {
|
|
23
|
+
if (source.joins.length === 0) {
|
|
24
|
+
return source;
|
|
43
25
|
}
|
|
44
|
-
|
|
26
|
+
const joins = source.joins.filter((join) => knownSourceNames.has(join.to));
|
|
27
|
+
return joins.length === source.joins.length ? source : { ...source, joins };
|
|
45
28
|
}
|
|
46
29
|
async function loadComputableSources(project, connectionId) {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
30
|
+
const records = (await loadLocalSlSourceRecords(project, { connectionId })).filter((record) => record.source.table || record.source.sql);
|
|
31
|
+
const knownSourceNames = new Set(records.map((record) => record.source.name));
|
|
32
|
+
return records.map((record) => toResolvedWire(withResolvableJoinsOnly(record.source, knownSourceNames)));
|
|
50
33
|
}
|
|
51
34
|
function headersFromColumns(columns) {
|
|
52
35
|
return columns
|
|
@@ -54,9 +37,13 @@ function headersFromColumns(columns) {
|
|
|
54
37
|
.filter((name) => typeof name === 'string' && name.length > 0);
|
|
55
38
|
}
|
|
56
39
|
export async function compileLocalSlQuery(project, options) {
|
|
40
|
+
if (options.connectionId === FEDERATED_CONNECTION_ID) {
|
|
41
|
+
throw new Error(FEDERATED_SL_QUERY_UNSUPPORTED);
|
|
42
|
+
}
|
|
57
43
|
await options.onProgress?.({ progress: 0, message: 'Compiling query' });
|
|
58
44
|
const connectionId = resolveLocalConnectionId(project, options.connectionId);
|
|
59
|
-
const
|
|
45
|
+
const driver = project.config.connections[connectionId]?.driver;
|
|
46
|
+
const dialect = sqlAnalysisDialectForDriver(driver);
|
|
60
47
|
const sources = await loadComputableSources(project, connectionId);
|
|
61
48
|
await options.onProgress?.({ progress: 0.3, message: 'Generating SQL' });
|
|
62
49
|
const response = await options.compute.query({
|
|
@@ -106,7 +93,7 @@ export async function compileLocalSlQuery(project, options) {
|
|
|
106
93
|
...response.plan,
|
|
107
94
|
execution: {
|
|
108
95
|
mode: 'executed',
|
|
109
|
-
driver:
|
|
96
|
+
driver: driver ?? 'unknown',
|
|
110
97
|
maxRows,
|
|
111
98
|
rowCount: execution.rowCount,
|
|
112
99
|
},
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { KtxEmbeddingPort } from '../../context/core/embedding.js';
|
|
2
|
-
import type { KtxFileWriteResult } from '../../context/core/file-store.js';
|
|
3
2
|
import type { KtxLocalProject } from '../../context/project/project.js';
|
|
4
3
|
import type { PgliteSlSearchPrototypeOwnerOptions } from './pglite-sl-search-prototype.js';
|
|
5
4
|
import type { SemanticLayerSource, SlDictionaryMatch, SlSearchLaneSummary, SlSearchMatchReason } from './types.js';
|
|
@@ -56,13 +55,6 @@ export declare function validateLocalSlSource(rawYaml: string, options?: {
|
|
|
56
55
|
connectionId?: string;
|
|
57
56
|
sourceName?: string;
|
|
58
57
|
}): Promise<LocalSlValidationResult>;
|
|
59
|
-
/** @internal */
|
|
60
|
-
export declare function writeLocalSlSource(project: KtxLocalProject, input: {
|
|
61
|
-
connectionId: string;
|
|
62
|
-
sourceName: string;
|
|
63
|
-
yaml: string;
|
|
64
|
-
}): Promise<KtxFileWriteResult>;
|
|
65
|
-
/** @internal */
|
|
66
58
|
export declare function readLocalSlSource(project: KtxLocalProject, input: {
|
|
67
59
|
connectionId: string;
|
|
68
60
|
sourceName: string;
|