@kaelio/ktx 0.12.0 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.12.0-py3-none-any.whl → kaelio_ktx-0.13.1-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/commands/setup-commands.js +13 -0
- package/dist/connection.js +14 -2
- package/dist/connectors/bigquery/connector.js +1 -14
- package/dist/connectors/clickhouse/connector.js +1 -15
- package/dist/connectors/duckdb/federated-attach.d.ts +7 -0
- package/dist/connectors/duckdb/federated-attach.js +86 -0
- package/dist/connectors/duckdb/federated-executor.d.ts +5 -0
- package/dist/connectors/duckdb/federated-executor.js +59 -0
- package/dist/connectors/mysql/connector.js +1 -15
- package/dist/connectors/postgres/connector.js +1 -14
- package/dist/connectors/shared/string-reference.d.ts +6 -0
- package/dist/connectors/shared/string-reference.js +19 -0
- package/dist/connectors/snowflake/connector.js +1 -14
- package/dist/connectors/sqlserver/connector.js +4 -16
- package/dist/context/connections/federation.d.ts +33 -0
- package/dist/context/connections/federation.js +51 -0
- package/dist/context/connections/local-warehouse-descriptor.d.ts +2 -0
- package/dist/context/connections/project-sql-executor.d.ts +18 -0
- package/dist/context/connections/project-sql-executor.js +39 -0
- package/dist/context/connections/query-executor.d.ts +2 -2
- package/dist/context/connections/read-only-sql.d.ts +5 -0
- package/dist/context/connections/read-only-sql.js +143 -4
- package/dist/context/connections/resolve-connection.d.ts +12 -0
- package/dist/context/connections/resolve-connection.js +37 -0
- package/dist/context/core/git-env.d.ts +4 -0
- package/dist/context/core/git-env.js +5 -1
- package/dist/context/ingest/adapters/live-database/manifest.d.ts +3 -0
- package/dist/context/ingest/adapters/live-database/manifest.js +19 -11
- package/dist/context/llm/claude-code-runtime.js +18 -2
- package/dist/context/mcp/context-tools.js +27 -2
- package/dist/context/mcp/local-project-ports.js +55 -50
- package/dist/context/mcp/types.d.ts +2 -0
- package/dist/context/scan/local-enrichment-artifacts.js +31 -3
- package/dist/context/sl/local-query.js +29 -12
- package/dist/context/sl/local-sl.js +27 -1
- package/dist/context/sl/source-files.d.ts +2 -0
- package/dist/context/sl/source-files.js +7 -0
- package/dist/ingest-query-executor.d.ts +2 -0
- package/dist/ingest-query-executor.js +8 -22
- package/dist/setup-agents.d.ts +21 -15
- package/dist/setup-agents.js +128 -42
- package/dist/setup-databases.d.ts +3 -0
- package/dist/setup-databases.js +16 -0
- package/dist/setup-sources.js +1 -5
- package/dist/setup.d.ts +1 -0
- package/dist/setup.js +1 -0
- package/dist/sql.d.ts +2 -0
- package/dist/sql.js +35 -53
- package/dist/telemetry/events.d.ts +2 -1
- package/dist/telemetry/events.js +11 -1
- package/package.json +2 -1
|
@@ -84,6 +84,7 @@ function shouldShowSetupEntryMenu(options, command) {
|
|
|
84
84
|
'target',
|
|
85
85
|
'global',
|
|
86
86
|
'local',
|
|
87
|
+
'installDir',
|
|
87
88
|
'skipAgents',
|
|
88
89
|
'yes',
|
|
89
90
|
'input',
|
|
@@ -139,6 +140,7 @@ export function registerSetupCommands(program, context) {
|
|
|
139
140
|
]))
|
|
140
141
|
.option('--global', 'Install agent integration into the global target scope', false)
|
|
141
142
|
.option('--local', 'Install Claude Code MCP config into the private per-project ~/.claude.json scope', false)
|
|
143
|
+
.option('--install-dir <path>', 'Directory to install project-scoped agent config into (defaults to the ktx project directory)')
|
|
142
144
|
.addOption(new Option('--skip-agents', 'Leave agent integration incomplete for now').hideHelp().default(false))
|
|
143
145
|
.option('--yes', 'Accept project creation and runtime install defaults where setup confirms', false)
|
|
144
146
|
.option('--no-input', 'Disable interactive terminal input')
|
|
@@ -264,6 +266,16 @@ export function registerSetupCommands(program, context) {
|
|
|
264
266
|
context.setExitCode(1);
|
|
265
267
|
return;
|
|
266
268
|
}
|
|
269
|
+
if (options.installDir && (options.global || options.local)) {
|
|
270
|
+
context.io.stderr.write('Choose either --install-dir or a scope flag (--global / --local), not both.\n');
|
|
271
|
+
context.setExitCode(1);
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
if (options.installDir && options.target === 'claude-desktop') {
|
|
275
|
+
context.io.stderr.write('--install-dir does not apply to --target claude-desktop, which is always global.\n');
|
|
276
|
+
context.setExitCode(1);
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
267
279
|
const creatingDatabaseConnection = options.database.length > 0 || options.databaseUrl !== undefined;
|
|
268
280
|
if (creatingDatabaseConnection && options.databaseConnectionId.length > 1) {
|
|
269
281
|
context.io.stderr.write('Choose only one new database connection id when configuring a database.\n');
|
|
@@ -279,6 +291,7 @@ export function registerSetupCommands(program, context) {
|
|
|
279
291
|
agents: options.agents === true,
|
|
280
292
|
...(options.target ? { target: options.target } : {}),
|
|
281
293
|
agentScope: resolvedAgentScope,
|
|
294
|
+
...(options.installDir ? { installRoot: options.installDir } : {}),
|
|
282
295
|
skipAgents: options.skipAgents === true,
|
|
283
296
|
inputMode: options.input === false ? 'disabled' : 'auto',
|
|
284
297
|
...(debugEnabled ? { debug: true } : {}),
|
package/dist/connection.js
CHANGED
|
@@ -4,6 +4,7 @@ import { NotionClient } from './context/ingest/adapters/notion/notion-client.js'
|
|
|
4
4
|
import { createLocalLookerCredentialResolver } from './context/ingest/adapters/looker/local-looker.adapter.js';
|
|
5
5
|
import { metabaseRuntimeConfigFromLocalConnection } from './context/ingest/adapters/metabase/local-metabase.adapter.js';
|
|
6
6
|
import { testRepoConnection } from './context/ingest/repo-fetch.js';
|
|
7
|
+
import { federatedConnectionListing } from './context/connections/federation.js';
|
|
7
8
|
import { getDriverRegistration } from './context/connections/drivers.js';
|
|
8
9
|
import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from './context/connections/notion-config.js';
|
|
9
10
|
import { resolveKtxConfigReference } from './context/core/config-reference.js';
|
|
@@ -292,12 +293,23 @@ export async function runKtxConnection(args, io = process, deps = {}) {
|
|
|
292
293
|
io.stdout.write('No connections configured. Run `ktx setup` to add one.\n');
|
|
293
294
|
return 0;
|
|
294
295
|
}
|
|
295
|
-
const
|
|
296
|
-
const
|
|
296
|
+
const federated = federatedConnectionListing(project.config.connections, args.projectDir);
|
|
297
|
+
const idCandidates = [...entries.map(([id]) => id), ...(federated ? [federated.id] : [])];
|
|
298
|
+
const driverLengths = [
|
|
299
|
+
...entries.map(([, c]) => (c.driver ?? 'unknown').length),
|
|
300
|
+
...(federated ? [federated.driver.length] : []),
|
|
301
|
+
];
|
|
302
|
+
const idWidth = Math.max('ID'.length, ...idCandidates.map((id) => id.length));
|
|
303
|
+
const driverWidth = Math.max('DRIVER'.length, ...driverLengths);
|
|
297
304
|
io.stdout.write(`${'ID'.padEnd(idWidth)} ${'DRIVER'.padEnd(driverWidth)}\n`);
|
|
298
305
|
for (const [id, connection] of entries) {
|
|
299
306
|
io.stdout.write(`${id.padEnd(idWidth)} ${(connection.driver ?? 'unknown').padEnd(driverWidth)}\n`);
|
|
300
307
|
}
|
|
308
|
+
if (federated) {
|
|
309
|
+
io.stdout.write(`${federated.id.padEnd(idWidth)} ${federated.driver.padEnd(driverWidth)}\n`);
|
|
310
|
+
io.stdout.write(` federates: ${federated.members.join(', ')}\n`);
|
|
311
|
+
io.stdout.write(` ${federated.hint}\n`);
|
|
312
|
+
}
|
|
301
313
|
return 0;
|
|
302
314
|
}
|
|
303
315
|
if (args.command === 'test-all') {
|
|
@@ -5,9 +5,7 @@ import { assertReadOnlySql, limitSqlForExecution } from '../../context/connectio
|
|
|
5
5
|
import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
|
|
6
6
|
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
|
7
7
|
import { connectorTestFailure, createKtxConnectorCapabilities, } from '../../context/scan/types.js';
|
|
8
|
-
import {
|
|
9
|
-
import { homedir } from 'node:os';
|
|
10
|
-
import { resolve } from 'node:path';
|
|
8
|
+
import { resolveStringReference } from '../shared/string-reference.js';
|
|
11
9
|
class DefaultBigQueryClientFactory {
|
|
12
10
|
createClient(input) {
|
|
13
11
|
const client = new BigQuery(input);
|
|
@@ -24,17 +22,6 @@ class DefaultBigQueryClientFactory {
|
|
|
24
22
|
};
|
|
25
23
|
}
|
|
26
24
|
}
|
|
27
|
-
function resolveStringReference(value, env) {
|
|
28
|
-
if (value.startsWith('env:')) {
|
|
29
|
-
return env[value.slice('env:'.length)] ?? '';
|
|
30
|
-
}
|
|
31
|
-
if (value.startsWith('file:')) {
|
|
32
|
-
const rawPath = value.slice('file:'.length);
|
|
33
|
-
const path = rawPath.startsWith('~') ? resolve(homedir(), rawPath.slice(1)) : rawPath;
|
|
34
|
-
return readFileSync(path, 'utf-8').trim();
|
|
35
|
-
}
|
|
36
|
-
return value;
|
|
37
|
-
}
|
|
38
25
|
function stringConfigValue(connection, key, env) {
|
|
39
26
|
const value = connection?.[key];
|
|
40
27
|
return typeof value === 'string' && value.trim().length > 0 ? resolveStringReference(value.trim(), env) : undefined;
|
|
@@ -3,10 +3,8 @@ import { getDialectForDriver } from '../../context/connections/dialects.js';
|
|
|
3
3
|
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
|
4
4
|
import { connectorTestFailure, createKtxConnectorCapabilities } from '../../context/scan/types.js';
|
|
5
5
|
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
|
6
|
-
import {
|
|
6
|
+
import { resolveStringReference } from '../shared/string-reference.js';
|
|
7
7
|
import { Agent as HttpsAgent } from 'node:https';
|
|
8
|
-
import { homedir } from 'node:os';
|
|
9
|
-
import { resolve } from 'node:path';
|
|
10
8
|
class DefaultClickHouseClientFactory {
|
|
11
9
|
createClient(config) {
|
|
12
10
|
return createClient(config);
|
|
@@ -16,18 +14,6 @@ function stringConfigValue(connection, key, env) {
|
|
|
16
14
|
const value = connection?.[key];
|
|
17
15
|
return typeof value === 'string' && value.trim().length > 0 ? resolveStringReference(value.trim(), env) : undefined;
|
|
18
16
|
}
|
|
19
|
-
function resolveStringReference(value, env) {
|
|
20
|
-
if (value.startsWith('env:')) {
|
|
21
|
-
const envName = value.slice('env:'.length);
|
|
22
|
-
return env[envName] ?? '';
|
|
23
|
-
}
|
|
24
|
-
if (value.startsWith('file:')) {
|
|
25
|
-
const rawPath = value.slice('file:'.length);
|
|
26
|
-
const path = rawPath.startsWith('~') ? resolve(homedir(), rawPath.slice(1)) : rawPath;
|
|
27
|
-
return readFileSync(path, 'utf-8').trim();
|
|
28
|
-
}
|
|
29
|
-
return value;
|
|
30
|
-
}
|
|
31
17
|
function maybeNumber(value) {
|
|
32
18
|
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
|
33
19
|
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { FederatedMember } from '../../context/connections/federation.js';
|
|
2
|
+
/**
|
|
3
|
+
* Resolves a federated member's ktx.yaml config into the connection target
|
|
4
|
+
* DuckDB's ATTACH wants for that driver, reusing each connector's canonical
|
|
5
|
+
* resolver so federation and standalone scans agree on config interpretation.
|
|
6
|
+
*/
|
|
7
|
+
export declare function federatedAttachTarget(member: FederatedMember, env: NodeJS.ProcessEnv): string;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { sqliteDatabasePathFromConfig } from '../sqlite/connector.js';
|
|
2
|
+
import { postgresPoolConfigFromConfig } from '../postgres/connector.js';
|
|
3
|
+
import { mysqlConnectionPoolConfigFromConfig, } from '../mysql/connector.js';
|
|
4
|
+
function kvKeyword(value) {
|
|
5
|
+
// libpq/DuckDB key-value values quote with single quotes and backslash-escape.
|
|
6
|
+
return /[\s'\\]/.test(value) ? `'${value.replaceAll('\\', '\\\\').replaceAll("'", "\\'")}'` : value;
|
|
7
|
+
}
|
|
8
|
+
function withRequiredSslMode(connectionString) {
|
|
9
|
+
// DuckDB passes this libpq URL straight to the server, so an ssl:true member
|
|
10
|
+
// must carry sslmode in the URL itself; keep a stronger mode the URL already pins.
|
|
11
|
+
const url = new URL(connectionString);
|
|
12
|
+
if (url.searchParams.has('sslmode')) {
|
|
13
|
+
return connectionString;
|
|
14
|
+
}
|
|
15
|
+
url.searchParams.set('sslmode', 'require');
|
|
16
|
+
return url.toString();
|
|
17
|
+
}
|
|
18
|
+
function postgresAttachString(member, env) {
|
|
19
|
+
const cfg = postgresPoolConfigFromConfig({
|
|
20
|
+
connectionId: member.connectionId,
|
|
21
|
+
connection: member.connection,
|
|
22
|
+
env,
|
|
23
|
+
});
|
|
24
|
+
if (cfg.connectionString) {
|
|
25
|
+
return cfg.ssl ? withRequiredSslMode(cfg.connectionString) : cfg.connectionString;
|
|
26
|
+
}
|
|
27
|
+
const parts = [];
|
|
28
|
+
if (cfg.host)
|
|
29
|
+
parts.push(`host=${kvKeyword(cfg.host)}`);
|
|
30
|
+
if (cfg.port)
|
|
31
|
+
parts.push(`port=${cfg.port}`);
|
|
32
|
+
if (cfg.database)
|
|
33
|
+
parts.push(`dbname=${kvKeyword(cfg.database)}`);
|
|
34
|
+
if (cfg.user)
|
|
35
|
+
parts.push(`user=${kvKeyword(cfg.user)}`);
|
|
36
|
+
if (cfg.password)
|
|
37
|
+
parts.push(`password=${kvKeyword(cfg.password)}`);
|
|
38
|
+
if (cfg.ssl) {
|
|
39
|
+
parts.push('sslmode=require');
|
|
40
|
+
}
|
|
41
|
+
if (cfg.options) {
|
|
42
|
+
parts.push(`options=${kvKeyword(cfg.options)}`);
|
|
43
|
+
}
|
|
44
|
+
return parts.join(' ');
|
|
45
|
+
}
|
|
46
|
+
function mysqlAttachString(member, env) {
|
|
47
|
+
const cfg = mysqlConnectionPoolConfigFromConfig({
|
|
48
|
+
connectionId: member.connectionId,
|
|
49
|
+
connection: member.connection,
|
|
50
|
+
env,
|
|
51
|
+
});
|
|
52
|
+
const parts = [
|
|
53
|
+
`host=${kvKeyword(cfg.host)}`,
|
|
54
|
+
`port=${cfg.port}`,
|
|
55
|
+
`database=${kvKeyword(cfg.database)}`,
|
|
56
|
+
`user=${kvKeyword(cfg.user)}`,
|
|
57
|
+
];
|
|
58
|
+
if (cfg.password) {
|
|
59
|
+
parts.push(`password=${kvKeyword(cfg.password)}`);
|
|
60
|
+
}
|
|
61
|
+
if (cfg.ssl) {
|
|
62
|
+
parts.push('ssl_mode=REQUIRED');
|
|
63
|
+
}
|
|
64
|
+
return parts.join(' ');
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Resolves a federated member's ktx.yaml config into the connection target
|
|
68
|
+
* DuckDB's ATTACH wants for that driver, reusing each connector's canonical
|
|
69
|
+
* resolver so federation and standalone scans agree on config interpretation.
|
|
70
|
+
*/
|
|
71
|
+
export function federatedAttachTarget(member, env) {
|
|
72
|
+
switch (member.driver.toLowerCase()) {
|
|
73
|
+
case 'sqlite':
|
|
74
|
+
return sqliteDatabasePathFromConfig({
|
|
75
|
+
connectionId: member.connectionId,
|
|
76
|
+
projectDir: member.projectDir,
|
|
77
|
+
connection: member.connection,
|
|
78
|
+
});
|
|
79
|
+
case 'postgres':
|
|
80
|
+
return postgresAttachString(member, env);
|
|
81
|
+
case 'mysql':
|
|
82
|
+
return mysqlAttachString(member, env);
|
|
83
|
+
default:
|
|
84
|
+
throw new Error(`Driver "${member.driver}" cannot be attached by DuckDB federation.`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { KtxSqlQueryExecutionInput, KtxSqlQueryExecutionResult } from '../../context/connections/query-executor.js';
|
|
2
|
+
import { type FederatedMember } from '../../context/connections/federation.js';
|
|
3
|
+
/** @internal */
|
|
4
|
+
export declare function buildAttachStatements(members: FederatedMember[], env: NodeJS.ProcessEnv): string[];
|
|
5
|
+
export declare function executeFederatedQuery(members: FederatedMember[], input: KtxSqlQueryExecutionInput, env?: NodeJS.ProcessEnv): Promise<KtxSqlQueryExecutionResult>;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { DuckDBInstance } from '@duckdb/node-api';
|
|
2
|
+
import { federatedAttachTarget } from './federated-attach.js';
|
|
3
|
+
import { normalizeQueryRows } from '../../context/connections/query-executor.js';
|
|
4
|
+
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
|
5
|
+
import { attachTypeForDriver } from '../../context/connections/federation.js';
|
|
6
|
+
function quoteDuckdbIdentifier(id) {
|
|
7
|
+
return `"${id.replaceAll('"', '""')}"`;
|
|
8
|
+
}
|
|
9
|
+
const MIN_SAFE_BIGINT = BigInt(Number.MIN_SAFE_INTEGER);
|
|
10
|
+
const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
|
|
11
|
+
// DuckDB returns integer columns as JS bigint (unserializable by JSON). Values
|
|
12
|
+
// in Number's safe range become Number; larger magnitudes become strings so a
|
|
13
|
+
// BIGINT beyond 2^53 keeps its exact value instead of silently rounding.
|
|
14
|
+
function jsonSafeBigint(value) {
|
|
15
|
+
return value >= MIN_SAFE_BIGINT && value <= MAX_SAFE_BIGINT ? Number(value) : value.toString();
|
|
16
|
+
}
|
|
17
|
+
function toJsonSafeRows(rows) {
|
|
18
|
+
return rows.map((row) => row.map((cell) => (typeof cell === 'bigint' ? jsonSafeBigint(cell) : cell)));
|
|
19
|
+
}
|
|
20
|
+
/** @internal */
|
|
21
|
+
export function buildAttachStatements(members, env) {
|
|
22
|
+
const attachments = members.map((member) => ({
|
|
23
|
+
type: attachTypeForDriver(member.driver),
|
|
24
|
+
url: federatedAttachTarget(member, env),
|
|
25
|
+
alias: member.connectionId,
|
|
26
|
+
}));
|
|
27
|
+
const loadStatements = [...new Set(attachments.map((a) => a.type))].map((type) => `INSTALL ${type}; LOAD ${type};`);
|
|
28
|
+
const attachStatements = attachments.map(({ type, url, alias }) => `ATTACH '${url.replaceAll("'", "''")}' AS ${quoteDuckdbIdentifier(alias)} (TYPE ${type}, READ_ONLY);`);
|
|
29
|
+
return [...loadStatements, ...attachStatements];
|
|
30
|
+
}
|
|
31
|
+
export async function executeFederatedQuery(members, input, env = process.env) {
|
|
32
|
+
const sql = limitSqlForExecution(assertReadOnlySql(input.sql), input.maxRows);
|
|
33
|
+
const attachStatements = buildAttachStatements(members, env);
|
|
34
|
+
const instance = await DuckDBInstance.create(':memory:');
|
|
35
|
+
try {
|
|
36
|
+
const connection = await instance.connect();
|
|
37
|
+
try {
|
|
38
|
+
for (const statement of attachStatements) {
|
|
39
|
+
await connection.run(statement);
|
|
40
|
+
}
|
|
41
|
+
const reader = await connection.runAndReadAll(sql);
|
|
42
|
+
const rows = toJsonSafeRows(normalizeQueryRows(reader.getRows()));
|
|
43
|
+
const headers = reader.columnNames();
|
|
44
|
+
return {
|
|
45
|
+
headers,
|
|
46
|
+
rows,
|
|
47
|
+
totalRows: rows.length,
|
|
48
|
+
command: 'SELECT',
|
|
49
|
+
rowCount: rows.length,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
finally {
|
|
53
|
+
connection.closeSync();
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
finally {
|
|
57
|
+
instance.closeSync();
|
|
58
|
+
}
|
|
59
|
+
}
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import mysql from 'mysql2/promise';
|
|
2
|
-
import { readFileSync } from 'node:fs';
|
|
3
|
-
import { homedir } from 'node:os';
|
|
4
|
-
import { resolve } from 'node:path';
|
|
5
2
|
import { getDialectForDriver } from '../../context/connections/dialects.js';
|
|
3
|
+
import { resolveStringReference } from '../shared/string-reference.js';
|
|
6
4
|
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
|
7
5
|
import { constraintDiscoveryWarning, tryConstraintQuery, } from '../../context/scan/constraint-discovery.js';
|
|
8
6
|
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
|
@@ -16,18 +14,6 @@ function stringConfigValue(connection, key, env) {
|
|
|
16
14
|
const value = connection?.[key];
|
|
17
15
|
return typeof value === 'string' && value.trim().length > 0 ? resolveStringReference(value.trim(), env) : undefined;
|
|
18
16
|
}
|
|
19
|
-
function resolveStringReference(value, env) {
|
|
20
|
-
if (value.startsWith('env:')) {
|
|
21
|
-
const envName = value.slice('env:'.length);
|
|
22
|
-
return env[envName] ?? '';
|
|
23
|
-
}
|
|
24
|
-
if (value.startsWith('file:')) {
|
|
25
|
-
const rawPath = value.slice('file:'.length);
|
|
26
|
-
const path = rawPath.startsWith('~') ? resolve(homedir(), rawPath.slice(1)) : rawPath;
|
|
27
|
-
return readFileSync(path, 'utf-8').trim();
|
|
28
|
-
}
|
|
29
|
-
return value;
|
|
30
|
-
}
|
|
31
17
|
function maybeNumber(value) {
|
|
32
18
|
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
|
33
19
|
}
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { homedir } from 'node:os';
|
|
3
|
-
import { resolve } from 'node:path';
|
|
1
|
+
import { resolveStringReference } from '../shared/string-reference.js';
|
|
4
2
|
import { getDialectForDriver } from '../../context/connections/dialects.js';
|
|
5
3
|
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
|
6
4
|
import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
|
|
@@ -88,17 +86,6 @@ function stringConfigValue(connection, key, env) {
|
|
|
88
86
|
const value = connection?.[key];
|
|
89
87
|
return typeof value === 'string' && value.trim().length > 0 ? resolveStringReference(value.trim(), env) : undefined;
|
|
90
88
|
}
|
|
91
|
-
function resolveStringReference(value, env) {
|
|
92
|
-
if (value.startsWith('env:')) {
|
|
93
|
-
return env[value.slice('env:'.length)] ?? '';
|
|
94
|
-
}
|
|
95
|
-
if (value.startsWith('file:')) {
|
|
96
|
-
const rawPath = value.slice('file:'.length);
|
|
97
|
-
const path = rawPath.startsWith('~') ? resolve(homedir(), rawPath.slice(1)) : rawPath;
|
|
98
|
-
return readFileSync(path, 'utf-8').trim();
|
|
99
|
-
}
|
|
100
|
-
return value;
|
|
101
|
-
}
|
|
102
89
|
function numberValue(value) {
|
|
103
90
|
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
|
104
91
|
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolves a config string that may reference an environment variable
|
|
3
|
+
* (`env:NAME`) or a file (`file:/path`, `~` expands to the home dir).
|
|
4
|
+
* Plain values pass through unchanged.
|
|
5
|
+
*/
|
|
6
|
+
export declare function resolveStringReference(value: string, env: NodeJS.ProcessEnv): string;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { homedir } from 'node:os';
|
|
3
|
+
import { resolve } from 'node:path';
|
|
4
|
+
/**
|
|
5
|
+
* Resolves a config string that may reference an environment variable
|
|
6
|
+
* (`env:NAME`) or a file (`file:/path`, `~` expands to the home dir).
|
|
7
|
+
* Plain values pass through unchanged.
|
|
8
|
+
*/
|
|
9
|
+
export function resolveStringReference(value, env) {
|
|
10
|
+
if (value.startsWith('env:')) {
|
|
11
|
+
return env[value.slice('env:'.length)] ?? '';
|
|
12
|
+
}
|
|
13
|
+
if (value.startsWith('file:')) {
|
|
14
|
+
const rawPath = value.slice('file:'.length);
|
|
15
|
+
const path = rawPath.startsWith('~') ? resolve(homedir(), rawPath.slice(rawPath[1] === '/' ? 2 : 1)) : rawPath;
|
|
16
|
+
return readFileSync(path, 'utf-8').trim();
|
|
17
|
+
}
|
|
18
|
+
return value;
|
|
19
|
+
}
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import { createPrivateKey } from 'node:crypto';
|
|
2
|
-
import { readFileSync } from 'node:fs';
|
|
3
|
-
import { homedir } from 'node:os';
|
|
4
|
-
import { resolve } from 'node:path';
|
|
5
2
|
import { getDialectForDriver } from '../../context/connections/dialects.js';
|
|
3
|
+
import { resolveStringReference } from '../shared/string-reference.js';
|
|
6
4
|
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
|
7
5
|
import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
|
|
8
6
|
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
|
@@ -11,17 +9,6 @@ import snowflake from 'snowflake-sdk';
|
|
|
11
9
|
import { assertSafeSnowflakeIdentifier, quoteSnowflakeIdentifier } from './identifiers.js';
|
|
12
10
|
import { configureSnowflakeSdkLogger } from './sdk-logger.js';
|
|
13
11
|
const DATE_TYPES = ['DATE', 'TIMESTAMP', 'TIMESTAMP_LTZ', 'TIMESTAMP_NTZ', 'TIMESTAMP_TZ', 'TIME'];
|
|
14
|
-
function resolveStringReference(value, env) {
|
|
15
|
-
if (value.startsWith('env:')) {
|
|
16
|
-
return env[value.slice('env:'.length)] ?? '';
|
|
17
|
-
}
|
|
18
|
-
if (value.startsWith('file:')) {
|
|
19
|
-
const rawPath = value.slice('file:'.length);
|
|
20
|
-
const path = rawPath.startsWith('~') ? resolve(homedir(), rawPath.slice(1)) : rawPath;
|
|
21
|
-
return readFileSync(path, 'utf-8').trim();
|
|
22
|
-
}
|
|
23
|
-
return value;
|
|
24
|
-
}
|
|
25
12
|
function stringConfigValue(connection, key, env) {
|
|
26
13
|
const value = connection?.[key];
|
|
27
14
|
return typeof value === 'string' && value.trim().length > 0 ? resolveStringReference(value.trim(), env) : undefined;
|
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
import { assertReadOnlySql, stripTrailingSqlNoise } from '../../context/connections/read-only-sql.js';
|
|
1
|
+
import { assertReadOnlySql, hoistLeadingCte, stripTrailingSqlNoise } from '../../context/connections/read-only-sql.js';
|
|
2
2
|
import { getDialectForDriver } from '../../context/connections/dialects.js';
|
|
3
3
|
import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
|
|
4
4
|
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
|
5
5
|
import { connectorTestFailure, createKtxConnectorCapabilities, } from '../../context/scan/types.js';
|
|
6
|
-
import { readFileSync } from 'node:fs';
|
|
7
|
-
import { homedir } from 'node:os';
|
|
8
|
-
import { resolve } from 'node:path';
|
|
9
6
|
import sql from 'mssql';
|
|
7
|
+
import { resolveStringReference } from '../shared/string-reference.js';
|
|
10
8
|
function sqlTypeDeclaration(type) {
|
|
11
9
|
if (typeof type === 'function') {
|
|
12
10
|
try {
|
|
@@ -79,17 +77,6 @@ function stringConfigValue(connection, key, env) {
|
|
|
79
77
|
const value = connection?.[key];
|
|
80
78
|
return typeof value === 'string' && value.trim().length > 0 ? resolveStringReference(value.trim(), env) : undefined;
|
|
81
79
|
}
|
|
82
|
-
function resolveStringReference(value, env) {
|
|
83
|
-
if (value.startsWith('env:')) {
|
|
84
|
-
return env[value.slice('env:'.length)] ?? '';
|
|
85
|
-
}
|
|
86
|
-
if (value.startsWith('file:')) {
|
|
87
|
-
const rawPath = value.slice('file:'.length);
|
|
88
|
-
const path = rawPath.startsWith('~') ? resolve(homedir(), rawPath.slice(1)) : rawPath;
|
|
89
|
-
return readFileSync(path, 'utf-8').trim();
|
|
90
|
-
}
|
|
91
|
-
return value;
|
|
92
|
-
}
|
|
93
80
|
function parseSqlServerUrl(url) {
|
|
94
81
|
const parsed = new URL(url);
|
|
95
82
|
return {
|
|
@@ -149,7 +136,8 @@ function limitSqlForSqlServerExecution(sqlText, maxRows) {
|
|
|
149
136
|
if (!Number.isInteger(maxRows) || maxRows <= 0) {
|
|
150
137
|
throw new Error('maxRows must be a positive integer.');
|
|
151
138
|
}
|
|
152
|
-
|
|
139
|
+
const { withPrefix, body } = hoistLeadingCte(trimmed);
|
|
140
|
+
return `${withPrefix}SELECT TOP ${maxRows} * FROM (${body}) AS ktx_query_result`;
|
|
153
141
|
}
|
|
154
142
|
export function isKtxSqlServerConnectionConfig(connection) {
|
|
155
143
|
return String(connection?.driver ?? '').toLowerCase() === 'sqlserver';
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { KtxProjectConnectionConfig } from '../project/config.js';
|
|
2
|
+
/** Stable id for the runtime-derived federated connection. Never written to ktx.yaml. */
|
|
3
|
+
export declare const FEDERATED_CONNECTION_ID = "_ktx_federated";
|
|
4
|
+
export declare function attachTypeForDriver(driver: string): string;
|
|
5
|
+
export interface FederatedMember {
|
|
6
|
+
connectionId: string;
|
|
7
|
+
driver: string;
|
|
8
|
+
projectDir: string;
|
|
9
|
+
connection: KtxProjectConnectionConfig;
|
|
10
|
+
}
|
|
11
|
+
export interface FederatedConnectionDescriptor {
|
|
12
|
+
id: typeof FEDERATED_CONNECTION_ID;
|
|
13
|
+
driver: 'duckdb';
|
|
14
|
+
members: FederatedMember[];
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Derives a virtual federated connection when a project declares 2+
|
|
18
|
+
* attach-compatible databases. Returns null otherwise — single-DB and
|
|
19
|
+
* incompatible projects are unaffected.
|
|
20
|
+
*/
|
|
21
|
+
export declare function deriveFederatedConnection(connections: Record<string, KtxProjectConnectionConfig>, projectDir: string): FederatedConnectionDescriptor | null;
|
|
22
|
+
export interface FederatedConnectionListing {
|
|
23
|
+
id: typeof FEDERATED_CONNECTION_ID;
|
|
24
|
+
driver: 'duckdb';
|
|
25
|
+
members: string[];
|
|
26
|
+
hint: string;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Listing-facing view of the virtual federated connection for `ktx connection`
|
|
30
|
+
* and MCP `connection_list`. Derived from the same declared state as
|
|
31
|
+
* deriveFederatedConnection, so both surfaces describe one connection.
|
|
32
|
+
*/
|
|
33
|
+
export declare function federatedConnectionListing(connections: Record<string, KtxProjectConnectionConfig>, projectDir: string): FederatedConnectionListing | null;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/** Stable id for the runtime-derived federated connection. Never written to ktx.yaml. */
|
|
2
|
+
export const FEDERATED_CONNECTION_ID = '_ktx_federated';
|
|
3
|
+
/**
|
|
4
|
+
* Drivers DuckDB can ATTACH for federation. The driver name doubles as the
|
|
5
|
+
* DuckDB extension/TYPE name, so this set is the single source of truth for
|
|
6
|
+
* both membership (a driver participates iff it appears here) and attach type.
|
|
7
|
+
*/
|
|
8
|
+
const ATTACH_COMPATIBLE_DRIVERS = new Set(['postgres', 'mysql', 'sqlite']);
|
|
9
|
+
export function attachTypeForDriver(driver) {
|
|
10
|
+
const normalized = driver.toLowerCase();
|
|
11
|
+
if (!ATTACH_COMPATIBLE_DRIVERS.has(normalized)) {
|
|
12
|
+
throw new Error(`Driver "${driver}" cannot be attached by DuckDB federation.`);
|
|
13
|
+
}
|
|
14
|
+
return normalized;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Derives a virtual federated connection when a project declares 2+
|
|
18
|
+
* attach-compatible databases. Returns null otherwise — single-DB and
|
|
19
|
+
* incompatible projects are unaffected.
|
|
20
|
+
*/
|
|
21
|
+
export function deriveFederatedConnection(connections, projectDir) {
|
|
22
|
+
const members = Object.entries(connections)
|
|
23
|
+
.filter(([, config]) => ATTACH_COMPATIBLE_DRIVERS.has(config.driver.toLowerCase()))
|
|
24
|
+
.map(([connectionId, config]) => ({
|
|
25
|
+
connectionId,
|
|
26
|
+
driver: config.driver.toLowerCase(),
|
|
27
|
+
projectDir,
|
|
28
|
+
connection: config,
|
|
29
|
+
}));
|
|
30
|
+
if (members.length < 2) {
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
return { id: FEDERATED_CONNECTION_ID, driver: 'duckdb', members };
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Listing-facing view of the virtual federated connection for `ktx connection`
|
|
37
|
+
* and MCP `connection_list`. Derived from the same declared state as
|
|
38
|
+
* deriveFederatedConnection, so both surfaces describe one connection.
|
|
39
|
+
*/
|
|
40
|
+
export function federatedConnectionListing(connections, projectDir) {
|
|
41
|
+
const descriptor = deriveFederatedConnection(connections, projectDir);
|
|
42
|
+
if (!descriptor) {
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
return {
|
|
46
|
+
id: FEDERATED_CONNECTION_ID,
|
|
47
|
+
driver: 'duckdb',
|
|
48
|
+
members: descriptor.members.map((member) => member.connectionId),
|
|
49
|
+
hint: 'Cross-database queries run here. Name tables connectionId.schema.table (or connectionId.table for sqlite); double-quote any id that is not a bare SQL identifier, e.g. "books-db".public.books.',
|
|
50
|
+
};
|
|
51
|
+
}
|
|
@@ -14,6 +14,8 @@ export interface LocalConnectionInfo {
|
|
|
14
14
|
id: string;
|
|
15
15
|
name: string;
|
|
16
16
|
connectionType: string;
|
|
17
|
+
members?: string[];
|
|
18
|
+
hint?: string;
|
|
17
19
|
}
|
|
18
20
|
export declare function localConnectionToWarehouseDescriptor(id: string, connection: KtxProjectConnectionConfig | undefined): LocalWarehouseDescriptor | null;
|
|
19
21
|
export declare function localConnectionTypeForConfig(id: string, connection: KtxProjectConnectionConfig | undefined): string;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { executeFederatedQuery } from '../../connectors/duckdb/federated-executor.js';
|
|
2
|
+
import type { KtxLocalProject } from '../project/project.js';
|
|
3
|
+
import type { KtxScanConnector } from '../scan/types.js';
|
|
4
|
+
import type { KtxSqlQueryExecutionInput, KtxSqlQueryExecutionResult } from './query-executor.js';
|
|
5
|
+
export interface ExecuteProjectReadOnlySqlDeps {
|
|
6
|
+
project: KtxLocalProject;
|
|
7
|
+
input: KtxSqlQueryExecutionInput;
|
|
8
|
+
createConnector: (connectionId: string) => Promise<KtxScanConnector> | KtxScanConnector;
|
|
9
|
+
executeFederated?: typeof executeFederatedQuery;
|
|
10
|
+
runId?: string;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Single resolve-and-execute path for project read-only SQL. The federated
|
|
14
|
+
* connection is derived from declared state here so every executor entry point
|
|
15
|
+
* routes `_ktx_federated` identically; standard connections go through the
|
|
16
|
+
* scan connector.
|
|
17
|
+
*/
|
|
18
|
+
export declare function executeProjectReadOnlySql(deps: ExecuteProjectReadOnlySqlDeps): Promise<KtxSqlQueryExecutionResult>;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { executeFederatedQuery } from '../../connectors/duckdb/federated-executor.js';
|
|
2
|
+
import { deriveFederatedConnection, FEDERATED_CONNECTION_ID } from './federation.js';
|
|
3
|
+
/**
|
|
4
|
+
* Single resolve-and-execute path for project read-only SQL. The federated
|
|
5
|
+
* connection is derived from declared state here so every executor entry point
|
|
6
|
+
* routes `_ktx_federated` identically; standard connections go through the
|
|
7
|
+
* scan connector.
|
|
8
|
+
*/
|
|
9
|
+
export async function executeProjectReadOnlySql(deps) {
|
|
10
|
+
const { project, input } = deps;
|
|
11
|
+
if (input.connectionId === FEDERATED_CONNECTION_ID) {
|
|
12
|
+
const descriptor = deriveFederatedConnection(project.config.connections, project.projectDir);
|
|
13
|
+
if (!descriptor) {
|
|
14
|
+
throw new Error('Federated execution requested but fewer than 2 attach-compatible connections exist.');
|
|
15
|
+
}
|
|
16
|
+
const runFederated = deps.executeFederated ?? executeFederatedQuery;
|
|
17
|
+
return runFederated(descriptor.members, input);
|
|
18
|
+
}
|
|
19
|
+
let connector = null;
|
|
20
|
+
try {
|
|
21
|
+
connector = await deps.createConnector(input.connectionId);
|
|
22
|
+
if (!connector.capabilities.readOnlySql || !connector.executeReadOnly) {
|
|
23
|
+
throw new Error(`Connection "${input.connectionId}" driver "${connector.driver}" does not support read-only SQL execution.`);
|
|
24
|
+
}
|
|
25
|
+
const ctx = { runId: deps.runId ?? 'sql-execution' };
|
|
26
|
+
const result = await connector.executeReadOnly({ connectionId: input.connectionId, sql: input.sql, maxRows: input.maxRows }, ctx);
|
|
27
|
+
return {
|
|
28
|
+
headers: result.headers,
|
|
29
|
+
...(result.headerTypes ? { headerTypes: result.headerTypes } : {}),
|
|
30
|
+
rows: result.rows,
|
|
31
|
+
totalRows: result.totalRows,
|
|
32
|
+
command: 'SELECT',
|
|
33
|
+
rowCount: result.rowCount,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
finally {
|
|
37
|
+
await connector?.cleanup?.();
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -6,8 +6,9 @@ export interface KtxSqlQueryExecutionInput {
|
|
|
6
6
|
sql: string;
|
|
7
7
|
maxRows?: number;
|
|
8
8
|
}
|
|
9
|
-
interface KtxSqlQueryExecutionResult {
|
|
9
|
+
export interface KtxSqlQueryExecutionResult {
|
|
10
10
|
headers: string[];
|
|
11
|
+
headerTypes?: string[];
|
|
11
12
|
rows: unknown[][];
|
|
12
13
|
totalRows: number;
|
|
13
14
|
command: string;
|
|
@@ -17,4 +18,3 @@ export interface KtxSqlQueryExecutorPort {
|
|
|
17
18
|
execute(input: KtxSqlQueryExecutionInput): Promise<KtxSqlQueryExecutionResult>;
|
|
18
19
|
}
|
|
19
20
|
export declare function normalizeQueryRows(rows: unknown[]): unknown[][];
|
|
20
|
-
export {};
|
|
@@ -1,3 +1,8 @@
|
|
|
1
1
|
export declare function assertReadOnlySql(sql: string): string;
|
|
2
|
+
/** @internal */
|
|
3
|
+
export declare function hoistLeadingCte(sql: string): {
|
|
4
|
+
withPrefix: string;
|
|
5
|
+
body: string;
|
|
6
|
+
};
|
|
2
7
|
export declare function stripTrailingSqlNoise(sql: string): string;
|
|
3
8
|
export declare function limitSqlForExecution(sql: string, maxRows: number | undefined): string;
|