@kaelio/ktx 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.8.0-py3-none-any.whl → kaelio_ktx-0.9.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli-runtime.js +50 -3
- package/dist/commands/setup-commands.js +1 -1
- package/dist/connection-recovery.d.ts +34 -0
- package/dist/connection-recovery.js +82 -0
- package/dist/connection.js +3 -1
- package/dist/context/ingest/adapters/historic-sql/bigquery-query-history-reader.js +71 -20
- package/dist/context/ingest/adapters/historic-sql/chunk-unified.js +2 -1
- package/dist/context/ingest/adapters/historic-sql/connection-dialect.d.ts +9 -0
- package/dist/context/ingest/adapters/historic-sql/connection-dialect.js +15 -4
- package/dist/context/ingest/adapters/historic-sql/pattern-inputs.js +8 -2
- package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.d.ts +29 -0
- package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.js +190 -0
- package/dist/context/ingest/adapters/historic-sql/scope-floor.d.ts +18 -0
- package/dist/context/ingest/adapters/historic-sql/scope-floor.js +229 -0
- package/dist/context/ingest/adapters/historic-sql/scope-membership.d.ts +8 -0
- package/dist/context/ingest/adapters/historic-sql/scope-membership.js +29 -0
- package/dist/context/ingest/adapters/historic-sql/snowflake-query-history-reader.js +68 -19
- package/dist/context/ingest/adapters/historic-sql/stage-unified.js +57 -50
- package/dist/context/ingest/adapters/historic-sql/types.d.ts +36 -3
- package/dist/context/ingest/adapters/historic-sql/types.js +14 -2
- package/dist/context/ingest/context-evidence/sqlite-context-evidence-store.d.ts +1 -1
- package/dist/context/ingest/isolated-diff/patch-integrator.js +75 -5
- package/dist/context/ingest/local-adapters.js +21 -4
- package/dist/context/ingest/local-bundle-runtime.js +3 -2
- package/dist/context/llm/codex-exec-events.d.ts +20 -0
- package/dist/context/llm/codex-exec-events.js +155 -0
- package/dist/context/llm/codex-isolation.d.ts +3 -0
- package/dist/context/llm/codex-isolation.js +5 -0
- package/dist/context/llm/codex-mcp-runtime-server.d.ts +24 -0
- package/dist/context/llm/codex-mcp-runtime-server.js +51 -0
- package/dist/context/llm/codex-models.d.ts +2 -0
- package/dist/context/llm/codex-models.js +17 -0
- package/dist/context/llm/codex-runtime-config.d.ts +16 -0
- package/dist/context/llm/codex-runtime-config.js +19 -0
- package/dist/context/llm/codex-runtime.d.ts +37 -0
- package/dist/context/llm/codex-runtime.js +304 -0
- package/dist/context/llm/codex-sdk-runner.d.ts +21 -0
- package/dist/context/llm/codex-sdk-runner.js +63 -0
- package/dist/context/llm/local-config.d.ts +2 -0
- package/dist/context/llm/local-config.js +12 -1
- package/dist/context/project/config.d.ts +2 -0
- package/dist/context/project/config.js +2 -2
- package/dist/context/sql-analysis/http-sql-analysis-port.js +32 -2
- package/dist/context/sql-analysis/ports.d.ts +12 -2
- package/dist/context/tools/context-candidate-mark.tool.d.ts +2 -2
- package/dist/context-build-view.js +4 -32
- package/dist/io/buffered-command-io.d.ts +11 -0
- package/dist/io/buffered-command-io.js +28 -0
- package/dist/llm/types.d.ts +1 -1
- package/dist/local-adapters.d.ts +10 -2
- package/dist/local-adapters.js +19 -3
- package/dist/next-steps.js +1 -2
- package/dist/progress-port-adapter.d.ts +6 -0
- package/dist/progress-port-adapter.js +18 -0
- package/dist/public-ingest.d.ts +20 -1
- package/dist/public-ingest.js +178 -27
- package/dist/scan.js +3 -1
- package/dist/setup-context.d.ts +2 -0
- package/dist/setup-context.js +133 -27
- package/dist/setup-databases.d.ts +17 -1
- package/dist/setup-databases.js +358 -249
- package/dist/setup-models.d.ts +10 -1
- package/dist/setup-models.js +90 -2
- package/dist/setup-ready-menu.d.ts +16 -2
- package/dist/setup-ready-menu.js +37 -5
- package/dist/setup-sources.js +108 -28
- package/dist/setup.js +22 -10
- package/dist/status-project.d.ts +11 -0
- package/dist/status-project.js +50 -1
- package/dist/telemetry/command-hook.d.ts +1 -0
- package/dist/telemetry/command-hook.js +3 -1
- package/dist/telemetry/events.d.ts +11 -6
- package/dist/telemetry/events.js +10 -2
- package/dist/telemetry/identity.d.ts +0 -1
- package/dist/telemetry/identity.js +6 -6
- package/dist/telemetry/index.d.ts +12 -0
- package/dist/telemetry/index.js +13 -2
- package/dist/telemetry/scrubber.d.ts +10 -0
- package/dist/telemetry/scrubber.js +20 -0
- package/package.json +5 -4
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Captures stdout/stderr from a command (e.g. `runKtxConnection`) into buffers
|
|
3
|
+
* instead of the terminal. Callers decide whether to flush the captured text to
|
|
4
|
+
* the user or discard it.
|
|
5
|
+
*/
|
|
6
|
+
export function createBufferedCommandIo() {
|
|
7
|
+
let stdout = '';
|
|
8
|
+
let stderr = '';
|
|
9
|
+
return {
|
|
10
|
+
stdout: {
|
|
11
|
+
isTTY: false,
|
|
12
|
+
write(chunk) {
|
|
13
|
+
stdout += chunk;
|
|
14
|
+
},
|
|
15
|
+
},
|
|
16
|
+
stderr: {
|
|
17
|
+
write(chunk) {
|
|
18
|
+
stderr += chunk;
|
|
19
|
+
},
|
|
20
|
+
},
|
|
21
|
+
stdoutText() {
|
|
22
|
+
return stdout;
|
|
23
|
+
},
|
|
24
|
+
stderrText() {
|
|
25
|
+
return stderr;
|
|
26
|
+
},
|
|
27
|
+
};
|
|
28
|
+
}
|
package/dist/llm/types.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { LanguageModel, TelemetrySettings, ToolCallRepairFunction, ToolSet } from 'ai';
|
|
2
2
|
export declare const KTX_MODEL_ROLES: readonly ["default", "triage", "candidateExtraction", "curator", "reconcile", "repair"];
|
|
3
3
|
export type KtxModelRole = (typeof KTX_MODEL_ROLES)[number];
|
|
4
|
-
type KtxLlmBackend = 'anthropic' | 'vertex' | 'gateway' | 'claude-code';
|
|
4
|
+
type KtxLlmBackend = 'anthropic' | 'vertex' | 'gateway' | 'claude-code' | 'codex';
|
|
5
5
|
export type KtxPromptCacheTtl = '5m' | '1h';
|
|
6
6
|
type KtxJsonValue = null | string | number | boolean | KtxJsonValue[] | {
|
|
7
7
|
[key: string]: KtxJsonValue | undefined;
|
package/dist/local-adapters.d.ts
CHANGED
|
@@ -1,14 +1,22 @@
|
|
|
1
1
|
import { type DefaultLocalIngestAdaptersOptions } from './context/ingest/local-adapters.js';
|
|
2
|
+
import type { HistoricSqlDialect, HistoricSqlReader } from './context/ingest/adapters/historic-sql/types.js';
|
|
2
3
|
import type { SourceAdapter } from './context/ingest/types.js';
|
|
3
4
|
import type { KtxLocalProject } from './context/project/project.js';
|
|
4
5
|
import type { SqlAnalysisPort } from './context/sql-analysis/ports.js';
|
|
5
|
-
import { type
|
|
6
|
+
import { type ManagedPythonDaemonHttpOptions } from './managed-python-http.js';
|
|
6
7
|
import type { KtxOperationalLogger } from './io/logger.js';
|
|
7
8
|
export interface KtxCliLocalIngestAdaptersOptions extends DefaultLocalIngestAdaptersOptions {
|
|
8
9
|
historicSqlConnectionId?: string;
|
|
9
10
|
sqlAnalysis?: SqlAnalysisPort;
|
|
10
11
|
sqlAnalysisUrl?: string;
|
|
11
|
-
managedDaemon?:
|
|
12
|
+
managedDaemon?: ManagedPythonDaemonHttpOptions;
|
|
12
13
|
logger?: KtxOperationalLogger;
|
|
13
14
|
}
|
|
15
|
+
export interface KtxCliHistoricSqlRuntime {
|
|
16
|
+
dialect: HistoricSqlDialect;
|
|
17
|
+
sqlAnalysis: SqlAnalysisPort;
|
|
18
|
+
reader: HistoricSqlReader;
|
|
19
|
+
queryClient: unknown;
|
|
20
|
+
}
|
|
21
|
+
export declare function createKtxCliHistoricSqlRuntime(project: KtxLocalProject, connectionId: string, options?: KtxCliLocalIngestAdaptersOptions): KtxCliHistoricSqlRuntime | undefined;
|
|
14
22
|
export declare function createKtxCliLocalIngestAdapters(project: KtxLocalProject, options?: KtxCliLocalIngestAdaptersOptions): SourceAdapter[];
|
package/dist/local-adapters.js
CHANGED
|
@@ -12,7 +12,7 @@ import { isKtxSqliteConnectionConfig } from './connectors/sqlite/connector.js';
|
|
|
12
12
|
import { createSqlServerLiveDatabaseIntrospection } from './connectors/sqlserver/live-database-introspection.js';
|
|
13
13
|
import { isKtxSqlServerConnectionConfig } from './connectors/sqlserver/connector.js';
|
|
14
14
|
import { BigQueryHistoricSqlQueryHistoryReader } from './context/ingest/adapters/historic-sql/bigquery-query-history-reader.js';
|
|
15
|
-
import {
|
|
15
|
+
import { historicSqlDialectForConnectionDriver } from './context/ingest/adapters/historic-sql/connection-dialect.js';
|
|
16
16
|
import { createDaemonLiveDatabaseIntrospection } from './context/ingest/adapters/live-database/daemon-introspection.js';
|
|
17
17
|
import { createDefaultLocalIngestAdapters } from './context/ingest/local-adapters.js';
|
|
18
18
|
import { LiveDatabaseSourceAdapter } from './context/ingest/adapters/live-database/live-database.adapter.js';
|
|
@@ -224,7 +224,12 @@ function historicSqlOptionsForLocalRun(project, options) {
|
|
|
224
224
|
return undefined;
|
|
225
225
|
}
|
|
226
226
|
const connection = project.config.connections[connectionId];
|
|
227
|
-
|
|
227
|
+
// historicSqlConnectionId is only set when query history was explicitly
|
|
228
|
+
// requested for this run (e.g. `--query-history`), so resolve the dialect from
|
|
229
|
+
// driver capability rather than the persisted context.queryHistory.enabled
|
|
230
|
+
// flag — otherwise the adapter is missing and findAdapter('historic-sql')
|
|
231
|
+
// throws even though the run asked for it.
|
|
232
|
+
const dialect = historicSqlDialectForConnectionDriver(connection);
|
|
228
233
|
if (!dialect) {
|
|
229
234
|
return undefined;
|
|
230
235
|
}
|
|
@@ -234,6 +239,7 @@ function historicSqlOptionsForLocalRun(project, options) {
|
|
|
234
239
|
if (dialect === 'postgres') {
|
|
235
240
|
return {
|
|
236
241
|
...base,
|
|
242
|
+
dialect,
|
|
237
243
|
reader: new PostgresPgssReader(),
|
|
238
244
|
queryClient: createEphemeralPostgresHistoricSqlClient(project, connectionId),
|
|
239
245
|
};
|
|
@@ -245,6 +251,7 @@ function historicSqlOptionsForLocalRun(project, options) {
|
|
|
245
251
|
}
|
|
246
252
|
return {
|
|
247
253
|
...base,
|
|
254
|
+
dialect,
|
|
248
255
|
reader: new BigQueryHistoricSqlQueryHistoryReader({
|
|
249
256
|
projectId: bigQueryProjectId(connection, process.env),
|
|
250
257
|
region: bigQueryRegion(connection),
|
|
@@ -254,6 +261,7 @@ function historicSqlOptionsForLocalRun(project, options) {
|
|
|
254
261
|
}
|
|
255
262
|
return {
|
|
256
263
|
...base,
|
|
264
|
+
dialect,
|
|
257
265
|
reader: new SnowflakeHistoricSqlQueryHistoryReader(),
|
|
258
266
|
queryClient: {
|
|
259
267
|
async executeQuery(query) {
|
|
@@ -264,8 +272,16 @@ function historicSqlOptionsForLocalRun(project, options) {
|
|
|
264
272
|
},
|
|
265
273
|
};
|
|
266
274
|
}
|
|
275
|
+
export function createKtxCliHistoricSqlRuntime(project, connectionId, options = {}) {
|
|
276
|
+
return historicSqlOptionsForLocalRun(project, {
|
|
277
|
+
...options,
|
|
278
|
+
historicSqlConnectionId: connectionId,
|
|
279
|
+
});
|
|
280
|
+
}
|
|
267
281
|
export function createKtxCliLocalIngestAdapters(project, options = {}) {
|
|
268
|
-
const historicSql =
|
|
282
|
+
const historicSql = options.historicSqlConnectionId
|
|
283
|
+
? createKtxCliHistoricSqlRuntime(project, options.historicSqlConnectionId, options)
|
|
284
|
+
: undefined;
|
|
269
285
|
const base = createDefaultLocalIngestAdapters(project, {
|
|
270
286
|
...options,
|
|
271
287
|
databaseIntrospection: ktxCliDaemonDatabaseIntrospectionOptions(options),
|
package/dist/next-steps.js
CHANGED
|
@@ -53,8 +53,7 @@ export function formatSetupNextStepLines(state, indent = ' ') {
|
|
|
53
53
|
}
|
|
54
54
|
if (!state.contextReady) {
|
|
55
55
|
return [
|
|
56
|
-
`${indent}
|
|
57
|
-
`${indent}Run ingest to build database schema context before context-source ingest.`,
|
|
56
|
+
`${indent}Setup is complete. The only step left is to build context for your agents.`,
|
|
58
57
|
...commandLines(KTX_CONTEXT_BUILD_COMMANDS, indent),
|
|
59
58
|
];
|
|
60
59
|
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { KtxProgressPort } from './context/scan/types.js';
|
|
2
|
+
import type { KtxIngestProgressUpdate } from './ingest.js';
|
|
3
|
+
export interface AggregateProgressState {
|
|
4
|
+
progress: number;
|
|
5
|
+
}
|
|
6
|
+
export declare function createAggregateProgressPort(onProgress: (update: KtxIngestProgressUpdate) => void, state?: AggregateProgressState, start?: number, weight?: number): KtxProgressPort;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export function createAggregateProgressPort(onProgress, state = { progress: 0 }, start = 0, weight = 1) {
|
|
2
|
+
return {
|
|
3
|
+
async update(value, message, options) {
|
|
4
|
+
const absoluteValue = start + Math.max(0, Math.min(1, value)) * weight;
|
|
5
|
+
state.progress = Math.max(state.progress, Math.min(1, absoluteValue));
|
|
6
|
+
if (!message)
|
|
7
|
+
return;
|
|
8
|
+
onProgress({
|
|
9
|
+
percent: Math.max(0, Math.min(100, Math.round(state.progress * 100))),
|
|
10
|
+
message,
|
|
11
|
+
...(options?.transient !== undefined ? { transient: options.transient } : {}),
|
|
12
|
+
});
|
|
13
|
+
},
|
|
14
|
+
startPhase(phaseWeight) {
|
|
15
|
+
return createAggregateProgressPort(onProgress, state, state.progress, weight * phaseWeight);
|
|
16
|
+
},
|
|
17
|
+
};
|
|
18
|
+
}
|
package/dist/public-ingest.d.ts
CHANGED
|
@@ -5,6 +5,7 @@ import type { KtxIngestArgs, KtxIngestDeps, KtxIngestProgressUpdate } from './in
|
|
|
5
5
|
import { type KtxManagedPythonInstallPolicy, type ManagedPythonCommandRuntime } from './managed-python-command.js';
|
|
6
6
|
import type { KtxRuntimeFeature } from './managed-python-runtime.js';
|
|
7
7
|
import type { KtxScanArgs, KtxScanDeps } from './scan.js';
|
|
8
|
+
import type { KtxTableRef } from './context/scan/types.js';
|
|
8
9
|
type KtxPublicIngestStepName = 'database-schema' | 'query-history' | 'source-ingest' | 'memory-update';
|
|
9
10
|
type KtxPublicIngestStepStatus = 'done' | 'skipped' | 'failed' | 'not-run';
|
|
10
11
|
type KtxPublicIngestInputMode = 'auto' | 'disabled';
|
|
@@ -98,6 +99,17 @@ interface KtxPublicContextBuildArgs {
|
|
|
98
99
|
cliVersion?: string;
|
|
99
100
|
runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
|
|
100
101
|
}
|
|
102
|
+
export declare function publicProgressMessage(message: string, target: KtxPublicIngestPlanTarget): string;
|
|
103
|
+
/** @internal */
|
|
104
|
+
export declare function queryHistoryPullConfig(input: {
|
|
105
|
+
stored: Record<string, unknown>;
|
|
106
|
+
dialect: HistoricSqlDialect;
|
|
107
|
+
windowDays?: number;
|
|
108
|
+
enabledTables?: KtxTableRef[];
|
|
109
|
+
enabledSchemas?: string[];
|
|
110
|
+
modeledTableCatalog?: KtxTableRef[];
|
|
111
|
+
scopeFloorWarnings?: string[];
|
|
112
|
+
}): Record<string, unknown>;
|
|
101
113
|
export declare function buildPublicIngestPlan(project: KtxPublicIngestProject, args: {
|
|
102
114
|
projectDir: string;
|
|
103
115
|
targetConnectionId?: string;
|
|
@@ -108,8 +120,15 @@ export declare function buildPublicIngestPlan(project: KtxPublicIngestProject, a
|
|
|
108
120
|
command: 'run';
|
|
109
121
|
}>['mode'];
|
|
110
122
|
}): KtxPublicIngestPlan;
|
|
123
|
+
/**
|
|
124
|
+
* Run one ingest target through its scan/ingest steps. The single per-target
|
|
125
|
+
* chokepoint reached by every entrypoint — standalone `ktx ingest` (plain/json
|
|
126
|
+
* and foreground) and `ktx setup` (via `runContextBuild`). The exported
|
|
127
|
+
* `executePublicIngestTarget` wraps this and emits the `ingest_completed`
|
|
128
|
+
* telemetry event exactly once, so every path is counted.
|
|
129
|
+
*/
|
|
111
130
|
export declare function executePublicIngestTarget(target: KtxPublicIngestPlanTarget, args: Extract<KtxPublicIngestArgs, {
|
|
112
131
|
command: 'run';
|
|
113
|
-
}>, io: KtxCliIo, deps: KtxPublicIngestDeps): Promise<KtxPublicIngestTargetResult>;
|
|
132
|
+
}>, io: KtxCliIo, deps: KtxPublicIngestDeps, project: KtxPublicIngestProject): Promise<KtxPublicIngestTargetResult>;
|
|
114
133
|
export declare function runKtxPublicIngest(args: KtxPublicIngestArgs, io: KtxCliIo, deps?: KtxPublicIngestDeps): Promise<number>;
|
|
115
134
|
export {};
|
package/dist/public-ingest.js
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
import { getKtxCliPackageInfo } from './cli-runtime.js';
|
|
2
2
|
import { loadKtxProject } from './context/project/project.js';
|
|
3
3
|
import { isDatabaseDriver, normalizeConnectionDriver } from './connection-drivers.js';
|
|
4
|
+
import { resolveQueryHistoryScopeFloor } from './context/ingest/adapters/historic-sql/scope-floor.js';
|
|
4
5
|
import { ensureManagedPythonCommandRuntime, } from './managed-python-command.js';
|
|
5
|
-
import { publicIngestOutputLine } from './public-ingest-copy.js';
|
|
6
|
+
import { publicDatabaseIngestMessage, publicIngestOutputLine, publicQueryHistoryMessage, } from './public-ingest-copy.js';
|
|
7
|
+
import { createAggregateProgressPort } from './progress-port-adapter.js';
|
|
6
8
|
import { resolvePublicIngestRuntimeRequirements } from './runtime-requirements.js';
|
|
7
9
|
import { profileMark } from './startup-profile.js';
|
|
8
10
|
import { isDemoConnection } from './telemetry/demo-detect.js';
|
|
9
11
|
import { emitProjectStackSnapshot, emitTelemetryEvent } from './telemetry/index.js';
|
|
12
|
+
import { formatErrorDetail } from './telemetry/scrubber.js';
|
|
10
13
|
profileMark('module:public-ingest');
|
|
11
14
|
const sourceAdapterByDriver = new Map([
|
|
12
15
|
['metabase', 'metabase'],
|
|
@@ -17,6 +20,16 @@ const sourceAdapterByDriver = new Map([
|
|
|
17
20
|
['dbt', 'dbt'],
|
|
18
21
|
['lookml', 'lookml'],
|
|
19
22
|
]);
|
|
23
|
+
export function publicProgressMessage(message, target) {
|
|
24
|
+
let current = message;
|
|
25
|
+
if (target.operation === 'database-ingest') {
|
|
26
|
+
current = publicDatabaseIngestMessage(current);
|
|
27
|
+
}
|
|
28
|
+
if (target.steps.includes('query-history')) {
|
|
29
|
+
current = publicQueryHistoryMessage(current, target.connectionId);
|
|
30
|
+
}
|
|
31
|
+
return current;
|
|
32
|
+
}
|
|
20
33
|
const queryHistoryDialectByDriver = new Map([
|
|
21
34
|
['postgres', 'postgres'],
|
|
22
35
|
['bigquery', 'bigquery'],
|
|
@@ -99,20 +112,20 @@ function storedQueryHistory(connection) {
|
|
|
99
112
|
function positiveInteger(value) {
|
|
100
113
|
return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : undefined;
|
|
101
114
|
}
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
return undefined;
|
|
106
|
-
}
|
|
107
|
-
const tables = raw.filter((value) => typeof value === 'string' && value.trim().length > 0);
|
|
108
|
-
return tables.length > 0 ? tables : undefined;
|
|
109
|
-
}
|
|
110
|
-
function queryHistoryPullConfig(input) {
|
|
111
|
-
const { enabled: _enabled, dialect: _dialect, ...storedConfig } = input.stored;
|
|
115
|
+
/** @internal */
|
|
116
|
+
export function queryHistoryPullConfig(input) {
|
|
117
|
+
const { enabled: _enabled, dialect: _dialect, enabledTables: _enabledTables, enabledSchemas: _enabledSchemas, scopeFloorWarnings: _scopeFloorWarnings, ...storedConfig } = input.stored;
|
|
112
118
|
return {
|
|
113
119
|
...storedConfig,
|
|
114
120
|
dialect: input.dialect,
|
|
115
|
-
...(input.enabledTables ? { enabledTables: input.enabledTables } : {}),
|
|
121
|
+
...(input.enabledTables && input.enabledTables.length > 0 ? { enabledTables: input.enabledTables } : {}),
|
|
122
|
+
...(input.enabledSchemas && input.enabledSchemas.length > 0 ? { enabledSchemas: input.enabledSchemas } : {}),
|
|
123
|
+
...(input.modeledTableCatalog && input.modeledTableCatalog.length > 0
|
|
124
|
+
? { modeledTableCatalog: input.modeledTableCatalog }
|
|
125
|
+
: {}),
|
|
126
|
+
...(input.scopeFloorWarnings && input.scopeFloorWarnings.length > 0
|
|
127
|
+
? { scopeFloorWarnings: input.scopeFloorWarnings }
|
|
128
|
+
: {}),
|
|
116
129
|
...(input.windowDays !== undefined ? { windowDays: input.windowDays } : {}),
|
|
117
130
|
};
|
|
118
131
|
}
|
|
@@ -157,7 +170,6 @@ function resolveDatabaseTargetOptions(input) {
|
|
|
157
170
|
stored: storedQh,
|
|
158
171
|
dialect,
|
|
159
172
|
windowDays: queryHistory.windowDays,
|
|
160
|
-
enabledTables: enabledTablesForConnection(input.connection),
|
|
161
173
|
}),
|
|
162
174
|
},
|
|
163
175
|
steps: ['database-schema', 'query-history'],
|
|
@@ -168,6 +180,37 @@ function resolveDatabaseTargetOptions(input) {
|
|
|
168
180
|
steps: ['database-schema'],
|
|
169
181
|
};
|
|
170
182
|
}
|
|
183
|
+
async function resolvedQueryHistoryPullConfigForTarget(target, project) {
|
|
184
|
+
if (target.operation !== 'database-ingest' || target.queryHistory?.enabled !== true || !target.queryHistory.dialect) {
|
|
185
|
+
return null;
|
|
186
|
+
}
|
|
187
|
+
const connection = project.config.connections[target.connectionId];
|
|
188
|
+
if (!connection) {
|
|
189
|
+
return (target.queryHistory.pullConfig ??
|
|
190
|
+
queryHistoryPullConfig({
|
|
191
|
+
stored: {},
|
|
192
|
+
dialect: target.queryHistory.dialect,
|
|
193
|
+
windowDays: target.queryHistory.windowDays,
|
|
194
|
+
}));
|
|
195
|
+
}
|
|
196
|
+
const stored = storedQueryHistory(connection);
|
|
197
|
+
const scopeFloor = await resolveQueryHistoryScopeFloor({
|
|
198
|
+
projectDir: project.projectDir,
|
|
199
|
+
connectionId: target.connectionId,
|
|
200
|
+
driver: target.driver,
|
|
201
|
+
connection: connection,
|
|
202
|
+
storedQueryHistory: stored,
|
|
203
|
+
});
|
|
204
|
+
return queryHistoryPullConfig({
|
|
205
|
+
stored,
|
|
206
|
+
dialect: target.queryHistory.dialect,
|
|
207
|
+
windowDays: target.queryHistory.windowDays,
|
|
208
|
+
enabledTables: scopeFloor.enabledTables,
|
|
209
|
+
enabledSchemas: scopeFloor.enabledSchemas,
|
|
210
|
+
modeledTableCatalog: scopeFloor.modeledTableCatalog,
|
|
211
|
+
scopeFloorWarnings: scopeFloor.warnings,
|
|
212
|
+
});
|
|
213
|
+
}
|
|
171
214
|
function enrichmentReadinessGaps(config) {
|
|
172
215
|
const gaps = [];
|
|
173
216
|
if (config.llm.provider.backend === 'none' || !config.llm.models.default) {
|
|
@@ -348,6 +391,9 @@ function rowsBucket() {
|
|
|
348
391
|
}
|
|
349
392
|
async function emitIngestCompleted(input) {
|
|
350
393
|
const failed = resultFailed(input.result);
|
|
394
|
+
const failureDetail = failed
|
|
395
|
+
? formatErrorDetail(input.result.steps.find((step) => step.status === 'failed')?.detail)
|
|
396
|
+
: undefined;
|
|
351
397
|
await emitTelemetryEvent({
|
|
352
398
|
name: 'ingest_completed',
|
|
353
399
|
projectDir: input.args.projectDir,
|
|
@@ -361,6 +407,7 @@ async function emitIngestCompleted(input) {
|
|
|
361
407
|
rowsBucket: rowsBucket(),
|
|
362
408
|
durationMs: Math.max(0, performance.now() - input.startedAt),
|
|
363
409
|
outcome: failed ? 'error' : 'ok',
|
|
410
|
+
...(failureDetail ? { errorDetail: failureDetail } : {}),
|
|
364
411
|
},
|
|
365
412
|
});
|
|
366
413
|
}
|
|
@@ -427,6 +474,65 @@ function createCapturedPublicIngestIo() {
|
|
|
427
474
|
},
|
|
428
475
|
};
|
|
429
476
|
}
|
|
477
|
+
function isCapturedPublicIngestIo(io) {
|
|
478
|
+
return typeof io.capturedOutput === 'function';
|
|
479
|
+
}
|
|
480
|
+
const PLAIN_PUBLIC_INGEST_PHASE_LABELS = {
|
|
481
|
+
'database-schema': 'database schema',
|
|
482
|
+
'query-history': 'query history',
|
|
483
|
+
'source-ingest': 'source ingest',
|
|
484
|
+
};
|
|
485
|
+
function firstSummaryLine(summary) {
|
|
486
|
+
if (!summary)
|
|
487
|
+
return undefined;
|
|
488
|
+
return summary.split(/\r?\n/).find((line) => line.trim().length > 0)?.trim();
|
|
489
|
+
}
|
|
490
|
+
function plainPhaseHeader(options, phaseKey) {
|
|
491
|
+
const prefix = options.total > 1 ? `[${options.index + 1}/${options.total}] ` : '';
|
|
492
|
+
return `${prefix}${options.target.connectionId} · ${PLAIN_PUBLIC_INGEST_PHASE_LABELS[phaseKey]}`;
|
|
493
|
+
}
|
|
494
|
+
function plainPhaseEndLine(status, summary) {
|
|
495
|
+
const firstLine = firstSummaryLine(summary);
|
|
496
|
+
return firstLine ? ` ${status} · ${firstLine}` : ` ${status}`;
|
|
497
|
+
}
|
|
498
|
+
function createPlainPublicIngestProgress(io, options) {
|
|
499
|
+
let currentPhase = null;
|
|
500
|
+
const startedPhases = new Set();
|
|
501
|
+
const lastPercentByPhase = new Map();
|
|
502
|
+
const startPhase = (phaseKey) => {
|
|
503
|
+
currentPhase = phaseKey;
|
|
504
|
+
startedPhases.add(phaseKey);
|
|
505
|
+
lastPercentByPhase.set(phaseKey, -1);
|
|
506
|
+
io.stderr.write(`${plainPhaseHeader(options, phaseKey)}\n`);
|
|
507
|
+
};
|
|
508
|
+
const ensurePhaseStarted = (phaseKey) => {
|
|
509
|
+
if (!startedPhases.has(phaseKey)) {
|
|
510
|
+
startPhase(phaseKey);
|
|
511
|
+
return;
|
|
512
|
+
}
|
|
513
|
+
currentPhase = phaseKey;
|
|
514
|
+
};
|
|
515
|
+
const emitProgress = (update) => {
|
|
516
|
+
if (currentPhase === null)
|
|
517
|
+
return;
|
|
518
|
+
const rounded = Math.max(0, Math.min(100, Math.round(update.percent)));
|
|
519
|
+
const lastPercent = lastPercentByPhase.get(currentPhase) ?? -1;
|
|
520
|
+
if (rounded <= lastPercent)
|
|
521
|
+
return;
|
|
522
|
+
lastPercentByPhase.set(currentPhase, rounded);
|
|
523
|
+
io.stderr.write(` [${rounded}%] ${publicProgressMessage(update.message, options.target)}\n`);
|
|
524
|
+
};
|
|
525
|
+
return {
|
|
526
|
+
onPhaseStart: startPhase,
|
|
527
|
+
onPhaseEnd(phaseKey, status, summary) {
|
|
528
|
+
ensurePhaseStarted(phaseKey);
|
|
529
|
+
io.stderr.write(`${plainPhaseEndLine(status, summary)}\n`);
|
|
530
|
+
currentPhase = null;
|
|
531
|
+
},
|
|
532
|
+
scanProgress: createAggregateProgressPort(emitProgress),
|
|
533
|
+
ingestProgress: emitProgress,
|
|
534
|
+
};
|
|
535
|
+
}
|
|
430
536
|
const INTERNAL_STATUS_LINE_RE = /^(Report|Run|Job|Status|Adapter|Connection|Sync|Diff|Tasks|Work units|Failed tasks|Saved memory|Provenance rows):\s*/;
|
|
431
537
|
const ACTIONABLE_FAILURE_LINE_RE = /^(Missing bundled Python runtime manifest|KTX Python runtime is required|KTX daemon HTTP|Error:|Failed\b|Could not\b|Cannot\b)/;
|
|
432
538
|
const RUNTIME_BACKED_RETRY_LINE_RE = /^Then retry the runtime-backed KTX command\.?$/;
|
|
@@ -457,7 +563,23 @@ function capturedFailureMessage(output) {
|
|
|
457
563
|
.filter((line) => line.startsWith('In a source checkout, build the local runtime assets with:'));
|
|
458
564
|
return [firstLine, ...followupLines].join('\n');
|
|
459
565
|
}
|
|
460
|
-
|
|
566
|
+
/**
|
|
567
|
+
* Run one ingest target through its scan/ingest steps. The single per-target
|
|
568
|
+
* chokepoint reached by every entrypoint — standalone `ktx ingest` (plain/json
|
|
569
|
+
* and foreground) and `ktx setup` (via `runContextBuild`). The exported
|
|
570
|
+
* `executePublicIngestTarget` wraps this and emits the `ingest_completed`
|
|
571
|
+
* telemetry event exactly once, so every path is counted.
|
|
572
|
+
*/
|
|
573
|
+
export async function executePublicIngestTarget(target, args, io, deps, project) {
|
|
574
|
+
const startedAt = performance.now();
|
|
575
|
+
const result = await runIngestTargetSteps(target, args, io, deps, project);
|
|
576
|
+
// `io` may be a capture buffer for the scan/ingest step output; the telemetry
|
|
577
|
+
// debug echo belongs on the real user-facing stream, which callers expose as
|
|
578
|
+
// `deps.runtimeIo` (falling back to `io` when the step io is already real).
|
|
579
|
+
await emitIngestCompleted({ args, project, target, result, startedAt, io: deps.runtimeIo ?? io });
|
|
580
|
+
return result;
|
|
581
|
+
}
|
|
582
|
+
async function runIngestTargetSteps(target, args, io, deps, project) {
|
|
461
583
|
if (target.preflightFailure) {
|
|
462
584
|
if (target.operation === 'database-ingest') {
|
|
463
585
|
deps.onPhaseEnd?.('database-schema', 'failed', target.preflightFailure);
|
|
@@ -475,7 +597,7 @@ export async function executePublicIngestTarget(target, args, io, deps) {
|
|
|
475
597
|
? {
|
|
476
598
|
...step,
|
|
477
599
|
status: 'failed',
|
|
478
|
-
detail: target.preflightFailure
|
|
600
|
+
detail: `${target.connectionId} failed: ${target.preflightFailure}`,
|
|
479
601
|
}
|
|
480
602
|
: step),
|
|
481
603
|
};
|
|
@@ -493,7 +615,11 @@ export async function executePublicIngestTarget(target, args, io, deps) {
|
|
|
493
615
|
...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
|
|
494
616
|
};
|
|
495
617
|
const runScan = deps.runScan ?? runKtxScan;
|
|
496
|
-
const capturedScanIo = deps.scanProgress
|
|
618
|
+
const capturedScanIo = deps.scanProgress
|
|
619
|
+
? isCapturedPublicIngestIo(io)
|
|
620
|
+
? io
|
|
621
|
+
: null
|
|
622
|
+
: createCapturedPublicIngestIo();
|
|
497
623
|
const scanIo = capturedScanIo ?? io;
|
|
498
624
|
const scanDeps = {
|
|
499
625
|
...(deps.scanProgress ? { progress: deps.scanProgress } : {}),
|
|
@@ -512,6 +638,10 @@ export async function executePublicIngestTarget(target, args, io, deps) {
|
|
|
512
638
|
if (target.queryHistory?.enabled === true) {
|
|
513
639
|
const { runKtxIngest } = await import('./ingest.js');
|
|
514
640
|
const runIngest = deps.runIngest ?? runKtxIngest;
|
|
641
|
+
const historicSqlPullConfigOverride = (await resolvedQueryHistoryPullConfigForTarget(target, project)) ?? {
|
|
642
|
+
dialect: target.queryHistory.dialect,
|
|
643
|
+
...(target.queryHistory.windowDays !== undefined ? { windowDays: target.queryHistory.windowDays } : {}),
|
|
644
|
+
};
|
|
515
645
|
const ingestArgs = {
|
|
516
646
|
command: 'run',
|
|
517
647
|
projectDir: args.projectDir,
|
|
@@ -522,12 +652,14 @@ export async function executePublicIngestTarget(target, args, io, deps) {
|
|
|
522
652
|
...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
|
|
523
653
|
...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
|
|
524
654
|
allowImplicitAdapter: true,
|
|
525
|
-
historicSqlPullConfigOverride
|
|
526
|
-
dialect: target.queryHistory.dialect,
|
|
527
|
-
...(target.queryHistory.windowDays !== undefined ? { windowDays: target.queryHistory.windowDays } : {}),
|
|
528
|
-
},
|
|
655
|
+
historicSqlPullConfigOverride,
|
|
529
656
|
};
|
|
530
|
-
|
|
657
|
+
// Query history runs after the schema scan has already written its report
|
|
658
|
+
// into the shared target io, so it needs a phase-local capture. Reusing
|
|
659
|
+
// `io` here would let leftover scan text (e.g. "Mode: enriched") surface as
|
|
660
|
+
// the query-history failure detail. Only skip capture when progress is
|
|
661
|
+
// active and the caller manages its own buffer (io is not a capture).
|
|
662
|
+
const capturedIngestIo = deps.ingestProgress && !isCapturedPublicIngestIo(io) ? null : createCapturedPublicIngestIo();
|
|
531
663
|
const ingestIo = capturedIngestIo ?? io;
|
|
532
664
|
const ingestDeps = {
|
|
533
665
|
...(deps.ingestProgress ? { progress: deps.ingestProgress } : {}),
|
|
@@ -564,7 +696,11 @@ export async function executePublicIngestTarget(target, args, io, deps) {
|
|
|
564
696
|
allowImplicitAdapter: true,
|
|
565
697
|
};
|
|
566
698
|
const runIngest = deps.runIngest ?? runKtxIngest;
|
|
567
|
-
const capturedIngestIo = deps.ingestProgress
|
|
699
|
+
const capturedIngestIo = deps.ingestProgress
|
|
700
|
+
? isCapturedPublicIngestIo(io)
|
|
701
|
+
? io
|
|
702
|
+
: null
|
|
703
|
+
: createCapturedPublicIngestIo();
|
|
568
704
|
const ingestIo = capturedIngestIo ?? io;
|
|
569
705
|
const ingestDeps = {
|
|
570
706
|
...(deps.ingestProgress ? { progress: deps.ingestProgress } : {}),
|
|
@@ -628,11 +764,26 @@ export async function runKtxPublicIngest(args, io, deps = {}) {
|
|
|
628
764
|
io.stderr.write(`Warning: ${warning}\n`);
|
|
629
765
|
}
|
|
630
766
|
}
|
|
631
|
-
for (const target of plan.targets) {
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
767
|
+
for (const [index, target] of plan.targets.entries()) {
|
|
768
|
+
if (args.json) {
|
|
769
|
+
results.push(await executePublicIngestTarget(target, args, io, deps, project));
|
|
770
|
+
continue;
|
|
771
|
+
}
|
|
772
|
+
const capture = createCapturedPublicIngestIo();
|
|
773
|
+
const progress = createPlainPublicIngestProgress(io, {
|
|
774
|
+
target,
|
|
775
|
+
index,
|
|
776
|
+
total: plan.targets.length,
|
|
777
|
+
});
|
|
778
|
+
const targetDeps = {
|
|
779
|
+
...deps,
|
|
780
|
+
scanProgress: progress.scanProgress,
|
|
781
|
+
ingestProgress: progress.ingestProgress,
|
|
782
|
+
onPhaseStart: progress.onPhaseStart,
|
|
783
|
+
onPhaseEnd: progress.onPhaseEnd,
|
|
784
|
+
runtimeIo: deps.runtimeIo ?? io,
|
|
785
|
+
};
|
|
786
|
+
results.push(await executePublicIngestTarget(target, args, capture, targetDeps, project));
|
|
636
787
|
}
|
|
637
788
|
if (args.json) {
|
|
638
789
|
io.stdout.write(`${JSON.stringify({ plan, results }, null, 2)}\n`);
|
package/dist/scan.js
CHANGED
|
@@ -6,7 +6,7 @@ import { createKtxCliLocalIngestAdapters } from './local-adapters.js';
|
|
|
6
6
|
import { createKtxCliScanConnector } from './local-scan-connectors.js';
|
|
7
7
|
import { profileMark } from './startup-profile.js';
|
|
8
8
|
import { emitTelemetryEvent } from './telemetry/index.js';
|
|
9
|
-
import { scrubErrorClass } from './telemetry/scrubber.js';
|
|
9
|
+
import { formatErrorDetail, scrubErrorClass } from './telemetry/scrubber.js';
|
|
10
10
|
profileMark('module:scan');
|
|
11
11
|
function shouldUseStyledOutput(io) {
|
|
12
12
|
return io.stdout.isTTY === true && !process.env.NO_COLOR && process.env.TERM !== 'dumb' && !process.env.CI;
|
|
@@ -306,6 +306,7 @@ export async function runKtxScan(args, io = process, deps = {}) {
|
|
|
306
306
|
}
|
|
307
307
|
catch (error) {
|
|
308
308
|
const errorClass = scrubErrorClass(error);
|
|
309
|
+
const errorDetail = formatErrorDetail(error);
|
|
309
310
|
await emitTelemetryEvent({
|
|
310
311
|
name: 'scan_completed',
|
|
311
312
|
projectDir: args.projectDir,
|
|
@@ -319,6 +320,7 @@ export async function runKtxScan(args, io = process, deps = {}) {
|
|
|
319
320
|
durationMs: Math.max(0, performance.now() - startedAt),
|
|
320
321
|
outcome: 'error',
|
|
321
322
|
...(errorClass ? { errorClass } : {}),
|
|
323
|
+
...(errorDetail ? { errorDetail } : {}),
|
|
322
324
|
},
|
|
323
325
|
});
|
|
324
326
|
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
package/dist/setup-context.d.ts
CHANGED
|
@@ -54,6 +54,7 @@ export type KtxSetupContextResult = {
|
|
|
54
54
|
} | {
|
|
55
55
|
status: 'failed';
|
|
56
56
|
projectDir: string;
|
|
57
|
+
errorDetail?: string;
|
|
57
58
|
};
|
|
58
59
|
export interface KtxSetupContextStepArgs {
|
|
59
60
|
projectDir: string;
|
|
@@ -77,6 +78,7 @@ export interface KtxSetupContextDeps {
|
|
|
77
78
|
now?: () => Date;
|
|
78
79
|
runContextBuild?: typeof runContextBuild;
|
|
79
80
|
verifyContextReady?: (projectDir: string) => Promise<KtxSetupContextReadiness>;
|
|
81
|
+
testConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise<number>;
|
|
80
82
|
}
|
|
81
83
|
/** @internal */
|
|
82
84
|
export declare function contextBuildCommands(projectDir: string): KtxSetupContextCommands;
|