@kaelio/ktx 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/assets/python/{kaelio_ktx-0.9.0-py3-none-any.whl → kaelio_ktx-0.10.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/clack.d.ts +6 -0
  5. package/dist/clack.js +17 -2
  6. package/dist/cli-program.d.ts +3 -0
  7. package/dist/cli-program.js +42 -2
  8. package/dist/cli-runtime.d.ts +3 -0
  9. package/dist/cli-runtime.js +44 -0
  10. package/dist/commands/setup-commands.js +2 -3
  11. package/dist/connection.js +23 -1
  12. package/dist/connectors/bigquery/connector.d.ts +2 -5
  13. package/dist/connectors/bigquery/connector.js +2 -2
  14. package/dist/connectors/clickhouse/connector.d.ts +2 -5
  15. package/dist/connectors/clickhouse/connector.js +2 -2
  16. package/dist/connectors/mysql/connector.d.ts +7 -6
  17. package/dist/connectors/mysql/connector.js +25 -5
  18. package/dist/connectors/mysql/dialect.d.ts +1 -1
  19. package/dist/connectors/mysql/dialect.js +12 -2
  20. package/dist/connectors/postgres/connector.d.ts +2 -5
  21. package/dist/connectors/postgres/connector.js +2 -2
  22. package/dist/connectors/snowflake/connector.d.ts +2 -5
  23. package/dist/connectors/snowflake/connector.js +2 -2
  24. package/dist/connectors/sqlite/connector.d.ts +2 -5
  25. package/dist/connectors/sqlite/connector.js +2 -2
  26. package/dist/connectors/sqlserver/connector.d.ts +2 -5
  27. package/dist/connectors/sqlserver/connector.js +2 -2
  28. package/dist/context/connections/drivers.d.ts +0 -1
  29. package/dist/context/connections/drivers.js +0 -7
  30. package/dist/context/connections/query-executor.d.ts +2 -1
  31. package/dist/context/core/abort.d.ts +9 -0
  32. package/dist/context/core/abort.js +36 -0
  33. package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.d.ts +1 -0
  34. package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.js +6 -2
  35. package/dist/context/ingest/context-candidates/curator-pagination.service.d.ts +1 -5
  36. package/dist/context/ingest/context-candidates/curator-pagination.service.js +1 -3
  37. package/dist/context/ingest/context-evidence/sqlite-context-evidence-store.d.ts +1 -1
  38. package/dist/context/ingest/final-gate-repair.d.ts +1 -0
  39. package/dist/context/ingest/final-gate-repair.js +1 -0
  40. package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
  41. package/dist/context/ingest/ingest-bundle.runner.js +127 -53
  42. package/dist/context/ingest/isolated-diff/textual-conflict-resolver.d.ts +1 -0
  43. package/dist/context/ingest/isolated-diff/textual-conflict-resolver.js +1 -0
  44. package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
  45. package/dist/context/ingest/local-bundle-runtime.js +11 -4
  46. package/dist/context/ingest/local-ingest.d.ts +1 -0
  47. package/dist/context/ingest/local-ingest.js +13 -3
  48. package/dist/context/ingest/memory-flow/events.js +1 -1
  49. package/dist/context/ingest/memory-flow/schema.js +8 -3
  50. package/dist/context/ingest/memory-flow/types.d.ts +7 -3
  51. package/dist/context/ingest/ports.d.ts +3 -5
  52. package/dist/context/ingest/stages/stage-3-work-units.d.ts +1 -4
  53. package/dist/context/ingest/stages/stage-3-work-units.js +5 -1
  54. package/dist/context/ingest/stages/stage-4-reconciliation.d.ts +1 -4
  55. package/dist/context/ingest/stages/stage-4-reconciliation.js +1 -1
  56. package/dist/context/ingest/types.d.ts +1 -0
  57. package/dist/context/llm/ai-sdk-runtime.d.ts +3 -0
  58. package/dist/context/llm/ai-sdk-runtime.js +152 -16
  59. package/dist/context/llm/claude-code-runtime.d.ts +6 -4
  60. package/dist/context/llm/claude-code-runtime.js +127 -48
  61. package/dist/context/llm/codex-runtime.d.ts +3 -3
  62. package/dist/context/llm/codex-runtime.js +90 -47
  63. package/dist/context/llm/local-config.d.ts +15 -5
  64. package/dist/context/llm/local-config.js +6 -1
  65. package/dist/context/llm/rate-limit-governor.d.ts +103 -0
  66. package/dist/context/llm/rate-limit-governor.js +285 -0
  67. package/dist/context/llm/runtime-port.d.ts +3 -6
  68. package/dist/context/mcp/context-tools.js +43 -13
  69. package/dist/context/project/config.d.ts +12 -0
  70. package/dist/context/project/config.js +35 -0
  71. package/dist/context/scan/types.d.ts +15 -2
  72. package/dist/context/scan/types.js +12 -0
  73. package/dist/context/sl/description-normalization.js +4 -14
  74. package/dist/context/tools/context-candidate-mark.tool.d.ts +2 -2
  75. package/dist/context-build-view.d.ts +13 -0
  76. package/dist/context-build-view.js +60 -1
  77. package/dist/demo-metrics.d.ts +0 -2
  78. package/dist/demo-metrics.js +1 -11
  79. package/dist/ingest.d.ts +1 -0
  80. package/dist/ingest.js +32 -3
  81. package/dist/io/symbols.d.ts +2 -0
  82. package/dist/io/symbols.js +2 -0
  83. package/dist/memory-flow-hud.js +8 -16
  84. package/dist/public-ingest.js +50 -15
  85. package/dist/reveal-password-prompt.d.ts +24 -0
  86. package/dist/reveal-password-prompt.js +78 -0
  87. package/dist/scan.js +18 -2
  88. package/dist/setup-databases.d.ts +1 -0
  89. package/dist/setup-databases.js +23 -3
  90. package/dist/setup-demo-tour.js +1 -0
  91. package/dist/setup-embeddings.js +1 -1
  92. package/dist/setup-models.d.ts +1 -14
  93. package/dist/setup-models.js +116 -340
  94. package/dist/setup-prompts.js +3 -2
  95. package/dist/setup-sources.js +7 -7
  96. package/dist/setup.d.ts +1 -1
  97. package/dist/setup.js +1 -1
  98. package/dist/sl.d.ts +2 -2
  99. package/dist/sl.js +20 -4
  100. package/dist/sql.js +18 -2
  101. package/dist/star-prompt/cache.d.ts +16 -0
  102. package/dist/star-prompt/cache.js +45 -0
  103. package/dist/star-prompt/star-count.d.ts +7 -0
  104. package/dist/star-prompt/star-count.js +66 -0
  105. package/dist/star-prompt/star-line.d.ts +12 -0
  106. package/dist/star-prompt/star-line.js +26 -0
  107. package/dist/telemetry/emitter.d.ts +10 -0
  108. package/dist/telemetry/emitter.js +31 -0
  109. package/dist/telemetry/events.d.ts +24 -0
  110. package/dist/telemetry/events.js +15 -0
  111. package/dist/telemetry/exception.d.ts +18 -0
  112. package/dist/telemetry/exception.js +162 -0
  113. package/dist/telemetry/index.d.ts +3 -2
  114. package/dist/telemetry/index.js +2 -1
  115. package/dist/telemetry/redaction-secrets.d.ts +11 -0
  116. package/dist/telemetry/redaction-secrets.js +92 -0
  117. package/dist/update-check/cache.d.ts +21 -0
  118. package/dist/update-check/cache.js +38 -0
  119. package/dist/update-check/channel.d.ts +15 -0
  120. package/dist/update-check/channel.js +30 -0
  121. package/dist/update-check/registry.d.ts +1 -0
  122. package/dist/update-check/registry.js +45 -0
  123. package/dist/update-check/update-check.d.ts +43 -0
  124. package/dist/update-check/update-check.js +116 -0
  125. package/package.json +8 -1
  126. package/dist/context/connections/local-query-executor.d.ts +0 -6
  127. package/dist/context/connections/local-query-executor.js +0 -39
  128. package/dist/context/connections/postgres-query-executor.d.ts +0 -25
  129. package/dist/context/connections/postgres-query-executor.js +0 -53
  130. package/dist/context/connections/sqlite-query-executor.d.ts +0 -4
  131. package/dist/context/connections/sqlite-query-executor.js +0 -74
@@ -2,6 +2,7 @@ import type { KtxCliIo } from './index.js';
2
2
  import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
3
3
  import type { KtxPublicIngestArgs, KtxPublicIngestPlanTarget, KtxPublicIngestProject } from './public-ingest.js';
4
4
  import { executePublicIngestTarget } from './public-ingest.js';
5
+ import { fetchGitHubStarCount as defaultFetchGitHubStarCount } from './star-prompt/star-count.js';
5
6
  type PhaseKey = 'database-schema' | 'query-history' | 'source-ingest';
6
7
  type PhaseStatus = 'queued' | 'running' | 'done' | 'failed' | 'skipped';
7
8
  interface PhaseState {
@@ -32,6 +33,7 @@ export interface ContextBuildViewState {
32
33
  frame: number;
33
34
  startedAt: number | null;
34
35
  totalElapsedMs: number;
36
+ starCount: number | null;
35
37
  }
36
38
  export interface ContextBuildArgs {
37
39
  projectDir: string;
@@ -73,6 +75,8 @@ interface CompletedItemName {
73
75
  interface ContextBuildRenderOptions {
74
76
  styled?: boolean;
75
77
  showHint?: boolean;
78
+ showStarPrompt?: boolean;
79
+ columns?: number;
76
80
  hintText?: string;
77
81
  projectDir?: string;
78
82
  title?: string;
@@ -89,6 +93,14 @@ export interface ContextBuildDeps {
89
93
  now?: () => number;
90
94
  onSourceProgress?: (sources: ContextBuildSourceProgressUpdate[]) => void;
91
95
  sourceProgressThrottleMs?: number;
96
+ fetchStarCount?: typeof defaultFetchGitHubStarCount;
97
+ starPromptEnv?: StarPromptEnv;
98
+ starPromptHomeDir?: string;
99
+ }
100
+ interface StarPromptEnv extends NodeJS.ProcessEnv {
101
+ CI?: string;
102
+ DO_NOT_TRACK?: string;
103
+ KTX_NO_STAR?: string;
92
104
  }
93
105
  export declare function renderContextBuildView(state: ContextBuildViewState, options?: ContextBuildRenderOptions): string;
94
106
  /** @internal */
@@ -101,6 +113,7 @@ export declare function parseIngestSummary(output: string): string | null;
101
113
  export declare function viewStateFromSourceProgress(sources: ContextBuildSourceProgressUpdate[], now: number, startedAtMs?: number): ContextBuildViewState;
102
114
  export declare function createRepainter(io: KtxCliIo): {
103
115
  paint(content: string): void;
116
+ columns(): number;
104
117
  };
105
118
  export declare function initViewState(targets: KtxPublicIngestPlanTarget[]): ContextBuildViewState;
106
119
  export declare function runContextBuild(project: KtxPublicIngestProject, args: ContextBuildArgs, io: KtxCliIo, deps?: ContextBuildDeps): Promise<ContextBuildResult>;
@@ -2,6 +2,9 @@ import { buildPublicIngestPlan, executePublicIngestTarget, publicProgressMessage
2
2
  import { createAggregateProgressPort } from './progress-port-adapter.js';
3
3
  import { formatDuration } from './demo-metrics.js';
4
4
  import { profileMark } from './startup-profile.js';
5
+ import { isFreshStarCountCache, readStarCountCache, writeStarCountCache, } from './star-prompt/cache.js';
6
+ import { fetchGitHubStarCount as defaultFetchGitHubStarCount } from './star-prompt/star-count.js';
7
+ import { renderStarPromptLine } from './star-prompt/star-line.js';
5
8
  profileMark('module:context-build-view');
6
9
  const SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
7
10
  const ESC = String.fromCharCode(0x1b);
@@ -286,6 +289,13 @@ export function renderContextBuildView(state, options = {}) {
286
289
  lines.push(styled ? green(summary) : summary);
287
290
  lines.push('');
288
291
  }
292
+ if (options.showStarPrompt && hasActive) {
293
+ const starPrompt = renderStarPromptLine({
294
+ count: state.starCount,
295
+ columns: options.columns ?? 80,
296
+ });
297
+ lines.push(styled ? dim(starPrompt) : starPrompt);
298
+ }
289
299
  if (options.showHint && hasActive) {
290
300
  const hintContent = options.hintText ?? 'Ctrl+C to stop';
291
301
  const hint = ` ${hintContent}`;
@@ -423,6 +433,7 @@ export function viewStateFromSourceProgress(sources, now, startedAtMs) {
423
433
  frame: 0,
424
434
  startedAt: startedAtMs ?? null,
425
435
  totalElapsedMs: startedAtMs ? now - startedAtMs : 0,
436
+ starCount: null,
426
437
  };
427
438
  }
428
439
  // --- Repaint ---
@@ -465,6 +476,9 @@ export function createRepainter(io) {
465
476
  hasPainted = true;
466
477
  lastCursorUpRows = cursorUpRowsAfterWrite(content);
467
478
  },
479
+ columns() {
480
+ return terminalColumns();
481
+ },
468
482
  };
469
483
  }
470
484
  // --- Orchestration ---
@@ -616,12 +630,42 @@ export function initViewState(targets) {
616
630
  frame: 0,
617
631
  startedAt: null,
618
632
  totalElapsedMs: 0,
633
+ starCount: null,
619
634
  };
620
635
  }
621
636
  function formatProgressDetail(update, target) {
622
637
  const percent = Math.max(0, Math.min(100, Math.round(update.percent)));
623
638
  return `[${percent}%] ${publicProgressMessage(update.message, target)}`;
624
639
  }
640
+ const STAR_COUNT_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
641
+ function envFlag(value) {
642
+ return value !== undefined && value !== '' && value !== '0' && value !== 'false';
643
+ }
644
+ function shouldSuppressStarPrompt(env) {
645
+ return envFlag(env.CI) || envFlag(env.DO_NOT_TRACK) || envFlag(env.KTX_NO_STAR);
646
+ }
647
+ function startStarPromptCountRefresh(input) {
648
+ const cached = readStarCountCache({ homeDir: input.homeDir });
649
+ if (cached) {
650
+ input.state.starCount = cached.count;
651
+ }
652
+ if (isFreshStarCountCache(cached, new Date(input.now()), STAR_COUNT_CACHE_TTL_MS)) {
653
+ return;
654
+ }
655
+ void input.fetchStarCount()
656
+ .then((count) => {
657
+ if (typeof count !== 'number' || !Number.isFinite(count)) {
658
+ return;
659
+ }
660
+ input.state.starCount = count;
661
+ input.paint();
662
+ void writeStarCountCache({
663
+ count,
664
+ fetchedAt: new Date(input.now()).toISOString(),
665
+ }, { homeDir: input.homeDir });
666
+ })
667
+ .catch(() => undefined);
668
+ }
625
669
  export async function runContextBuild(project, args, io, deps = {}) {
626
670
  const plan = buildPublicIngestPlan(project, {
627
671
  projectDir: args.projectDir,
@@ -636,13 +680,28 @@ export async function runContextBuild(project, args, io, deps = {}) {
636
680
  const nowFn = deps.now ?? (() => Date.now());
637
681
  state.startedAt = nowFn();
638
682
  const repainter = isTTY ? createRepainter(io) : null;
683
+ const starPromptEnabled = repainter !== null && !shouldSuppressStarPrompt(deps.starPromptEnv ?? process.env);
639
684
  const viewOpts = {
640
685
  styled: true,
641
686
  projectDir: args.projectDir,
642
687
  notices: plan.notices ?? [],
643
688
  warnings: plan.warnings,
644
689
  };
645
- const paint = (hint) => repainter?.paint(renderContextBuildView(state, { ...viewOpts, showHint: hint }));
690
+ const paint = (hint) => repainter?.paint(renderContextBuildView(state, {
691
+ ...viewOpts,
692
+ showHint: hint,
693
+ showStarPrompt: starPromptEnabled && hint,
694
+ columns: repainter.columns(),
695
+ }));
696
+ if (starPromptEnabled) {
697
+ startStarPromptCountRefresh({
698
+ fetchStarCount: deps.fetchStarCount ?? defaultFetchGitHubStarCount,
699
+ homeDir: deps.starPromptHomeDir,
700
+ now: nowFn,
701
+ paint: () => paint(true),
702
+ state,
703
+ });
704
+ }
646
705
  paint(true);
647
706
  let spinnerInterval = null;
648
707
  if (repainter) {
@@ -8,8 +8,6 @@ interface DemoMetricsTuning {
8
8
  interface DemoMetricsSnapshot {
9
9
  elapsedMs: number;
10
10
  etaMs: number | null;
11
- agentSteps: number;
12
- agentStepBudget: number;
13
11
  toolCalls: number;
14
12
  workUnitsStarted: number;
15
13
  workUnitsFinished: number;
@@ -5,13 +5,6 @@ const DEFAULT_OUTPUT_PRICE_PER_MTOK_USD = 15;
5
5
  function eventsOf(events, type) {
6
6
  return events.filter((event) => event.type === type);
7
7
  }
8
- function maxAgentStep(events) {
9
- const steps = eventsOf(events, 'work_unit_step');
10
- const started = eventsOf(events, 'work_unit_started');
11
- const stepIndex = steps.reduce((max, event) => Math.max(max, event.stepIndex), 0);
12
- const stepBudget = Math.max(0, ...steps.map((event) => event.stepBudget), ...started.map((event) => event.stepBudget));
13
- return { step: stepIndex, budget: stepBudget };
14
- }
15
8
  function totalToolCalls(input) {
16
9
  return input.details.transcripts.reduce((total, transcript) => total + transcript.toolCallCount, 0);
17
10
  }
@@ -49,11 +42,10 @@ export function buildDemoMetrics(input, options = {}) {
49
42
  const outputPrice = tuning.outputPricePerMTokUsd ?? DEFAULT_OUTPUT_PRICE_PER_MTOK_USD;
50
43
  const nowMs = (options.now ?? Date.now)();
51
44
  const elapsedMs = elapsedMsFromEvents(input.events, nowMs);
52
- const { step, budget } = maxAgentStep(input.events);
53
45
  const toolCalls = totalToolCalls(input);
54
46
  const progress = workUnitProgress(input);
55
47
  const finishedCount = eventsOf(input.events, 'work_unit_finished').length;
56
- const stepDriver = Math.max(step, toolCalls, finishedCount * 4);
48
+ const stepDriver = Math.max(toolCalls, finishedCount * 4);
57
49
  const inputTokens = stepDriver * inputTokensPerStep;
58
50
  const outputTokens = stepDriver * outputTokensPerStep;
59
51
  const totalTokens = inputTokens + outputTokens;
@@ -63,8 +55,6 @@ export function buildDemoMetrics(input, options = {}) {
63
55
  return {
64
56
  elapsedMs,
65
57
  etaMs: estimateEtaMs(elapsedMs, progress.finished, progress.total, input.status),
66
- agentSteps: step,
67
- agentStepBudget: budget,
68
58
  toolCalls,
69
59
  workUnitsStarted: progress.started,
70
60
  workUnitsFinished: progress.finished,
package/dist/ingest.d.ts CHANGED
@@ -58,6 +58,7 @@ export interface KtxIngestDeps {
58
58
  readReportFile?: typeof readIngestReportSnapshotFile;
59
59
  renderStoredMemoryFlow?: typeof renderMemoryFlowTui;
60
60
  startLiveMemoryFlow?: typeof startLiveMemoryFlowTui;
61
+ abortSignal?: AbortSignal;
61
62
  env?: NodeJS.ProcessEnv;
62
63
  localIngestOptions?: Pick<RunLocalIngestOptions, 'agentRunner' | 'llmRuntime' | 'memoryModel' | 'semanticLayerCompute' | 'queryExecutor' | 'logger' | 'pullConfigOptions'>;
63
64
  progress?: (update: KtxIngestProgressUpdate) => void;
package/dist/ingest.js CHANGED
@@ -17,6 +17,22 @@ import { renderMemoryFlowTui, startLiveMemoryFlowTui, } from './memory-flow-tui.
17
17
  import { resolveVizFallback, warnVizFallbackOnce } from './viz-fallback.js';
18
18
  import { profileMark } from './startup-profile.js';
19
19
  profileMark('module:ingest');
20
+ function createCliAbortSignal() {
21
+ const controller = new AbortController();
22
+ let interrupted = false;
23
+ const onSigint = () => {
24
+ if (interrupted) {
25
+ process.exit(130);
26
+ }
27
+ interrupted = true;
28
+ controller.abort(new DOMException('Aborted', 'AbortError'));
29
+ };
30
+ process.on('SIGINT', onSigint);
31
+ return {
32
+ signal: controller.signal,
33
+ dispose: () => process.off('SIGINT', onSigint),
34
+ };
35
+ }
20
36
  const REPORT_SOURCE_LABELS = new Map([
21
37
  ['live-database', 'Database schema'],
22
38
  ['historic-sql', 'Query history'],
@@ -249,6 +265,12 @@ function plainIngestEventProgress(event, snapshot, eventIndex) {
249
265
  message: event.message,
250
266
  ...(event.transient !== undefined ? { transient: event.transient } : {}),
251
267
  };
268
+ case 'rate_limit_wait':
269
+ return {
270
+ percent: 50,
271
+ message: `Rate-limited (${event.provider}${event.rateLimitType ? ` ${event.rateLimitType}` : ''}); resuming in ${Math.ceil(event.remainingMs / 1_000)}s`,
272
+ transient: true,
273
+ };
252
274
  case 'work_unit_started': {
253
275
  const total = plannedWorkUnitCountThrough(snapshot, eventIndex);
254
276
  const ordinal = workUnitOrdinalThrough(snapshot, eventIndex, event.unitKey);
@@ -259,9 +281,8 @@ function plainIngestEventProgress(event, snapshot, eventIndex) {
259
281
  const total = plannedWorkUnitCountThrough(snapshot, eventIndex);
260
282
  const completed = completedWorkUnitCountThrough(snapshot, eventIndex);
261
283
  const active = activeWorkUnitCountThrough(snapshot, eventIndex);
262
- const stepFraction = event.stepBudget > 0 ? Math.min(1, event.stepIndex / event.stepBudget) : 0;
263
- const percent = total > 0 ? 55 + Math.ceil(((completed + stepFraction) / total) * 25) : 55;
264
- const latest = `${event.unitKey} step ${event.stepIndex}/${event.stepBudget}`;
284
+ const percent = total > 0 ? 55 + Math.ceil((completed / total) * 25) : 55;
285
+ const latest = `${event.unitKey} · ${pluralize(event.toolCalls, 'action')}`;
265
286
  return {
266
287
  percent,
267
288
  message: `Processing tasks: ${completed}/${total} complete, ${active} active; latest ${latest}`,
@@ -546,6 +567,8 @@ export async function runKtxIngest(args, io = process, deps = {}) {
546
567
  : io, deps.progress);
547
568
  plainProgress?.start();
548
569
  structuredProgress?.start();
570
+ const cliAbort = deps.abortSignal ? null : createCliAbortSignal();
571
+ const abortSignal = deps.abortSignal ?? cliAbort?.signal;
549
572
  let result;
550
573
  try {
551
574
  result = await executeMetabaseFanout({
@@ -559,6 +582,7 @@ export async function runKtxIngest(args, io = process, deps = {}) {
559
582
  embeddingProvider,
560
583
  ...(memoryFlow ? { memoryFlow } : {}),
561
584
  ...(progress ? { progress } : {}),
585
+ ...(abortSignal ? { abortSignal } : {}),
562
586
  });
563
587
  plainProgress?.flush();
564
588
  if (args.outputMode === 'json') {
@@ -570,6 +594,7 @@ export async function runKtxIngest(args, io = process, deps = {}) {
570
594
  }
571
595
  finally {
572
596
  plainProgress?.flush();
597
+ cliAbort?.dispose();
573
598
  }
574
599
  return result.status === 'all_failed' ? 1 : 0;
575
600
  }
@@ -612,6 +637,8 @@ export async function runKtxIngest(args, io = process, deps = {}) {
612
637
  : undefined;
613
638
  plainProgress?.start();
614
639
  structuredProgress?.start();
640
+ const cliAbort = deps.abortSignal ? null : createCliAbortSignal();
641
+ const abortSignal = deps.abortSignal ?? cliAbort?.signal;
615
642
  try {
616
643
  const result = await executeLocalIngest({
617
644
  project: ingestProject,
@@ -627,6 +654,7 @@ export async function runKtxIngest(args, io = process, deps = {}) {
627
654
  embeddingProvider,
628
655
  ...(args.debugLlmRequestFile ? { llmDebugRequestFile: args.debugLlmRequestFile } : {}),
629
656
  ...(memoryFlow ? { memoryFlow } : {}),
657
+ ...(abortSignal ? { abortSignal } : {}),
630
658
  });
631
659
  if (shouldUseLiveViz && memoryFlow) {
632
660
  latestMemoryFlowSnapshot = finalRunMemoryFlowInput(memoryFlow.snapshot(), result.report);
@@ -646,6 +674,7 @@ export async function runKtxIngest(args, io = process, deps = {}) {
646
674
  finally {
647
675
  plainProgress?.flush();
648
676
  liveTui?.close();
677
+ cliAbort?.dispose();
649
678
  }
650
679
  }
651
680
  if (args.reportFile) {
@@ -1,6 +1,8 @@
1
1
  export declare const SYMBOLS: {
2
2
  readonly middot: "-" | "·";
3
3
  readonly emDash: "--" | "—";
4
+ readonly star: "*" | "★";
5
+ readonly rightArrow: "→" | "->";
4
6
  };
5
7
  export declare function dim(text: string): string;
6
8
  export declare function bold(text: string): string;
@@ -12,6 +12,8 @@ const unicode = detectUnicodeSupport();
12
12
  export const SYMBOLS = {
13
13
  middot: unicode ? '·' : '-',
14
14
  emDash: unicode ? '—' : '--',
15
+ star: unicode ? '★' : '*',
16
+ rightArrow: unicode ? '→' : '->',
15
17
  };
16
18
  export function dim(text) {
17
19
  return styleText('dim', text);
@@ -97,27 +97,19 @@ function sourceDescription(input) {
97
97
  }
98
98
  function activeWorkUnits(input) {
99
99
  const finishedKeys = new Set();
100
- const unitMap = new Map();
101
100
  for (const e of input.events) {
102
- if (e.type === 'work_unit_started') {
103
- unitMap.set(e.unitKey, { stepIndex: 0, stepBudget: e.stepBudget });
104
- }
105
- if (e.type === 'work_unit_step') {
106
- const existing = unitMap.get(e.unitKey);
107
- if (existing) {
108
- existing.stepIndex = e.stepIndex;
109
- existing.stepBudget = e.stepBudget;
110
- }
111
- }
112
101
  if (e.type === 'work_unit_finished')
113
102
  finishedKeys.add(e.unitKey);
114
103
  }
115
- const result = [];
116
- for (const [unitKey, data] of unitMap) {
117
- if (!finishedKeys.has(unitKey))
118
- result.push({ unitKey, ...data });
104
+ const active = [];
105
+ const seen = new Set();
106
+ for (const e of input.events) {
107
+ if (e.type === 'work_unit_started' && !finishedKeys.has(e.unitKey) && !seen.has(e.unitKey)) {
108
+ seen.add(e.unitKey);
109
+ active.push(e.unitKey);
110
+ }
119
111
  }
120
- return result;
112
+ return active;
121
113
  }
122
114
  function queuedWorkUnits(input) {
123
115
  const startedKeys = new Set();
@@ -8,7 +8,8 @@ import { createAggregateProgressPort } from './progress-port-adapter.js';
8
8
  import { resolvePublicIngestRuntimeRequirements } from './runtime-requirements.js';
9
9
  import { profileMark } from './startup-profile.js';
10
10
  import { isDemoConnection } from './telemetry/demo-detect.js';
11
- import { emitProjectStackSnapshot, emitTelemetryEvent } from './telemetry/index.js';
11
+ import { emitProjectStackSnapshot, emitTelemetryEvent, reportException } from './telemetry/index.js';
12
+ import { collectTelemetryRedactionSecrets } from './telemetry/redaction-secrets.js';
12
13
  import { formatErrorDetail } from './telemetry/scrubber.js';
13
14
  profileMark('module:public-ingest');
14
15
  const sourceAdapterByDriver = new Map([
@@ -733,26 +734,60 @@ export async function runKtxPublicIngest(args, io, deps = {}) {
733
734
  });
734
735
  }
735
736
  catch (error) {
737
+ await reportException({
738
+ error,
739
+ context: { source: 'ingest runtime', handled: true, fatal: false },
740
+ projectDir: args.projectDir,
741
+ io,
742
+ redactionSecrets: await collectTelemetryRedactionSecrets({
743
+ project,
744
+ projectDir: args.projectDir,
745
+ connectionId: args.targetConnectionId,
746
+ includeLlm: true,
747
+ includeEmbeddings: true,
748
+ env: deps.env ?? process.env,
749
+ }),
750
+ });
736
751
  io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
737
752
  return 1;
738
753
  }
739
754
  }
740
755
  const { runContextBuild } = await import('./context-build-view.js');
741
756
  const contextBuild = deps.runContextBuild ?? runContextBuild;
742
- const result = await contextBuild(project, {
743
- projectDir: args.projectDir,
744
- ...(args.targetConnectionId ? { targetConnectionId: args.targetConnectionId } : {}),
745
- all: args.all,
746
- entrypoint: 'ingest',
747
- inputMode: args.inputMode,
748
- ...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
749
- ...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
750
- ...(args.scanMode ? { scanMode: args.scanMode } : {}),
751
- ...(args.detectRelationships !== undefined ? { detectRelationships: args.detectRelationships } : {}),
752
- ...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
753
- ...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
754
- }, io);
755
- return result.exitCode;
757
+ try {
758
+ const result = await contextBuild(project, {
759
+ projectDir: args.projectDir,
760
+ ...(args.targetConnectionId ? { targetConnectionId: args.targetConnectionId } : {}),
761
+ all: args.all,
762
+ entrypoint: 'ingest',
763
+ inputMode: args.inputMode,
764
+ ...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
765
+ ...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
766
+ ...(args.scanMode ? { scanMode: args.scanMode } : {}),
767
+ ...(args.detectRelationships !== undefined ? { detectRelationships: args.detectRelationships } : {}),
768
+ ...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
769
+ ...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
770
+ }, io);
771
+ return result.exitCode;
772
+ }
773
+ catch (error) {
774
+ await reportException({
775
+ error,
776
+ context: { source: 'ingest context-build', handled: true, fatal: false },
777
+ projectDir: args.projectDir,
778
+ io,
779
+ redactionSecrets: await collectTelemetryRedactionSecrets({
780
+ project,
781
+ projectDir: args.projectDir,
782
+ connectionId: args.targetConnectionId,
783
+ includeLlm: true,
784
+ includeEmbeddings: true,
785
+ env: deps.env ?? process.env,
786
+ }),
787
+ });
788
+ io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
789
+ return 1;
790
+ }
756
791
  }
757
792
  const plan = buildPublicIngestPlan(project, args);
758
793
  const results = [];
@@ -0,0 +1,24 @@
1
+ import { type PasswordOptions } from '@clack/core';
2
+ /**
3
+ * Mask every character of `userInput` except the last `tail`, but only reveal the
4
+ * tail once the secret is long enough that the hidden portion still dominates
5
+ * (`length > tail * 2`). Short secrets stay fully masked so we never expose most
6
+ * of a small value. The returned string keeps the same code-unit length as the
7
+ * input so clack's cursor slicing in `userInputWithCursor` stays aligned.
8
+ *
9
+ * @internal
10
+ */
11
+ export declare function maskRevealingTail(userInput: string, maskChar: string, tail: number): string;
12
+ export interface RevealPasswordOptions {
13
+ message: string;
14
+ mask?: string;
15
+ tail?: number;
16
+ validate?: PasswordOptions['validate'];
17
+ signal?: AbortSignal;
18
+ }
19
+ /**
20
+ * Drop-in replacement for clack's `password()` that reveals the last few
21
+ * characters of the entered value while typing. Resolves to the raw value or the
22
+ * clack cancel symbol, matching `password()`'s contract.
23
+ */
24
+ export declare function revealPassword(options: RevealPasswordOptions): Promise<string | symbol>;
@@ -0,0 +1,78 @@
1
+ import { styleText } from 'node:util';
2
+ import { PasswordPrompt } from '@clack/core';
3
+ import { S_BAR, S_BAR_END, S_PASSWORD_MASK, settings, symbol } from '@clack/prompts';
4
+ // How many trailing characters of a pasted secret to leave visible so the user
5
+ // can confirm what landed (e.g. `••••••a1b2`). Kept small on purpose.
6
+ const REVEAL_TAIL_COUNT = 4;
7
+ /**
8
+ * Mask every character of `userInput` except the last `tail`, but only reveal the
9
+ * tail once the secret is long enough that the hidden portion still dominates
10
+ * (`length > tail * 2`). Short secrets stay fully masked so we never expose most
11
+ * of a small value. The returned string keeps the same code-unit length as the
12
+ * input so clack's cursor slicing in `userInputWithCursor` stays aligned.
13
+ *
14
+ * @internal
15
+ */
16
+ export function maskRevealingTail(userInput, maskChar, tail) {
17
+ const revealLength = userInput.length > tail * 2 ? tail : 0;
18
+ const hiddenLength = userInput.length - revealLength;
19
+ return maskChar.repeat(hiddenLength) + userInput.slice(hiddenLength);
20
+ }
21
+ class RevealTailPasswordPrompt extends PasswordPrompt {
22
+ #maskChar;
23
+ #tail;
24
+ constructor(options) {
25
+ super(options);
26
+ this.#maskChar = options.mask ?? S_PASSWORD_MASK;
27
+ this.#tail = options.tail;
28
+ }
29
+ get masked() {
30
+ return maskRevealingTail(this.userInput, this.#maskChar, this.#tail);
31
+ }
32
+ }
33
+ // Reproduces the @clack/prompts password frame (pinned to the installed version)
34
+ // so this prompt is visually identical to every other setup prompt; the only
35
+ // behavioral change is the tail-revealing `masked` getter above.
36
+ function renderPasswordFrame(prompt, message) {
37
+ const withGuide = settings.withGuide;
38
+ const title = `${withGuide ? `${styleText('gray', S_BAR)}\n` : ''}${symbol(prompt.state)} ${message}\n`;
39
+ const masked = prompt.masked;
40
+ switch (prompt.state) {
41
+ case 'error': {
42
+ const bar = withGuide ? `${styleText('yellow', S_BAR)} ` : '';
43
+ const end = withGuide ? `${styleText('yellow', S_BAR_END)} ` : '';
44
+ return `${title.trim()}\n${bar}${masked}\n${end}${styleText('yellow', prompt.error)}\n`;
45
+ }
46
+ case 'submit': {
47
+ const bar = withGuide ? `${styleText('gray', S_BAR)} ` : '';
48
+ return `${title}${bar}${masked ? styleText('dim', masked) : ''}`;
49
+ }
50
+ case 'cancel': {
51
+ const bar = withGuide ? `${styleText('gray', S_BAR)} ` : '';
52
+ const body = masked ? styleText(['strikethrough', 'dim'], masked) : '';
53
+ return `${title}${bar}${body}${masked && withGuide ? `\n${styleText('gray', S_BAR)}` : ''}`;
54
+ }
55
+ default: {
56
+ const bar = withGuide ? `${styleText('cyan', S_BAR)} ` : '';
57
+ const end = withGuide ? styleText('cyan', S_BAR_END) : '';
58
+ return `${title}${bar}${prompt.userInputWithCursor}\n${end}\n`;
59
+ }
60
+ }
61
+ }
62
+ /**
63
+ * Drop-in replacement for clack's `password()` that reveals the last few
64
+ * characters of the entered value while typing. Resolves to the raw value or the
65
+ * clack cancel symbol, matching `password()`'s contract.
66
+ */
67
+ export function revealPassword(options) {
68
+ const prompt = new RevealTailPasswordPrompt({
69
+ mask: options.mask ?? S_PASSWORD_MASK,
70
+ tail: options.tail ?? REVEAL_TAIL_COUNT,
71
+ validate: options.validate,
72
+ signal: options.signal,
73
+ render() {
74
+ return renderPasswordFrame(this, options.message);
75
+ },
76
+ });
77
+ return prompt.prompt();
78
+ }
package/dist/scan.js CHANGED
@@ -5,7 +5,8 @@ import { resolveProjectEmbeddingProvider } from './embedding-resolution.js';
5
5
  import { createKtxCliLocalIngestAdapters } from './local-adapters.js';
6
6
  import { createKtxCliScanConnector } from './local-scan-connectors.js';
7
7
  import { profileMark } from './startup-profile.js';
8
- import { emitTelemetryEvent } from './telemetry/index.js';
8
+ import { emitTelemetryEvent, reportException } from './telemetry/index.js';
9
+ import { collectTelemetryRedactionSecrets } from './telemetry/redaction-secrets.js';
9
10
  import { formatErrorDetail, scrubErrorClass } from './telemetry/scrubber.js';
10
11
  profileMark('module:scan');
11
12
  function shouldUseStyledOutput(io) {
@@ -248,8 +249,9 @@ export function createCliScanProgress(io, state = { progress: 0, hasPendingTrans
248
249
  }
249
250
  export async function runKtxScan(args, io = process, deps = {}) {
250
251
  const startedAt = performance.now();
252
+ let project;
251
253
  try {
252
- const project = await loadKtxProject({ projectDir: args.projectDir });
254
+ project = await loadKtxProject({ projectDir: args.projectDir });
253
255
  const resolveEmbeddingProvider = deps.resolveEmbeddingProvider ?? resolveProjectEmbeddingProvider;
254
256
  const resolution = await resolveEmbeddingProvider(project, {
255
257
  mode: 'ensure',
@@ -323,6 +325,20 @@ export async function runKtxScan(args, io = process, deps = {}) {
323
325
  ...(errorDetail ? { errorDetail } : {}),
324
326
  },
325
327
  });
328
+ await reportException({
329
+ error,
330
+ context: { source: 'scan run', handled: true, fatal: false },
331
+ projectDir: args.projectDir,
332
+ io,
333
+ redactionSecrets: await collectTelemetryRedactionSecrets({
334
+ project,
335
+ projectDir: args.projectDir,
336
+ connectionId: args.connectionId,
337
+ includeLlm: true,
338
+ includeEmbeddings: true,
339
+ env: process.env,
340
+ }),
341
+ });
326
342
  io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
327
343
  return 1;
328
344
  }
@@ -12,6 +12,7 @@ export type KtxSetupDatabaseDriver = 'sqlite' | 'postgres' | 'mysql' | 'clickhou
12
12
  export interface KtxSetupDatabasesArgs {
13
13
  projectDir: string;
14
14
  inputMode: 'auto' | 'disabled';
15
+ debug?: boolean;
15
16
  yes?: boolean;
16
17
  cliVersion?: string;
17
18
  runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
@@ -1206,7 +1206,10 @@ function hasServiceAccountsBlock(connection) {
1206
1206
  }
1207
1207
  return 'serviceAccounts' in filters;
1208
1208
  }
1209
- function printQueryHistoryFilterProposal(io, proposal) {
1209
+ function printQueryHistoryFilterProposal(io, proposal, debug = false) {
1210
+ if (debug && proposal.parseFailedTemplateIds.length > 0) {
1211
+ io.stderr.write(`[debug] query-history filter picker could not parse ${proposal.parseFailedTemplateIds.length} template(s): ${proposal.parseFailedTemplateIds.join(', ')}\n`);
1212
+ }
1210
1213
  if (proposal.excludedRoles.length === 0) {
1211
1214
  if (proposal.skipped?.reason === 'no-llm') {
1212
1215
  io.stdout.write('│ Query-history filter picker skipped: no LLM is configured.\n');
@@ -1217,6 +1220,10 @@ function printQueryHistoryFilterProposal(io, proposal) {
1217
1220
  else if (proposal.skipped?.reason === 'no-in-scope-history') {
1218
1221
  io.stdout.write('│ Query-history filter picker found no in-scope service-account exclusions.\n');
1219
1222
  }
1223
+ if (proposal.parseFailedTemplateIds.length > 0) {
1224
+ const count = proposal.parseFailedTemplateIds.length;
1225
+ io.stdout.write(`│ Skipped ${count} query template${count === 1 ? '' : 's'} ktx could not parse (run with --debug to list them).\n`);
1226
+ }
1220
1227
  for (const warning of proposal.warnings) {
1221
1228
  io.stdout.write(`│ ! ${warning}\n`);
1222
1229
  }
@@ -1286,7 +1293,8 @@ async function maybeProposeQueryHistoryFilters(input) {
1286
1293
  consideredRoleCount: 0,
1287
1294
  skipped: { reason: 'no-llm' },
1288
1295
  warnings: [],
1289
- });
1296
+ parseFailedTemplateIds: [],
1297
+ }, input.args.debug === true);
1290
1298
  return;
1291
1299
  }
1292
1300
  const runtime = createKtxCliHistoricSqlRuntime(project, input.connectionId, {
@@ -1325,7 +1333,19 @@ async function maybeProposeQueryHistoryFilters(input) {
1325
1333
  pullConfig,
1326
1334
  userServiceAccountsPresent,
1327
1335
  });
1328
- printQueryHistoryFilterProposal(input.io, proposal);
1336
+ printQueryHistoryFilterProposal(input.io, proposal, input.args.debug === true);
1337
+ await emitTelemetryEvent({
1338
+ name: 'query_history_filter_completed',
1339
+ projectDir: input.projectDir,
1340
+ io: input.io,
1341
+ fields: {
1342
+ dialect,
1343
+ consideredRoleCount: proposal.consideredRoleCount,
1344
+ excludedRoleCount: proposal.excludedRoles.length,
1345
+ parseFailedCount: proposal.parseFailedTemplateIds.length,
1346
+ outcome: 'ok',
1347
+ },
1348
+ });
1329
1349
  if (proposal.skipped?.reason === 'user-block-present') {
1330
1350
  input.io.stdout.write('│ Existing query-history service-account filters left unchanged.\n');
1331
1351
  return;