@kaelio/ktx 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/assets/python/{kaelio_ktx-0.9.0-py3-none-any.whl → kaelio_ktx-0.10.0-py3-none-any.whl} +0 -0
  2. package/assets/python/manifest.json +4 -4
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/clack.d.ts +6 -0
  5. package/dist/clack.js +17 -2
  6. package/dist/cli-program.d.ts +3 -0
  7. package/dist/cli-program.js +42 -2
  8. package/dist/cli-runtime.d.ts +3 -0
  9. package/dist/cli-runtime.js +44 -0
  10. package/dist/commands/setup-commands.js +2 -3
  11. package/dist/connection.js +23 -1
  12. package/dist/connectors/bigquery/connector.d.ts +2 -5
  13. package/dist/connectors/bigquery/connector.js +2 -2
  14. package/dist/connectors/clickhouse/connector.d.ts +2 -5
  15. package/dist/connectors/clickhouse/connector.js +2 -2
  16. package/dist/connectors/mysql/connector.d.ts +7 -6
  17. package/dist/connectors/mysql/connector.js +25 -5
  18. package/dist/connectors/mysql/dialect.d.ts +1 -1
  19. package/dist/connectors/mysql/dialect.js +12 -2
  20. package/dist/connectors/postgres/connector.d.ts +2 -5
  21. package/dist/connectors/postgres/connector.js +2 -2
  22. package/dist/connectors/snowflake/connector.d.ts +2 -5
  23. package/dist/connectors/snowflake/connector.js +2 -2
  24. package/dist/connectors/sqlite/connector.d.ts +2 -5
  25. package/dist/connectors/sqlite/connector.js +2 -2
  26. package/dist/connectors/sqlserver/connector.d.ts +2 -5
  27. package/dist/connectors/sqlserver/connector.js +2 -2
  28. package/dist/context/connections/drivers.d.ts +0 -1
  29. package/dist/context/connections/drivers.js +0 -7
  30. package/dist/context/connections/query-executor.d.ts +2 -1
  31. package/dist/context/core/abort.d.ts +9 -0
  32. package/dist/context/core/abort.js +36 -0
  33. package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.d.ts +1 -0
  34. package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.js +6 -2
  35. package/dist/context/ingest/context-candidates/curator-pagination.service.d.ts +1 -5
  36. package/dist/context/ingest/context-candidates/curator-pagination.service.js +1 -3
  37. package/dist/context/ingest/context-evidence/sqlite-context-evidence-store.d.ts +1 -1
  38. package/dist/context/ingest/final-gate-repair.d.ts +1 -0
  39. package/dist/context/ingest/final-gate-repair.js +1 -0
  40. package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
  41. package/dist/context/ingest/ingest-bundle.runner.js +127 -53
  42. package/dist/context/ingest/isolated-diff/textual-conflict-resolver.d.ts +1 -0
  43. package/dist/context/ingest/isolated-diff/textual-conflict-resolver.js +1 -0
  44. package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
  45. package/dist/context/ingest/local-bundle-runtime.js +11 -4
  46. package/dist/context/ingest/local-ingest.d.ts +1 -0
  47. package/dist/context/ingest/local-ingest.js +13 -3
  48. package/dist/context/ingest/memory-flow/events.js +1 -1
  49. package/dist/context/ingest/memory-flow/schema.js +8 -3
  50. package/dist/context/ingest/memory-flow/types.d.ts +7 -3
  51. package/dist/context/ingest/ports.d.ts +3 -5
  52. package/dist/context/ingest/stages/stage-3-work-units.d.ts +1 -4
  53. package/dist/context/ingest/stages/stage-3-work-units.js +5 -1
  54. package/dist/context/ingest/stages/stage-4-reconciliation.d.ts +1 -4
  55. package/dist/context/ingest/stages/stage-4-reconciliation.js +1 -1
  56. package/dist/context/ingest/types.d.ts +1 -0
  57. package/dist/context/llm/ai-sdk-runtime.d.ts +3 -0
  58. package/dist/context/llm/ai-sdk-runtime.js +152 -16
  59. package/dist/context/llm/claude-code-runtime.d.ts +6 -4
  60. package/dist/context/llm/claude-code-runtime.js +127 -48
  61. package/dist/context/llm/codex-runtime.d.ts +3 -3
  62. package/dist/context/llm/codex-runtime.js +90 -47
  63. package/dist/context/llm/local-config.d.ts +15 -5
  64. package/dist/context/llm/local-config.js +6 -1
  65. package/dist/context/llm/rate-limit-governor.d.ts +103 -0
  66. package/dist/context/llm/rate-limit-governor.js +285 -0
  67. package/dist/context/llm/runtime-port.d.ts +3 -6
  68. package/dist/context/mcp/context-tools.js +43 -13
  69. package/dist/context/project/config.d.ts +12 -0
  70. package/dist/context/project/config.js +35 -0
  71. package/dist/context/scan/types.d.ts +15 -2
  72. package/dist/context/scan/types.js +12 -0
  73. package/dist/context/sl/description-normalization.js +4 -14
  74. package/dist/context/tools/context-candidate-mark.tool.d.ts +2 -2
  75. package/dist/context-build-view.d.ts +13 -0
  76. package/dist/context-build-view.js +60 -1
  77. package/dist/demo-metrics.d.ts +0 -2
  78. package/dist/demo-metrics.js +1 -11
  79. package/dist/ingest.d.ts +1 -0
  80. package/dist/ingest.js +32 -3
  81. package/dist/io/symbols.d.ts +2 -0
  82. package/dist/io/symbols.js +2 -0
  83. package/dist/memory-flow-hud.js +8 -16
  84. package/dist/public-ingest.js +50 -15
  85. package/dist/reveal-password-prompt.d.ts +24 -0
  86. package/dist/reveal-password-prompt.js +78 -0
  87. package/dist/scan.js +18 -2
  88. package/dist/setup-databases.d.ts +1 -0
  89. package/dist/setup-databases.js +23 -3
  90. package/dist/setup-demo-tour.js +1 -0
  91. package/dist/setup-embeddings.js +1 -1
  92. package/dist/setup-models.d.ts +1 -14
  93. package/dist/setup-models.js +116 -340
  94. package/dist/setup-prompts.js +3 -2
  95. package/dist/setup-sources.js +7 -7
  96. package/dist/setup.d.ts +1 -1
  97. package/dist/setup.js +1 -1
  98. package/dist/sl.d.ts +2 -2
  99. package/dist/sl.js +20 -4
  100. package/dist/sql.js +18 -2
  101. package/dist/star-prompt/cache.d.ts +16 -0
  102. package/dist/star-prompt/cache.js +45 -0
  103. package/dist/star-prompt/star-count.d.ts +7 -0
  104. package/dist/star-prompt/star-count.js +66 -0
  105. package/dist/star-prompt/star-line.d.ts +12 -0
  106. package/dist/star-prompt/star-line.js +26 -0
  107. package/dist/telemetry/emitter.d.ts +10 -0
  108. package/dist/telemetry/emitter.js +31 -0
  109. package/dist/telemetry/events.d.ts +24 -0
  110. package/dist/telemetry/events.js +15 -0
  111. package/dist/telemetry/exception.d.ts +18 -0
  112. package/dist/telemetry/exception.js +162 -0
  113. package/dist/telemetry/index.d.ts +3 -2
  114. package/dist/telemetry/index.js +2 -1
  115. package/dist/telemetry/redaction-secrets.d.ts +11 -0
  116. package/dist/telemetry/redaction-secrets.js +92 -0
  117. package/dist/update-check/cache.d.ts +21 -0
  118. package/dist/update-check/cache.js +38 -0
  119. package/dist/update-check/channel.d.ts +15 -0
  120. package/dist/update-check/channel.js +30 -0
  121. package/dist/update-check/registry.d.ts +1 -0
  122. package/dist/update-check/registry.js +45 -0
  123. package/dist/update-check/update-check.d.ts +43 -0
  124. package/dist/update-check/update-check.js +116 -0
  125. package/package.json +8 -1
  126. package/dist/context/connections/local-query-executor.d.ts +0 -6
  127. package/dist/context/connections/local-query-executor.js +0 -39
  128. package/dist/context/connections/postgres-query-executor.d.ts +0 -25
  129. package/dist/context/connections/postgres-query-executor.js +0 -53
  130. package/dist/context/connections/sqlite-query-executor.d.ts +0 -4
  131. package/dist/context/connections/sqlite-query-executor.js +0 -74
@@ -1,4 +1,4 @@
1
- import { type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaSnapshot, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
1
+ import { type KtxConnectorTestResult, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaSnapshot, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
2
2
  export interface KtxSqlServerConnectionConfig {
3
3
  driver?: string;
4
4
  host?: string;
@@ -113,10 +113,7 @@ export declare class KtxSqlServerScanConnector implements KtxScanConnector {
113
113
  private pool;
114
114
  private resolvedEndpoint;
115
115
  constructor(options: KtxSqlServerScanConnectorOptions);
116
- testConnection(): Promise<{
117
- success: boolean;
118
- error?: string;
119
- }>;
116
+ testConnection(): Promise<KtxConnectorTestResult>;
120
117
  introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise<KtxSchemaSnapshot>;
121
118
  sampleTable(input: KtxTableSampleInput, _ctx: KtxScanContext): Promise<KtxSqlServerTableSampleResult>;
122
119
  sampleColumn(input: KtxColumnSampleInput, _ctx: KtxScanContext): Promise<KtxColumnSampleResult>;
@@ -2,7 +2,7 @@ import { assertReadOnlySql } from '../../context/connections/read-only-sql.js';
2
2
  import { getDialectForDriver } from '../../context/connections/dialects.js';
3
3
  import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
4
4
  import { scopedTableNames } from '../../context/scan/table-ref.js';
5
- import { createKtxConnectorCapabilities, } from '../../context/scan/types.js';
5
+ import { connectorTestFailure, createKtxConnectorCapabilities, } from '../../context/scan/types.js';
6
6
  import { readFileSync } from 'node:fs';
7
7
  import { homedir } from 'node:os';
8
8
  import { resolve } from 'node:path';
@@ -235,7 +235,7 @@ export class KtxSqlServerScanConnector {
235
235
  return { success: true };
236
236
  }
237
237
  catch (error) {
238
- return { success: false, error: error instanceof Error ? error.message : String(error) };
238
+ return connectorTestFailure(error);
239
239
  }
240
240
  }
241
241
  async introspect(input, _ctx) {
@@ -14,7 +14,6 @@ export interface KtxDriverRegistration {
14
14
  readonly driver: KtxConnectionDriver;
15
15
  readonly scopeConfigKey: KtxScopeConfigKey | null;
16
16
  readonly hasHistoricSqlReader: boolean;
17
- readonly hasLocalQueryExecutor: boolean;
18
17
  load(): Promise<KtxDriverConnectorModule>;
19
18
  }
20
19
  /** @internal */
@@ -7,7 +7,6 @@ export const driverRegistrations = {
7
7
  driver: 'bigquery',
8
8
  scopeConfigKey: 'dataset_ids',
9
9
  hasHistoricSqlReader: true,
10
- hasLocalQueryExecutor: false,
11
10
  load: async () => {
12
11
  const m = await import('../../connectors/bigquery/connector.js');
13
12
  return {
@@ -29,7 +28,6 @@ export const driverRegistrations = {
29
28
  driver: 'clickhouse',
30
29
  scopeConfigKey: 'databases',
31
30
  hasHistoricSqlReader: false,
32
- hasLocalQueryExecutor: false,
33
31
  load: async () => {
34
32
  const m = await import('../../connectors/clickhouse/connector.js');
35
33
  return {
@@ -51,7 +49,6 @@ export const driverRegistrations = {
51
49
  driver: 'mysql',
52
50
  scopeConfigKey: 'schemas',
53
51
  hasHistoricSqlReader: false,
54
- hasLocalQueryExecutor: false,
55
52
  load: async () => {
56
53
  const m = await import('../../connectors/mysql/connector.js');
57
54
  return {
@@ -73,7 +70,6 @@ export const driverRegistrations = {
73
70
  driver: 'postgres',
74
71
  scopeConfigKey: 'schemas',
75
72
  hasHistoricSqlReader: true,
76
- hasLocalQueryExecutor: true,
77
73
  load: async () => {
78
74
  const m = await import('../../connectors/postgres/connector.js');
79
75
  return {
@@ -95,7 +91,6 @@ export const driverRegistrations = {
95
91
  driver: 'sqlite',
96
92
  scopeConfigKey: null,
97
93
  hasHistoricSqlReader: false,
98
- hasLocalQueryExecutor: true,
99
94
  load: async () => {
100
95
  const m = await import('../../connectors/sqlite/connector.js');
101
96
  return {
@@ -117,7 +112,6 @@ export const driverRegistrations = {
117
112
  driver: 'snowflake',
118
113
  scopeConfigKey: 'schema_names',
119
114
  hasHistoricSqlReader: true,
120
- hasLocalQueryExecutor: false,
121
115
  load: async () => {
122
116
  const m = await import('../../connectors/snowflake/connector.js');
123
117
  return {
@@ -139,7 +133,6 @@ export const driverRegistrations = {
139
133
  driver: 'sqlserver',
140
134
  scopeConfigKey: 'schemas',
141
135
  hasHistoricSqlReader: false,
142
- hasLocalQueryExecutor: false,
143
136
  load: async () => {
144
137
  const m = await import('../../connectors/sqlserver/connector.js');
145
138
  return {
@@ -6,7 +6,7 @@ export interface KtxSqlQueryExecutionInput {
6
6
  sql: string;
7
7
  maxRows?: number;
8
8
  }
9
- export interface KtxSqlQueryExecutionResult {
9
+ interface KtxSqlQueryExecutionResult {
10
10
  headers: string[];
11
11
  rows: unknown[][];
12
12
  totalRows: number;
@@ -17,3 +17,4 @@ export interface KtxSqlQueryExecutorPort {
17
17
  execute(input: KtxSqlQueryExecutionInput): Promise<KtxSqlQueryExecutionResult>;
18
18
  }
19
19
  export declare function normalizeQueryRows(rows: unknown[]): unknown[][];
20
+ export {};
@@ -0,0 +1,9 @@
1
+ /** @internal */
2
+ export declare function createAbortError(message?: string): DOMException;
3
+ export declare function isAbortError(error: unknown): boolean;
4
+ /** @internal */
5
+ export declare function throwIfAborted(signal?: AbortSignal): void;
6
+ export declare function linkAbortSignal(parent?: AbortSignal): {
7
+ controller: AbortController;
8
+ dispose: () => void;
9
+ };
@@ -0,0 +1,36 @@
1
+ /** @internal */
2
+ export function createAbortError(message = 'Aborted') {
3
+ return new DOMException(message, 'AbortError');
4
+ }
5
+ export function isAbortError(error) {
6
+ if (error instanceof DOMException && error.name === 'AbortError') {
7
+ return true;
8
+ }
9
+ if (!error || typeof error !== 'object') {
10
+ return false;
11
+ }
12
+ const record = error;
13
+ return record.name === 'AbortError' || record.code === 'ABORT_ERR';
14
+ }
15
+ /** @internal */
16
+ export function throwIfAborted(signal) {
17
+ if (signal?.aborted) {
18
+ throw createAbortError();
19
+ }
20
+ }
21
+ export function linkAbortSignal(parent) {
22
+ const controller = new AbortController();
23
+ if (!parent) {
24
+ return { controller, dispose: () => undefined };
25
+ }
26
+ if (parent.aborted) {
27
+ controller.abort(createAbortError());
28
+ return { controller, dispose: () => undefined };
29
+ }
30
+ const onAbort = () => controller.abort(createAbortError());
31
+ parent.addEventListener('abort', onAbort, { once: true });
32
+ return {
33
+ controller,
34
+ dispose: () => parent.removeEventListener('abort', onAbort),
35
+ };
36
+ }
@@ -12,6 +12,7 @@ export interface QueryHistoryFilterProposal {
12
12
  reason: 'no-llm' | 'no-daemon' | 'no-in-scope-history' | 'user-block-present';
13
13
  } | null;
14
14
  warnings: string[];
15
+ parseFailedTemplateIds: string[];
15
16
  }
16
17
  export interface ProposeQueryHistoryServiceAccountFiltersInput {
17
18
  connectionId: string;
@@ -12,7 +12,7 @@ const queryHistoryFilterAdjudicationSchema = z.object({
12
12
  }).strict()),
13
13
  }).strict();
14
14
  function emptyProposal(skipped, warnings = []) {
15
- return { excludedRoles: [], consideredRoleCount: 0, skipped, warnings };
15
+ return { excludedRoles: [], consideredRoleCount: 0, skipped, warnings, parseFailedTemplateIds: [] };
16
16
  }
17
17
  function displayTableRef(ref) {
18
18
  return [ref.catalog, ref.db, ref.name].filter((part) => !!part && part.length > 0).join('.');
@@ -98,6 +98,7 @@ export async function proposeQueryHistoryServiceAccountFilters(input) {
98
98
  const windowDays = 'windowDays' in config ? config.windowDays : 90;
99
99
  const windowStart = new Date(now.getTime() - windowDays * 24 * 60 * 60 * 1000);
100
100
  const warnings = [];
101
+ const parseFailedTemplateIds = [];
101
102
  const snapshot = [];
102
103
  try {
103
104
  for await (const row of input.reader.fetchAggregated(input.queryClient, { start: windowStart, end: now }, config)) {
@@ -127,7 +128,7 @@ export async function proposeQueryHistoryServiceAccountFilters(input) {
127
128
  for (const template of snapshot) {
128
129
  const parsed = analysis.get(template.templateId);
129
130
  if (!parsed || parsed.error) {
130
- warnings.push(`query_history_filter_picker_parse_failed:${template.templateId}`);
131
+ parseFailedTemplateIds.push(template.templateId);
131
132
  continue;
132
133
  }
133
134
  const tablesTouched = [...new Map(parsed.tablesTouched.map((ref) => [tableRefKey(ref), ref])).values()]
@@ -150,6 +151,7 @@ export async function proposeQueryHistoryServiceAccountFilters(input) {
150
151
  consideredRoleCount: records.length,
151
152
  skipped: { reason: 'no-in-scope-history' },
152
153
  warnings,
154
+ parseFailedTemplateIds,
153
155
  };
154
156
  }
155
157
  let generated;
@@ -170,6 +172,7 @@ export async function proposeQueryHistoryServiceAccountFilters(input) {
170
172
  ...warnings,
171
173
  `query_history_filter_picker_llm_failed:${error instanceof Error ? error.message : String(error)}`,
172
174
  ],
175
+ parseFailedTemplateIds,
173
176
  };
174
177
  }
175
178
  const knownRoles = new Set(records.map((record) => record.role));
@@ -186,5 +189,6 @@ export async function proposeQueryHistoryServiceAccountFilters(input) {
186
189
  consideredRoleCount: records.length,
187
190
  skipped: input.userServiceAccountsPresent ? { reason: 'user-block-present' } : null,
188
191
  warnings,
192
+ parseFailedTemplateIds,
189
193
  };
190
194
  }
@@ -31,11 +31,7 @@ export interface CuratorPaginationInput {
31
31
  buildUserPrompt: (input: CuratorPaginationPromptInput) => string;
32
32
  buildToolSet: (passNumber: number) => KtxRuntimeToolSet;
33
33
  getReconciliationActions: () => MemoryAction[];
34
- onStepFinish?: (info: {
35
- passNumber: number;
36
- stepIndex: number;
37
- stepBudget: number;
38
- }) => void;
34
+ abortSignal?: AbortSignal;
39
35
  }
40
36
  interface CuratorPaginationResult extends ReconciliationOutcome {
41
37
  report: CuratorPaginationReport;
@@ -163,9 +163,7 @@ export class CuratorPaginationService {
163
163
  sourceKey: params.input.sourceKey,
164
164
  jobId: params.input.jobId,
165
165
  forceRun: params.forceRun,
166
- onStepFinish: params.input.onStepFinish
167
- ? ({ stepIndex, stepBudget }) => params.input.onStepFinish?.({ passNumber: params.passNumber, stepIndex, stepBudget })
168
- : undefined,
166
+ abortSignal: params.input.abortSignal,
169
167
  });
170
168
  }
171
169
  batchSummary(items) {
@@ -47,7 +47,7 @@ export declare class SqliteContextEvidenceStore implements ContextEvidenceIndexS
47
47
  assertion: string;
48
48
  rationale: string;
49
49
  actionHint: string;
50
- status: "conflict" | "merged" | "pending" | "rejected" | "promoted";
50
+ status: "conflict" | "merged" | "rejected" | "pending" | "promoted";
51
51
  promotionScore: number;
52
52
  suggestedPageKey: string | null;
53
53
  evidenceRefs: JsonValue;
@@ -20,6 +20,7 @@ export interface RepairFinalGateFailureInput {
20
20
  repairKind: FinalGateRepairKind;
21
21
  maxAttempts?: number;
22
22
  stepBudget?: number;
23
+ abortSignal?: AbortSignal;
23
24
  }
24
25
  export declare function finalGateRepairPaths(input: {
25
26
  changedWikiPageKeys: string[];
@@ -153,6 +153,7 @@ export async function repairFinalGateFailure(input) {
153
153
  jobId: input.trace.context.jobId,
154
154
  repairKind: input.repairKind,
155
155
  },
156
+ abortSignal: input.abortSignal,
156
157
  }));
157
158
  if (result.stopReason === 'error') {
158
159
  lastFailure = result.error?.message ?? 'gate repair agent loop errored';
@@ -9,6 +9,9 @@ export declare class IngestBundleRunner {
9
9
  private readonly chainByConnection;
10
10
  constructor(deps: IngestBundleRunnerDeps);
11
11
  run(job: IngestBundleJob, ctx?: IngestJobContext): Promise<IngestBundleResult>;
12
+ private formatRateLimitWait;
13
+ private subscribeRateLimitGovernor;
14
+ private withRateLimitWorkSlot;
12
15
  /**
13
16
  * When profiling is enabled — via the `KTX_PROFILE_INGEST` env var or the
14
17
  * `ingest.profile` config setting — read the job's trace + tool transcripts
@@ -115,6 +115,10 @@ export class IngestBundleRunner {
115
115
  this.logger = deps.logger ?? noopLogger;
116
116
  }
117
117
  async run(job, ctx) {
118
+ const unsubscribeRateLimitGovernor = this.subscribeRateLimitGovernor({
119
+ trace: this.createTrace(job),
120
+ memoryFlow: ctx?.memoryFlow,
121
+ });
118
122
  const key = job.connectionId;
119
123
  const previous = this.chainByConnection.get(key);
120
124
  if (previous) {
@@ -139,9 +143,64 @@ export class IngestBundleRunner {
139
143
  throw error;
140
144
  }
141
145
  finally {
146
+ unsubscribeRateLimitGovernor();
142
147
  await this.maybeEmitIngestProfile(job.jobId);
143
148
  }
144
149
  }
150
+ formatRateLimitWait(state) {
151
+ const seconds = Math.ceil(state.remainingMs / 1_000);
152
+ const minutes = Math.floor(seconds / 60);
153
+ const remainder = seconds % 60;
154
+ const duration = minutes > 0 ? `${minutes}m${String(remainder).padStart(2, '0')}s` : `${seconds}s`;
155
+ const type = state.rateLimitType ? ` ${state.rateLimitType}` : '';
156
+ return `Rate-limited (${state.provider}${type}); resuming in ${duration}; Ctrl+C to stop`;
157
+ }
158
+ subscribeRateLimitGovernor(input) {
159
+ const governor = this.deps.settings.rateLimitGovernor;
160
+ if (!governor) {
161
+ return () => undefined;
162
+ }
163
+ return governor.subscribe((state) => {
164
+ if (state.kind === 'rate_limit_observed') {
165
+ void input.trace.event('info', 'rate_limit', 'rate_limit_observed', { ...state });
166
+ return;
167
+ }
168
+ if (state.kind === 'concurrency_adjusted') {
169
+ void input.trace.event('info', 'rate_limit', 'concurrency_adjusted', { ...state });
170
+ return;
171
+ }
172
+ void input.trace.event('info', 'rate_limit', state.kind, { ...state });
173
+ if (state.kind === 'wait_tick' || state.kind === 'wait_started') {
174
+ input.memoryFlow?.emit({
175
+ type: 'rate_limit_wait',
176
+ provider: state.provider,
177
+ ...(state.rateLimitType ? { rateLimitType: state.rateLimitType } : {}),
178
+ resumeAtMs: state.resumeAtMs,
179
+ remainingMs: state.remainingMs,
180
+ });
181
+ input.memoryFlow?.emit({
182
+ type: 'stage_progress',
183
+ stage: 'integration',
184
+ percent: 50,
185
+ message: this.formatRateLimitWait(state),
186
+ transient: true,
187
+ });
188
+ }
189
+ });
190
+ }
191
+ async withRateLimitWorkSlot(abortSignal, fn) {
192
+ const governor = this.deps.settings.rateLimitGovernor;
193
+ if (!governor) {
194
+ return fn();
195
+ }
196
+ const release = await governor.acquireWorkSlot(abortSignal);
197
+ try {
198
+ return await fn();
199
+ }
200
+ finally {
201
+ release();
202
+ }
203
+ }
145
204
  /**
146
205
  * When profiling is enabled — via the `KTX_PROFILE_INGEST` env var or the
147
206
  * `ingest.profile` config setting — read the job's trace + tool transcripts
@@ -711,7 +770,6 @@ export class IngestBundleRunner {
711
770
  type: 'work_unit_started',
712
771
  unitKey: input.wu.unitKey,
713
772
  skills: input.wuSkillNames,
714
- stepBudget: input.workUnitSettings.stepBudget,
715
773
  });
716
774
  return executeWorkUnit({
717
775
  sessionWorktreeGit: input.worktree.git,
@@ -731,14 +789,30 @@ export class IngestBundleRunner {
731
789
  slIndex: input.slIndex,
732
790
  priorProvenance: input.priorProvenance,
733
791
  }),
734
- buildToolSet: (wuInner) => wrapToolsWithLogger(buildWuToolSet({
735
- sourceKey: input.job.sourceKey,
736
- stagedDir: input.stagedDir,
737
- wu: wuInner,
738
- loadSkillTool,
739
- emitUnmappedFallbackTool: wuEmitUnmappedFallbackTool,
740
- toolsetTools: wuToolset.toRuntimeTools(wuToolContext),
741
- }), join(input.transcriptDir, `${wuInner.unitKey}.jsonl`), wuInner.unitKey, { onEntry: input.recordTranscriptEntry(join(input.transcriptDir, `${wuInner.unitKey}.jsonl`)) }),
792
+ buildToolSet: (wuInner) => {
793
+ const transcriptPath = join(input.transcriptDir, `${wuInner.unitKey}.jsonl`);
794
+ const record = input.recordTranscriptEntry(transcriptPath);
795
+ return wrapToolsWithLogger(buildWuToolSet({
796
+ sourceKey: input.job.sourceKey,
797
+ stagedDir: input.stagedDir,
798
+ wu: wuInner,
799
+ loadSkillTool,
800
+ emitUnmappedFallbackTool: wuEmitUnmappedFallbackTool,
801
+ toolsetTools: wuToolset.toRuntimeTools(wuToolContext),
802
+ }), transcriptPath, wuInner.unitKey, {
803
+ // Drive the live HUD heartbeat from real tool calls: each invocation
804
+ // ticks the running per-unit count. This is an observed signal, not a
805
+ // re-derived turn count, so it can never overshoot a budget.
806
+ onEntry: (entry) => {
807
+ const summary = record(entry);
808
+ input.memoryFlow?.emit({
809
+ type: 'work_unit_step',
810
+ unitKey: wuInner.unitKey,
811
+ toolCalls: summary.toolCallCount,
812
+ });
813
+ },
814
+ });
815
+ },
742
816
  captureSession: session,
743
817
  sessionActions,
744
818
  modelRole: 'candidateExtraction',
@@ -747,7 +821,7 @@ export class IngestBundleRunner {
747
821
  connectionId: input.job.connectionId,
748
822
  jobId: input.job.jobId,
749
823
  toolFailureCount: (unitKey) => input.transcriptSummaries.get(unitKey)?.fatalErrorCount ?? 0,
750
- onStepFinish: input.onStepFinish,
824
+ abortSignal: input.abortSignal,
751
825
  }, input.wu);
752
826
  }
753
827
  async runInner(job, ctx) {
@@ -797,6 +871,7 @@ export class IngestBundleRunner {
797
871
  const current = transcriptSummaries.get(entry.wuKey) ?? createMutableToolTranscriptSummary(entry.wuKey, path);
798
872
  recordToolTranscriptEntry(current, entry);
799
873
  transcriptSummaries.set(entry.wuKey, current);
874
+ return current;
800
875
  };
801
876
  const overrideReport = await this.loadOverrideReport(job);
802
877
  const stage1 = ctx?.startPhase(0.08);
@@ -1111,7 +1186,7 @@ export class IngestBundleRunner {
1111
1186
  await stage3?.updateProgress(1.0, '0 of 0 work units complete');
1112
1187
  }
1113
1188
  try {
1114
- await Promise.all(workUnits.map((wu, index) => limitWorkUnit(async () => {
1189
+ await Promise.all(workUnits.map((wu, index) => limitWorkUnit(() => this.withRateLimitWorkSlot(ctx?.abortSignal, async () => {
1115
1190
  const outcome = await runIsolatedWorkUnit({
1116
1191
  unitIndex: index,
1117
1192
  ingestionBaseSha,
@@ -1119,6 +1194,7 @@ export class IngestBundleRunner {
1119
1194
  patchDir,
1120
1195
  trace: runTrace,
1121
1196
  workUnit: wu,
1197
+ abortSignal: ctx?.abortSignal,
1122
1198
  afterSuccess: (child) => copyTransientIngestEvidence(child.workdir, sessionWorktree.workdir),
1123
1199
  run: async (child) => {
1124
1200
  const scopedWikiService = this.deps.wikiService.forWorktree(child.workdir);
@@ -1147,11 +1223,9 @@ export class IngestBundleRunner {
1147
1223
  stageIndex,
1148
1224
  includeContextEvidenceTools: adapter.evidenceIndexing === 'documents' && !!contextReport,
1149
1225
  currentTableExists: (tableRef) => this.tableRefExistsInSemanticLayer(scopedSemanticLayerService, slConnectionIds, tableRef),
1226
+ abortSignal: ctx?.abortSignal,
1150
1227
  memoryFlow,
1151
1228
  wuSkillNames,
1152
- onStepFinish: ({ stepIndex, stepBudget }) => {
1153
- memoryFlow?.emit({ type: 'work_unit_step', unitKey: wu.unitKey, stepIndex, stepBudget });
1154
- },
1155
1229
  });
1156
1230
  },
1157
1231
  });
@@ -1173,7 +1247,7 @@ export class IngestBundleRunner {
1173
1247
  });
1174
1248
  completedWorkUnits += 1;
1175
1249
  await stage3?.updateProgress(completedWorkUnits / workUnits.length, `${completedWorkUnits} of ${workUnits.length} work units complete`);
1176
- })));
1250
+ }))));
1177
1251
  }
1178
1252
  catch (error) {
1179
1253
  await this.deps.runs.markFailed(runRow.id);
@@ -1250,6 +1324,7 @@ export class IngestBundleRunner {
1250
1324
  reason: context.reason,
1251
1325
  maxAttempts: 1,
1252
1326
  stepBudget: 12,
1327
+ abortSignal: ctx?.abortSignal,
1253
1328
  });
1254
1329
  emitStageProgress('integration', 82, result.status === 'repaired'
1255
1330
  ? `Resolved text conflict for ${context.unitKey}`
@@ -1267,6 +1342,7 @@ export class IngestBundleRunner {
1267
1342
  repairKind: 'patch_semantic_gate',
1268
1343
  maxAttempts: 1,
1269
1344
  stepBudget: 16,
1345
+ abortSignal: ctx?.abortSignal,
1270
1346
  });
1271
1347
  emitStageProgress('integration', 83, result.status === 'repaired'
1272
1348
  ? `Repaired semantic gate for ${context.unitKey}`
@@ -1451,6 +1527,37 @@ export class IngestBundleRunner {
1451
1527
  let curatorReport = null;
1452
1528
  let curatorWarnings = [];
1453
1529
  let reconcileOutcome;
1530
+ // Reconcile shares the work-unit liveness model: the HUD heartbeat is driven
1531
+ // by real tool calls (a monotonic, observed count), not a re-derived turn
1532
+ // counter. The soft cap only paces the phase progress bar; it is never shown
1533
+ // to the user, so it cannot read as a misleading "X/Y" fraction.
1534
+ const reconcileTranscriptPath = join(transcriptDir, 'reconcile.jsonl');
1535
+ const reconcileProgressSoftCap = 40;
1536
+ const buildReconcileToolSetWithHeartbeat = () => {
1537
+ const record = recordTranscriptEntry(reconcileTranscriptPath);
1538
+ return wrapToolsWithLogger(buildReconcileToolSet({
1539
+ loadSkillTool: rcLoadSkill,
1540
+ stageListTool: rcStageListTool,
1541
+ stageDiffTool: rcStageDiffTool,
1542
+ evictionListTool: rcEvictionListTool,
1543
+ emitConflictResolutionTool: rcEmitConflictResolutionTool,
1544
+ emitEvictionDecisionTool: rcEmitEvictionDecisionTool,
1545
+ emitArtifactResolutionTool: rcEmitArtifactResolutionTool,
1546
+ emitUnmappedFallbackTool: rcEmitUnmappedFallbackTool,
1547
+ readRawSpanTool: rcRawSpanTool,
1548
+ toolsetTools: rcToolset.toRuntimeTools(rcToolContext),
1549
+ }), reconcileTranscriptPath, 'reconcile', {
1550
+ onEntry: (entry) => {
1551
+ const summary = record(entry);
1552
+ if (!stage4) {
1553
+ return;
1554
+ }
1555
+ const label = `Reconciling results · ${summary.toolCallCount} action${summary.toolCallCount === 1 ? '' : 's'}`;
1556
+ emitStageProgress('reconciliation', 85, label, { transient: true });
1557
+ void stage4.updateProgress(Math.min(0.95, summary.toolCallCount / reconcileProgressSoftCap), label);
1558
+ },
1559
+ });
1560
+ };
1454
1561
  const reconcileStartedAt = Date.now();
1455
1562
  const reconcileMode = contextReport && this.deps.curatorPagination ? 'curator' : 'single';
1456
1563
  if (contextReport && this.deps.curatorPagination) {
@@ -1471,25 +1578,9 @@ export class IngestBundleRunner {
1471
1578
  canonicalPins: relevantCanonicalPins,
1472
1579
  }),
1473
1580
  buildUserPrompt: ({ summary, items, runState }) => buildReconcileUserPrompt(stageIndex, eviction, { summary, items }, reconcileNotes, runState),
1474
- buildToolSet: (_passNumber) => wrapToolsWithLogger(buildReconcileToolSet({
1475
- loadSkillTool: rcLoadSkill,
1476
- stageListTool: rcStageListTool,
1477
- stageDiffTool: rcStageDiffTool,
1478
- evictionListTool: rcEvictionListTool,
1479
- emitConflictResolutionTool: rcEmitConflictResolutionTool,
1480
- emitEvictionDecisionTool: rcEmitEvictionDecisionTool,
1481
- emitArtifactResolutionTool: rcEmitArtifactResolutionTool,
1482
- emitUnmappedFallbackTool: rcEmitUnmappedFallbackTool,
1483
- readRawSpanTool: rcRawSpanTool,
1484
- toolsetTools: rcToolset.toRuntimeTools(rcToolContext),
1485
- }), join(transcriptDir, 'reconcile.jsonl'), 'reconcile', { onEntry: recordTranscriptEntry(join(transcriptDir, 'reconcile.jsonl')) }),
1581
+ buildToolSet: (_passNumber) => buildReconcileToolSetWithHeartbeat(),
1486
1582
  getReconciliationActions: () => reconcileActions,
1487
- onStepFinish: stage4
1488
- ? ({ passNumber, stepIndex, stepBudget }) => {
1489
- emitStageProgress('reconciliation', 85, `Reconciling results: pass ${passNumber} step ${stepIndex}/${stepBudget}`, { transient: true });
1490
- void stage4.updateProgress(stepIndex / stepBudget, `Reconciling results · pass ${passNumber} step ${stepIndex}`);
1491
- }
1492
- : undefined,
1583
+ abortSignal: ctx?.abortSignal,
1493
1584
  });
1494
1585
  curatorReport = curatorOutcome.report;
1495
1586
  curatorWarnings = curatorOutcome.warnings;
@@ -1512,31 +1603,13 @@ export class IngestBundleRunner {
1512
1603
  canonicalPins: relevantCanonicalPins,
1513
1604
  }),
1514
1605
  buildUserPrompt: (idx, ev) => buildReconcileUserPrompt(idx, ev, undefined, reconcileNotes),
1515
- buildToolSet: () => wrapToolsWithLogger(buildReconcileToolSet({
1516
- loadSkillTool: rcLoadSkill,
1517
- stageListTool: rcStageListTool,
1518
- stageDiffTool: rcStageDiffTool,
1519
- evictionListTool: rcEvictionListTool,
1520
- emitConflictResolutionTool: rcEmitConflictResolutionTool,
1521
- emitEvictionDecisionTool: rcEmitEvictionDecisionTool,
1522
- emitArtifactResolutionTool: rcEmitArtifactResolutionTool,
1523
- emitUnmappedFallbackTool: rcEmitUnmappedFallbackTool,
1524
- readRawSpanTool: rcRawSpanTool,
1525
- toolsetTools: rcToolset.toRuntimeTools(rcToolContext),
1526
- }), join(transcriptDir, 'reconcile.jsonl'), 'reconcile', { onEntry: recordTranscriptEntry(join(transcriptDir, 'reconcile.jsonl')) }),
1606
+ buildToolSet: () => buildReconcileToolSetWithHeartbeat(),
1527
1607
  modelRole: 'reconcile',
1528
1608
  stepBudget: 60,
1529
1609
  sourceKey: job.sourceKey,
1530
1610
  jobId: job.jobId,
1531
1611
  force: !!overrideReport,
1532
- onStepFinish: stage4
1533
- ? ({ stepIndex, stepBudget }) => {
1534
- emitStageProgress('reconciliation', 85, `Reconciling results: step ${stepIndex}/${stepBudget}`, {
1535
- transient: true,
1536
- });
1537
- void stage4.updateProgress(stepIndex / stepBudget, `Reconciling results · step ${stepIndex}`);
1538
- }
1539
- : undefined,
1612
+ abortSignal: ctx?.abortSignal,
1540
1613
  });
1541
1614
  }
1542
1615
  await runTrace.event('debug', 'reconciliation', 'reconciliation_executed', {
@@ -1890,6 +1963,7 @@ export class IngestBundleRunner {
1890
1963
  repairKind: 'final_artifact_gate',
1891
1964
  maxAttempts: 1,
1892
1965
  stepBudget: 16,
1966
+ abortSignal: ctx?.abortSignal,
1893
1967
  });
1894
1968
  isolatedDiffSummary.gateRepairAttempts += gateRepair.attempts;
1895
1969
  if (gateRepair.status === 'failed') {
@@ -19,5 +19,6 @@ export interface ResolveTextualConflictInput {
19
19
  reason: string;
20
20
  maxAttempts?: number;
21
21
  stepBudget?: number;
22
+ abortSignal?: AbortSignal;
22
23
  }
23
24
  export declare function resolveTextualConflict(input: ResolveTextualConflictInput): Promise<TextualConflictResolutionResult>;
@@ -165,6 +165,7 @@ export async function resolveTextualConflict(input) {
165
165
  jobId: input.trace.context.jobId,
166
166
  unitKey: input.unitKey,
167
167
  },
168
+ abortSignal: input.abortSignal,
168
169
  }));
169
170
  if (result.stopReason === 'error') {
170
171
  lastFailure = result.error?.message ?? 'resolver agent loop errored';
@@ -9,6 +9,7 @@ export interface RunIsolatedWorkUnitInput {
9
9
  patchDir: string;
10
10
  trace: IngestTraceWriter;
11
11
  workUnit: WorkUnit;
12
+ abortSignal?: AbortSignal;
12
13
  run(child: IngestSessionWorktree): Promise<WorkUnitOutcome>;
13
14
  afterSuccess?(child: IngestSessionWorktree): Promise<void>;
14
15
  }