@kaelio/ktx 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.9.0-py3-none-any.whl → kaelio_ktx-0.10.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/clack.d.ts +6 -0
- package/dist/clack.js +17 -2
- package/dist/cli-program.d.ts +3 -0
- package/dist/cli-program.js +42 -2
- package/dist/cli-runtime.d.ts +3 -0
- package/dist/cli-runtime.js +44 -0
- package/dist/commands/setup-commands.js +2 -3
- package/dist/connection.js +23 -1
- package/dist/connectors/bigquery/connector.d.ts +2 -5
- package/dist/connectors/bigquery/connector.js +2 -2
- package/dist/connectors/clickhouse/connector.d.ts +2 -5
- package/dist/connectors/clickhouse/connector.js +2 -2
- package/dist/connectors/mysql/connector.d.ts +7 -6
- package/dist/connectors/mysql/connector.js +25 -5
- package/dist/connectors/mysql/dialect.d.ts +1 -1
- package/dist/connectors/mysql/dialect.js +12 -2
- package/dist/connectors/postgres/connector.d.ts +2 -5
- package/dist/connectors/postgres/connector.js +2 -2
- package/dist/connectors/snowflake/connector.d.ts +2 -5
- package/dist/connectors/snowflake/connector.js +2 -2
- package/dist/connectors/sqlite/connector.d.ts +2 -5
- package/dist/connectors/sqlite/connector.js +2 -2
- package/dist/connectors/sqlserver/connector.d.ts +2 -5
- package/dist/connectors/sqlserver/connector.js +2 -2
- package/dist/context/connections/drivers.d.ts +0 -1
- package/dist/context/connections/drivers.js +0 -7
- package/dist/context/connections/query-executor.d.ts +2 -1
- package/dist/context/core/abort.d.ts +9 -0
- package/dist/context/core/abort.js +36 -0
- package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.d.ts +1 -0
- package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.js +6 -2
- package/dist/context/ingest/context-candidates/curator-pagination.service.d.ts +1 -5
- package/dist/context/ingest/context-candidates/curator-pagination.service.js +1 -3
- package/dist/context/ingest/context-evidence/sqlite-context-evidence-store.d.ts +1 -1
- package/dist/context/ingest/final-gate-repair.d.ts +1 -0
- package/dist/context/ingest/final-gate-repair.js +1 -0
- package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
- package/dist/context/ingest/ingest-bundle.runner.js +127 -53
- package/dist/context/ingest/isolated-diff/textual-conflict-resolver.d.ts +1 -0
- package/dist/context/ingest/isolated-diff/textual-conflict-resolver.js +1 -0
- package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
- package/dist/context/ingest/local-bundle-runtime.js +11 -4
- package/dist/context/ingest/local-ingest.d.ts +1 -0
- package/dist/context/ingest/local-ingest.js +13 -3
- package/dist/context/ingest/memory-flow/events.js +1 -1
- package/dist/context/ingest/memory-flow/schema.js +8 -3
- package/dist/context/ingest/memory-flow/types.d.ts +7 -3
- package/dist/context/ingest/ports.d.ts +3 -5
- package/dist/context/ingest/stages/stage-3-work-units.d.ts +1 -4
- package/dist/context/ingest/stages/stage-3-work-units.js +5 -1
- package/dist/context/ingest/stages/stage-4-reconciliation.d.ts +1 -4
- package/dist/context/ingest/stages/stage-4-reconciliation.js +1 -1
- package/dist/context/ingest/types.d.ts +1 -0
- package/dist/context/llm/ai-sdk-runtime.d.ts +3 -0
- package/dist/context/llm/ai-sdk-runtime.js +152 -16
- package/dist/context/llm/claude-code-runtime.d.ts +6 -4
- package/dist/context/llm/claude-code-runtime.js +127 -48
- package/dist/context/llm/codex-runtime.d.ts +3 -3
- package/dist/context/llm/codex-runtime.js +90 -47
- package/dist/context/llm/local-config.d.ts +15 -5
- package/dist/context/llm/local-config.js +6 -1
- package/dist/context/llm/rate-limit-governor.d.ts +103 -0
- package/dist/context/llm/rate-limit-governor.js +285 -0
- package/dist/context/llm/runtime-port.d.ts +3 -6
- package/dist/context/mcp/context-tools.js +43 -13
- package/dist/context/project/config.d.ts +12 -0
- package/dist/context/project/config.js +35 -0
- package/dist/context/scan/types.d.ts +15 -2
- package/dist/context/scan/types.js +12 -0
- package/dist/context/sl/description-normalization.js +4 -14
- package/dist/context/tools/context-candidate-mark.tool.d.ts +2 -2
- package/dist/context-build-view.d.ts +13 -0
- package/dist/context-build-view.js +60 -1
- package/dist/demo-metrics.d.ts +0 -2
- package/dist/demo-metrics.js +1 -11
- package/dist/ingest.d.ts +1 -0
- package/dist/ingest.js +32 -3
- package/dist/io/symbols.d.ts +2 -0
- package/dist/io/symbols.js +2 -0
- package/dist/memory-flow-hud.js +8 -16
- package/dist/public-ingest.js +50 -15
- package/dist/reveal-password-prompt.d.ts +24 -0
- package/dist/reveal-password-prompt.js +78 -0
- package/dist/scan.js +18 -2
- package/dist/setup-databases.d.ts +1 -0
- package/dist/setup-databases.js +23 -3
- package/dist/setup-demo-tour.js +1 -0
- package/dist/setup-embeddings.js +1 -1
- package/dist/setup-models.d.ts +1 -14
- package/dist/setup-models.js +116 -340
- package/dist/setup-prompts.js +3 -2
- package/dist/setup-sources.js +7 -7
- package/dist/setup.d.ts +1 -1
- package/dist/setup.js +1 -1
- package/dist/sl.d.ts +2 -2
- package/dist/sl.js +20 -4
- package/dist/sql.js +18 -2
- package/dist/star-prompt/cache.d.ts +16 -0
- package/dist/star-prompt/cache.js +45 -0
- package/dist/star-prompt/star-count.d.ts +7 -0
- package/dist/star-prompt/star-count.js +66 -0
- package/dist/star-prompt/star-line.d.ts +12 -0
- package/dist/star-prompt/star-line.js +26 -0
- package/dist/telemetry/emitter.d.ts +10 -0
- package/dist/telemetry/emitter.js +31 -0
- package/dist/telemetry/events.d.ts +24 -0
- package/dist/telemetry/events.js +15 -0
- package/dist/telemetry/exception.d.ts +18 -0
- package/dist/telemetry/exception.js +162 -0
- package/dist/telemetry/index.d.ts +3 -2
- package/dist/telemetry/index.js +2 -1
- package/dist/telemetry/redaction-secrets.d.ts +11 -0
- package/dist/telemetry/redaction-secrets.js +92 -0
- package/dist/update-check/cache.d.ts +21 -0
- package/dist/update-check/cache.js +38 -0
- package/dist/update-check/channel.d.ts +15 -0
- package/dist/update-check/channel.js +30 -0
- package/dist/update-check/registry.d.ts +1 -0
- package/dist/update-check/registry.js +45 -0
- package/dist/update-check/update-check.d.ts +43 -0
- package/dist/update-check/update-check.js +116 -0
- package/package.json +8 -1
- package/dist/context/connections/local-query-executor.d.ts +0 -6
- package/dist/context/connections/local-query-executor.js +0 -39
- package/dist/context/connections/postgres-query-executor.d.ts +0 -25
- package/dist/context/connections/postgres-query-executor.js +0 -53
- package/dist/context/connections/sqlite-query-executor.d.ts +0 -4
- package/dist/context/connections/sqlite-query-executor.js +0 -74
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaSnapshot, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
|
|
1
|
+
import { type KtxConnectorTestResult, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaSnapshot, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
|
|
2
2
|
export interface KtxSqlServerConnectionConfig {
|
|
3
3
|
driver?: string;
|
|
4
4
|
host?: string;
|
|
@@ -113,10 +113,7 @@ export declare class KtxSqlServerScanConnector implements KtxScanConnector {
|
|
|
113
113
|
private pool;
|
|
114
114
|
private resolvedEndpoint;
|
|
115
115
|
constructor(options: KtxSqlServerScanConnectorOptions);
|
|
116
|
-
testConnection(): Promise<
|
|
117
|
-
success: boolean;
|
|
118
|
-
error?: string;
|
|
119
|
-
}>;
|
|
116
|
+
testConnection(): Promise<KtxConnectorTestResult>;
|
|
120
117
|
introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise<KtxSchemaSnapshot>;
|
|
121
118
|
sampleTable(input: KtxTableSampleInput, _ctx: KtxScanContext): Promise<KtxSqlServerTableSampleResult>;
|
|
122
119
|
sampleColumn(input: KtxColumnSampleInput, _ctx: KtxScanContext): Promise<KtxColumnSampleResult>;
|
|
@@ -2,7 +2,7 @@ import { assertReadOnlySql } from '../../context/connections/read-only-sql.js';
|
|
|
2
2
|
import { getDialectForDriver } from '../../context/connections/dialects.js';
|
|
3
3
|
import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
|
|
4
4
|
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
|
5
|
-
import { createKtxConnectorCapabilities, } from '../../context/scan/types.js';
|
|
5
|
+
import { connectorTestFailure, createKtxConnectorCapabilities, } from '../../context/scan/types.js';
|
|
6
6
|
import { readFileSync } from 'node:fs';
|
|
7
7
|
import { homedir } from 'node:os';
|
|
8
8
|
import { resolve } from 'node:path';
|
|
@@ -235,7 +235,7 @@ export class KtxSqlServerScanConnector {
|
|
|
235
235
|
return { success: true };
|
|
236
236
|
}
|
|
237
237
|
catch (error) {
|
|
238
|
-
return
|
|
238
|
+
return connectorTestFailure(error);
|
|
239
239
|
}
|
|
240
240
|
}
|
|
241
241
|
async introspect(input, _ctx) {
|
|
@@ -14,7 +14,6 @@ export interface KtxDriverRegistration {
|
|
|
14
14
|
readonly driver: KtxConnectionDriver;
|
|
15
15
|
readonly scopeConfigKey: KtxScopeConfigKey | null;
|
|
16
16
|
readonly hasHistoricSqlReader: boolean;
|
|
17
|
-
readonly hasLocalQueryExecutor: boolean;
|
|
18
17
|
load(): Promise<KtxDriverConnectorModule>;
|
|
19
18
|
}
|
|
20
19
|
/** @internal */
|
|
@@ -7,7 +7,6 @@ export const driverRegistrations = {
|
|
|
7
7
|
driver: 'bigquery',
|
|
8
8
|
scopeConfigKey: 'dataset_ids',
|
|
9
9
|
hasHistoricSqlReader: true,
|
|
10
|
-
hasLocalQueryExecutor: false,
|
|
11
10
|
load: async () => {
|
|
12
11
|
const m = await import('../../connectors/bigquery/connector.js');
|
|
13
12
|
return {
|
|
@@ -29,7 +28,6 @@ export const driverRegistrations = {
|
|
|
29
28
|
driver: 'clickhouse',
|
|
30
29
|
scopeConfigKey: 'databases',
|
|
31
30
|
hasHistoricSqlReader: false,
|
|
32
|
-
hasLocalQueryExecutor: false,
|
|
33
31
|
load: async () => {
|
|
34
32
|
const m = await import('../../connectors/clickhouse/connector.js');
|
|
35
33
|
return {
|
|
@@ -51,7 +49,6 @@ export const driverRegistrations = {
|
|
|
51
49
|
driver: 'mysql',
|
|
52
50
|
scopeConfigKey: 'schemas',
|
|
53
51
|
hasHistoricSqlReader: false,
|
|
54
|
-
hasLocalQueryExecutor: false,
|
|
55
52
|
load: async () => {
|
|
56
53
|
const m = await import('../../connectors/mysql/connector.js');
|
|
57
54
|
return {
|
|
@@ -73,7 +70,6 @@ export const driverRegistrations = {
|
|
|
73
70
|
driver: 'postgres',
|
|
74
71
|
scopeConfigKey: 'schemas',
|
|
75
72
|
hasHistoricSqlReader: true,
|
|
76
|
-
hasLocalQueryExecutor: true,
|
|
77
73
|
load: async () => {
|
|
78
74
|
const m = await import('../../connectors/postgres/connector.js');
|
|
79
75
|
return {
|
|
@@ -95,7 +91,6 @@ export const driverRegistrations = {
|
|
|
95
91
|
driver: 'sqlite',
|
|
96
92
|
scopeConfigKey: null,
|
|
97
93
|
hasHistoricSqlReader: false,
|
|
98
|
-
hasLocalQueryExecutor: true,
|
|
99
94
|
load: async () => {
|
|
100
95
|
const m = await import('../../connectors/sqlite/connector.js');
|
|
101
96
|
return {
|
|
@@ -117,7 +112,6 @@ export const driverRegistrations = {
|
|
|
117
112
|
driver: 'snowflake',
|
|
118
113
|
scopeConfigKey: 'schema_names',
|
|
119
114
|
hasHistoricSqlReader: true,
|
|
120
|
-
hasLocalQueryExecutor: false,
|
|
121
115
|
load: async () => {
|
|
122
116
|
const m = await import('../../connectors/snowflake/connector.js');
|
|
123
117
|
return {
|
|
@@ -139,7 +133,6 @@ export const driverRegistrations = {
|
|
|
139
133
|
driver: 'sqlserver',
|
|
140
134
|
scopeConfigKey: 'schemas',
|
|
141
135
|
hasHistoricSqlReader: false,
|
|
142
|
-
hasLocalQueryExecutor: false,
|
|
143
136
|
load: async () => {
|
|
144
137
|
const m = await import('../../connectors/sqlserver/connector.js');
|
|
145
138
|
return {
|
|
@@ -6,7 +6,7 @@ export interface KtxSqlQueryExecutionInput {
|
|
|
6
6
|
sql: string;
|
|
7
7
|
maxRows?: number;
|
|
8
8
|
}
|
|
9
|
-
|
|
9
|
+
interface KtxSqlQueryExecutionResult {
|
|
10
10
|
headers: string[];
|
|
11
11
|
rows: unknown[][];
|
|
12
12
|
totalRows: number;
|
|
@@ -17,3 +17,4 @@ export interface KtxSqlQueryExecutorPort {
|
|
|
17
17
|
execute(input: KtxSqlQueryExecutionInput): Promise<KtxSqlQueryExecutionResult>;
|
|
18
18
|
}
|
|
19
19
|
export declare function normalizeQueryRows(rows: unknown[]): unknown[][];
|
|
20
|
+
export {};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** @internal */
|
|
2
|
+
export declare function createAbortError(message?: string): DOMException;
|
|
3
|
+
export declare function isAbortError(error: unknown): boolean;
|
|
4
|
+
/** @internal */
|
|
5
|
+
export declare function throwIfAborted(signal?: AbortSignal): void;
|
|
6
|
+
export declare function linkAbortSignal(parent?: AbortSignal): {
|
|
7
|
+
controller: AbortController;
|
|
8
|
+
dispose: () => void;
|
|
9
|
+
};
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/** @internal */
|
|
2
|
+
export function createAbortError(message = 'Aborted') {
|
|
3
|
+
return new DOMException(message, 'AbortError');
|
|
4
|
+
}
|
|
5
|
+
export function isAbortError(error) {
|
|
6
|
+
if (error instanceof DOMException && error.name === 'AbortError') {
|
|
7
|
+
return true;
|
|
8
|
+
}
|
|
9
|
+
if (!error || typeof error !== 'object') {
|
|
10
|
+
return false;
|
|
11
|
+
}
|
|
12
|
+
const record = error;
|
|
13
|
+
return record.name === 'AbortError' || record.code === 'ABORT_ERR';
|
|
14
|
+
}
|
|
15
|
+
/** @internal */
|
|
16
|
+
export function throwIfAborted(signal) {
|
|
17
|
+
if (signal?.aborted) {
|
|
18
|
+
throw createAbortError();
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
export function linkAbortSignal(parent) {
|
|
22
|
+
const controller = new AbortController();
|
|
23
|
+
if (!parent) {
|
|
24
|
+
return { controller, dispose: () => undefined };
|
|
25
|
+
}
|
|
26
|
+
if (parent.aborted) {
|
|
27
|
+
controller.abort(createAbortError());
|
|
28
|
+
return { controller, dispose: () => undefined };
|
|
29
|
+
}
|
|
30
|
+
const onAbort = () => controller.abort(createAbortError());
|
|
31
|
+
parent.addEventListener('abort', onAbort, { once: true });
|
|
32
|
+
return {
|
|
33
|
+
controller,
|
|
34
|
+
dispose: () => parent.removeEventListener('abort', onAbort),
|
|
35
|
+
};
|
|
36
|
+
}
|
|
@@ -12,6 +12,7 @@ export interface QueryHistoryFilterProposal {
|
|
|
12
12
|
reason: 'no-llm' | 'no-daemon' | 'no-in-scope-history' | 'user-block-present';
|
|
13
13
|
} | null;
|
|
14
14
|
warnings: string[];
|
|
15
|
+
parseFailedTemplateIds: string[];
|
|
15
16
|
}
|
|
16
17
|
export interface ProposeQueryHistoryServiceAccountFiltersInput {
|
|
17
18
|
connectionId: string;
|
|
@@ -12,7 +12,7 @@ const queryHistoryFilterAdjudicationSchema = z.object({
|
|
|
12
12
|
}).strict()),
|
|
13
13
|
}).strict();
|
|
14
14
|
function emptyProposal(skipped, warnings = []) {
|
|
15
|
-
return { excludedRoles: [], consideredRoleCount: 0, skipped, warnings };
|
|
15
|
+
return { excludedRoles: [], consideredRoleCount: 0, skipped, warnings, parseFailedTemplateIds: [] };
|
|
16
16
|
}
|
|
17
17
|
function displayTableRef(ref) {
|
|
18
18
|
return [ref.catalog, ref.db, ref.name].filter((part) => !!part && part.length > 0).join('.');
|
|
@@ -98,6 +98,7 @@ export async function proposeQueryHistoryServiceAccountFilters(input) {
|
|
|
98
98
|
const windowDays = 'windowDays' in config ? config.windowDays : 90;
|
|
99
99
|
const windowStart = new Date(now.getTime() - windowDays * 24 * 60 * 60 * 1000);
|
|
100
100
|
const warnings = [];
|
|
101
|
+
const parseFailedTemplateIds = [];
|
|
101
102
|
const snapshot = [];
|
|
102
103
|
try {
|
|
103
104
|
for await (const row of input.reader.fetchAggregated(input.queryClient, { start: windowStart, end: now }, config)) {
|
|
@@ -127,7 +128,7 @@ export async function proposeQueryHistoryServiceAccountFilters(input) {
|
|
|
127
128
|
for (const template of snapshot) {
|
|
128
129
|
const parsed = analysis.get(template.templateId);
|
|
129
130
|
if (!parsed || parsed.error) {
|
|
130
|
-
|
|
131
|
+
parseFailedTemplateIds.push(template.templateId);
|
|
131
132
|
continue;
|
|
132
133
|
}
|
|
133
134
|
const tablesTouched = [...new Map(parsed.tablesTouched.map((ref) => [tableRefKey(ref), ref])).values()]
|
|
@@ -150,6 +151,7 @@ export async function proposeQueryHistoryServiceAccountFilters(input) {
|
|
|
150
151
|
consideredRoleCount: records.length,
|
|
151
152
|
skipped: { reason: 'no-in-scope-history' },
|
|
152
153
|
warnings,
|
|
154
|
+
parseFailedTemplateIds,
|
|
153
155
|
};
|
|
154
156
|
}
|
|
155
157
|
let generated;
|
|
@@ -170,6 +172,7 @@ export async function proposeQueryHistoryServiceAccountFilters(input) {
|
|
|
170
172
|
...warnings,
|
|
171
173
|
`query_history_filter_picker_llm_failed:${error instanceof Error ? error.message : String(error)}`,
|
|
172
174
|
],
|
|
175
|
+
parseFailedTemplateIds,
|
|
173
176
|
};
|
|
174
177
|
}
|
|
175
178
|
const knownRoles = new Set(records.map((record) => record.role));
|
|
@@ -186,5 +189,6 @@ export async function proposeQueryHistoryServiceAccountFilters(input) {
|
|
|
186
189
|
consideredRoleCount: records.length,
|
|
187
190
|
skipped: input.userServiceAccountsPresent ? { reason: 'user-block-present' } : null,
|
|
188
191
|
warnings,
|
|
192
|
+
parseFailedTemplateIds,
|
|
189
193
|
};
|
|
190
194
|
}
|
|
@@ -31,11 +31,7 @@ export interface CuratorPaginationInput {
|
|
|
31
31
|
buildUserPrompt: (input: CuratorPaginationPromptInput) => string;
|
|
32
32
|
buildToolSet: (passNumber: number) => KtxRuntimeToolSet;
|
|
33
33
|
getReconciliationActions: () => MemoryAction[];
|
|
34
|
-
|
|
35
|
-
passNumber: number;
|
|
36
|
-
stepIndex: number;
|
|
37
|
-
stepBudget: number;
|
|
38
|
-
}) => void;
|
|
34
|
+
abortSignal?: AbortSignal;
|
|
39
35
|
}
|
|
40
36
|
interface CuratorPaginationResult extends ReconciliationOutcome {
|
|
41
37
|
report: CuratorPaginationReport;
|
|
@@ -163,9 +163,7 @@ export class CuratorPaginationService {
|
|
|
163
163
|
sourceKey: params.input.sourceKey,
|
|
164
164
|
jobId: params.input.jobId,
|
|
165
165
|
forceRun: params.forceRun,
|
|
166
|
-
|
|
167
|
-
? ({ stepIndex, stepBudget }) => params.input.onStepFinish?.({ passNumber: params.passNumber, stepIndex, stepBudget })
|
|
168
|
-
: undefined,
|
|
166
|
+
abortSignal: params.input.abortSignal,
|
|
169
167
|
});
|
|
170
168
|
}
|
|
171
169
|
batchSummary(items) {
|
|
@@ -47,7 +47,7 @@ export declare class SqliteContextEvidenceStore implements ContextEvidenceIndexS
|
|
|
47
47
|
assertion: string;
|
|
48
48
|
rationale: string;
|
|
49
49
|
actionHint: string;
|
|
50
|
-
status: "conflict" | "merged" | "
|
|
50
|
+
status: "conflict" | "merged" | "rejected" | "pending" | "promoted";
|
|
51
51
|
promotionScore: number;
|
|
52
52
|
suggestedPageKey: string | null;
|
|
53
53
|
evidenceRefs: JsonValue;
|
|
@@ -153,6 +153,7 @@ export async function repairFinalGateFailure(input) {
|
|
|
153
153
|
jobId: input.trace.context.jobId,
|
|
154
154
|
repairKind: input.repairKind,
|
|
155
155
|
},
|
|
156
|
+
abortSignal: input.abortSignal,
|
|
156
157
|
}));
|
|
157
158
|
if (result.stopReason === 'error') {
|
|
158
159
|
lastFailure = result.error?.message ?? 'gate repair agent loop errored';
|
|
@@ -9,6 +9,9 @@ export declare class IngestBundleRunner {
|
|
|
9
9
|
private readonly chainByConnection;
|
|
10
10
|
constructor(deps: IngestBundleRunnerDeps);
|
|
11
11
|
run(job: IngestBundleJob, ctx?: IngestJobContext): Promise<IngestBundleResult>;
|
|
12
|
+
private formatRateLimitWait;
|
|
13
|
+
private subscribeRateLimitGovernor;
|
|
14
|
+
private withRateLimitWorkSlot;
|
|
12
15
|
/**
|
|
13
16
|
* When profiling is enabled — via the `KTX_PROFILE_INGEST` env var or the
|
|
14
17
|
* `ingest.profile` config setting — read the job's trace + tool transcripts
|
|
@@ -115,6 +115,10 @@ export class IngestBundleRunner {
|
|
|
115
115
|
this.logger = deps.logger ?? noopLogger;
|
|
116
116
|
}
|
|
117
117
|
async run(job, ctx) {
|
|
118
|
+
const unsubscribeRateLimitGovernor = this.subscribeRateLimitGovernor({
|
|
119
|
+
trace: this.createTrace(job),
|
|
120
|
+
memoryFlow: ctx?.memoryFlow,
|
|
121
|
+
});
|
|
118
122
|
const key = job.connectionId;
|
|
119
123
|
const previous = this.chainByConnection.get(key);
|
|
120
124
|
if (previous) {
|
|
@@ -139,9 +143,64 @@ export class IngestBundleRunner {
|
|
|
139
143
|
throw error;
|
|
140
144
|
}
|
|
141
145
|
finally {
|
|
146
|
+
unsubscribeRateLimitGovernor();
|
|
142
147
|
await this.maybeEmitIngestProfile(job.jobId);
|
|
143
148
|
}
|
|
144
149
|
}
|
|
150
|
+
formatRateLimitWait(state) {
|
|
151
|
+
const seconds = Math.ceil(state.remainingMs / 1_000);
|
|
152
|
+
const minutes = Math.floor(seconds / 60);
|
|
153
|
+
const remainder = seconds % 60;
|
|
154
|
+
const duration = minutes > 0 ? `${minutes}m${String(remainder).padStart(2, '0')}s` : `${seconds}s`;
|
|
155
|
+
const type = state.rateLimitType ? ` ${state.rateLimitType}` : '';
|
|
156
|
+
return `Rate-limited (${state.provider}${type}); resuming in ${duration}; Ctrl+C to stop`;
|
|
157
|
+
}
|
|
158
|
+
subscribeRateLimitGovernor(input) {
|
|
159
|
+
const governor = this.deps.settings.rateLimitGovernor;
|
|
160
|
+
if (!governor) {
|
|
161
|
+
return () => undefined;
|
|
162
|
+
}
|
|
163
|
+
return governor.subscribe((state) => {
|
|
164
|
+
if (state.kind === 'rate_limit_observed') {
|
|
165
|
+
void input.trace.event('info', 'rate_limit', 'rate_limit_observed', { ...state });
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
if (state.kind === 'concurrency_adjusted') {
|
|
169
|
+
void input.trace.event('info', 'rate_limit', 'concurrency_adjusted', { ...state });
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
void input.trace.event('info', 'rate_limit', state.kind, { ...state });
|
|
173
|
+
if (state.kind === 'wait_tick' || state.kind === 'wait_started') {
|
|
174
|
+
input.memoryFlow?.emit({
|
|
175
|
+
type: 'rate_limit_wait',
|
|
176
|
+
provider: state.provider,
|
|
177
|
+
...(state.rateLimitType ? { rateLimitType: state.rateLimitType } : {}),
|
|
178
|
+
resumeAtMs: state.resumeAtMs,
|
|
179
|
+
remainingMs: state.remainingMs,
|
|
180
|
+
});
|
|
181
|
+
input.memoryFlow?.emit({
|
|
182
|
+
type: 'stage_progress',
|
|
183
|
+
stage: 'integration',
|
|
184
|
+
percent: 50,
|
|
185
|
+
message: this.formatRateLimitWait(state),
|
|
186
|
+
transient: true,
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
async withRateLimitWorkSlot(abortSignal, fn) {
|
|
192
|
+
const governor = this.deps.settings.rateLimitGovernor;
|
|
193
|
+
if (!governor) {
|
|
194
|
+
return fn();
|
|
195
|
+
}
|
|
196
|
+
const release = await governor.acquireWorkSlot(abortSignal);
|
|
197
|
+
try {
|
|
198
|
+
return await fn();
|
|
199
|
+
}
|
|
200
|
+
finally {
|
|
201
|
+
release();
|
|
202
|
+
}
|
|
203
|
+
}
|
|
145
204
|
/**
|
|
146
205
|
* When profiling is enabled — via the `KTX_PROFILE_INGEST` env var or the
|
|
147
206
|
* `ingest.profile` config setting — read the job's trace + tool transcripts
|
|
@@ -711,7 +770,6 @@ export class IngestBundleRunner {
|
|
|
711
770
|
type: 'work_unit_started',
|
|
712
771
|
unitKey: input.wu.unitKey,
|
|
713
772
|
skills: input.wuSkillNames,
|
|
714
|
-
stepBudget: input.workUnitSettings.stepBudget,
|
|
715
773
|
});
|
|
716
774
|
return executeWorkUnit({
|
|
717
775
|
sessionWorktreeGit: input.worktree.git,
|
|
@@ -731,14 +789,30 @@ export class IngestBundleRunner {
|
|
|
731
789
|
slIndex: input.slIndex,
|
|
732
790
|
priorProvenance: input.priorProvenance,
|
|
733
791
|
}),
|
|
734
|
-
buildToolSet: (wuInner) =>
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
792
|
+
buildToolSet: (wuInner) => {
|
|
793
|
+
const transcriptPath = join(input.transcriptDir, `${wuInner.unitKey}.jsonl`);
|
|
794
|
+
const record = input.recordTranscriptEntry(transcriptPath);
|
|
795
|
+
return wrapToolsWithLogger(buildWuToolSet({
|
|
796
|
+
sourceKey: input.job.sourceKey,
|
|
797
|
+
stagedDir: input.stagedDir,
|
|
798
|
+
wu: wuInner,
|
|
799
|
+
loadSkillTool,
|
|
800
|
+
emitUnmappedFallbackTool: wuEmitUnmappedFallbackTool,
|
|
801
|
+
toolsetTools: wuToolset.toRuntimeTools(wuToolContext),
|
|
802
|
+
}), transcriptPath, wuInner.unitKey, {
|
|
803
|
+
// Drive the live HUD heartbeat from real tool calls: each invocation
|
|
804
|
+
// ticks the running per-unit count. This is an observed signal, not a
|
|
805
|
+
// re-derived turn count, so it can never overshoot a budget.
|
|
806
|
+
onEntry: (entry) => {
|
|
807
|
+
const summary = record(entry);
|
|
808
|
+
input.memoryFlow?.emit({
|
|
809
|
+
type: 'work_unit_step',
|
|
810
|
+
unitKey: wuInner.unitKey,
|
|
811
|
+
toolCalls: summary.toolCallCount,
|
|
812
|
+
});
|
|
813
|
+
},
|
|
814
|
+
});
|
|
815
|
+
},
|
|
742
816
|
captureSession: session,
|
|
743
817
|
sessionActions,
|
|
744
818
|
modelRole: 'candidateExtraction',
|
|
@@ -747,7 +821,7 @@ export class IngestBundleRunner {
|
|
|
747
821
|
connectionId: input.job.connectionId,
|
|
748
822
|
jobId: input.job.jobId,
|
|
749
823
|
toolFailureCount: (unitKey) => input.transcriptSummaries.get(unitKey)?.fatalErrorCount ?? 0,
|
|
750
|
-
|
|
824
|
+
abortSignal: input.abortSignal,
|
|
751
825
|
}, input.wu);
|
|
752
826
|
}
|
|
753
827
|
async runInner(job, ctx) {
|
|
@@ -797,6 +871,7 @@ export class IngestBundleRunner {
|
|
|
797
871
|
const current = transcriptSummaries.get(entry.wuKey) ?? createMutableToolTranscriptSummary(entry.wuKey, path);
|
|
798
872
|
recordToolTranscriptEntry(current, entry);
|
|
799
873
|
transcriptSummaries.set(entry.wuKey, current);
|
|
874
|
+
return current;
|
|
800
875
|
};
|
|
801
876
|
const overrideReport = await this.loadOverrideReport(job);
|
|
802
877
|
const stage1 = ctx?.startPhase(0.08);
|
|
@@ -1111,7 +1186,7 @@ export class IngestBundleRunner {
|
|
|
1111
1186
|
await stage3?.updateProgress(1.0, '0 of 0 work units complete');
|
|
1112
1187
|
}
|
|
1113
1188
|
try {
|
|
1114
|
-
await Promise.all(workUnits.map((wu, index) => limitWorkUnit(async () => {
|
|
1189
|
+
await Promise.all(workUnits.map((wu, index) => limitWorkUnit(() => this.withRateLimitWorkSlot(ctx?.abortSignal, async () => {
|
|
1115
1190
|
const outcome = await runIsolatedWorkUnit({
|
|
1116
1191
|
unitIndex: index,
|
|
1117
1192
|
ingestionBaseSha,
|
|
@@ -1119,6 +1194,7 @@ export class IngestBundleRunner {
|
|
|
1119
1194
|
patchDir,
|
|
1120
1195
|
trace: runTrace,
|
|
1121
1196
|
workUnit: wu,
|
|
1197
|
+
abortSignal: ctx?.abortSignal,
|
|
1122
1198
|
afterSuccess: (child) => copyTransientIngestEvidence(child.workdir, sessionWorktree.workdir),
|
|
1123
1199
|
run: async (child) => {
|
|
1124
1200
|
const scopedWikiService = this.deps.wikiService.forWorktree(child.workdir);
|
|
@@ -1147,11 +1223,9 @@ export class IngestBundleRunner {
|
|
|
1147
1223
|
stageIndex,
|
|
1148
1224
|
includeContextEvidenceTools: adapter.evidenceIndexing === 'documents' && !!contextReport,
|
|
1149
1225
|
currentTableExists: (tableRef) => this.tableRefExistsInSemanticLayer(scopedSemanticLayerService, slConnectionIds, tableRef),
|
|
1226
|
+
abortSignal: ctx?.abortSignal,
|
|
1150
1227
|
memoryFlow,
|
|
1151
1228
|
wuSkillNames,
|
|
1152
|
-
onStepFinish: ({ stepIndex, stepBudget }) => {
|
|
1153
|
-
memoryFlow?.emit({ type: 'work_unit_step', unitKey: wu.unitKey, stepIndex, stepBudget });
|
|
1154
|
-
},
|
|
1155
1229
|
});
|
|
1156
1230
|
},
|
|
1157
1231
|
});
|
|
@@ -1173,7 +1247,7 @@ export class IngestBundleRunner {
|
|
|
1173
1247
|
});
|
|
1174
1248
|
completedWorkUnits += 1;
|
|
1175
1249
|
await stage3?.updateProgress(completedWorkUnits / workUnits.length, `${completedWorkUnits} of ${workUnits.length} work units complete`);
|
|
1176
|
-
})));
|
|
1250
|
+
}))));
|
|
1177
1251
|
}
|
|
1178
1252
|
catch (error) {
|
|
1179
1253
|
await this.deps.runs.markFailed(runRow.id);
|
|
@@ -1250,6 +1324,7 @@ export class IngestBundleRunner {
|
|
|
1250
1324
|
reason: context.reason,
|
|
1251
1325
|
maxAttempts: 1,
|
|
1252
1326
|
stepBudget: 12,
|
|
1327
|
+
abortSignal: ctx?.abortSignal,
|
|
1253
1328
|
});
|
|
1254
1329
|
emitStageProgress('integration', 82, result.status === 'repaired'
|
|
1255
1330
|
? `Resolved text conflict for ${context.unitKey}`
|
|
@@ -1267,6 +1342,7 @@ export class IngestBundleRunner {
|
|
|
1267
1342
|
repairKind: 'patch_semantic_gate',
|
|
1268
1343
|
maxAttempts: 1,
|
|
1269
1344
|
stepBudget: 16,
|
|
1345
|
+
abortSignal: ctx?.abortSignal,
|
|
1270
1346
|
});
|
|
1271
1347
|
emitStageProgress('integration', 83, result.status === 'repaired'
|
|
1272
1348
|
? `Repaired semantic gate for ${context.unitKey}`
|
|
@@ -1451,6 +1527,37 @@ export class IngestBundleRunner {
|
|
|
1451
1527
|
let curatorReport = null;
|
|
1452
1528
|
let curatorWarnings = [];
|
|
1453
1529
|
let reconcileOutcome;
|
|
1530
|
+
// Reconcile shares the work-unit liveness model: the HUD heartbeat is driven
|
|
1531
|
+
// by real tool calls (a monotonic, observed count), not a re-derived turn
|
|
1532
|
+
// counter. The soft cap only paces the phase progress bar; it is never shown
|
|
1533
|
+
// to the user, so it cannot read as a misleading "X/Y" fraction.
|
|
1534
|
+
const reconcileTranscriptPath = join(transcriptDir, 'reconcile.jsonl');
|
|
1535
|
+
const reconcileProgressSoftCap = 40;
|
|
1536
|
+
const buildReconcileToolSetWithHeartbeat = () => {
|
|
1537
|
+
const record = recordTranscriptEntry(reconcileTranscriptPath);
|
|
1538
|
+
return wrapToolsWithLogger(buildReconcileToolSet({
|
|
1539
|
+
loadSkillTool: rcLoadSkill,
|
|
1540
|
+
stageListTool: rcStageListTool,
|
|
1541
|
+
stageDiffTool: rcStageDiffTool,
|
|
1542
|
+
evictionListTool: rcEvictionListTool,
|
|
1543
|
+
emitConflictResolutionTool: rcEmitConflictResolutionTool,
|
|
1544
|
+
emitEvictionDecisionTool: rcEmitEvictionDecisionTool,
|
|
1545
|
+
emitArtifactResolutionTool: rcEmitArtifactResolutionTool,
|
|
1546
|
+
emitUnmappedFallbackTool: rcEmitUnmappedFallbackTool,
|
|
1547
|
+
readRawSpanTool: rcRawSpanTool,
|
|
1548
|
+
toolsetTools: rcToolset.toRuntimeTools(rcToolContext),
|
|
1549
|
+
}), reconcileTranscriptPath, 'reconcile', {
|
|
1550
|
+
onEntry: (entry) => {
|
|
1551
|
+
const summary = record(entry);
|
|
1552
|
+
if (!stage4) {
|
|
1553
|
+
return;
|
|
1554
|
+
}
|
|
1555
|
+
const label = `Reconciling results · ${summary.toolCallCount} action${summary.toolCallCount === 1 ? '' : 's'}`;
|
|
1556
|
+
emitStageProgress('reconciliation', 85, label, { transient: true });
|
|
1557
|
+
void stage4.updateProgress(Math.min(0.95, summary.toolCallCount / reconcileProgressSoftCap), label);
|
|
1558
|
+
},
|
|
1559
|
+
});
|
|
1560
|
+
};
|
|
1454
1561
|
const reconcileStartedAt = Date.now();
|
|
1455
1562
|
const reconcileMode = contextReport && this.deps.curatorPagination ? 'curator' : 'single';
|
|
1456
1563
|
if (contextReport && this.deps.curatorPagination) {
|
|
@@ -1471,25 +1578,9 @@ export class IngestBundleRunner {
|
|
|
1471
1578
|
canonicalPins: relevantCanonicalPins,
|
|
1472
1579
|
}),
|
|
1473
1580
|
buildUserPrompt: ({ summary, items, runState }) => buildReconcileUserPrompt(stageIndex, eviction, { summary, items }, reconcileNotes, runState),
|
|
1474
|
-
buildToolSet: (_passNumber) =>
|
|
1475
|
-
loadSkillTool: rcLoadSkill,
|
|
1476
|
-
stageListTool: rcStageListTool,
|
|
1477
|
-
stageDiffTool: rcStageDiffTool,
|
|
1478
|
-
evictionListTool: rcEvictionListTool,
|
|
1479
|
-
emitConflictResolutionTool: rcEmitConflictResolutionTool,
|
|
1480
|
-
emitEvictionDecisionTool: rcEmitEvictionDecisionTool,
|
|
1481
|
-
emitArtifactResolutionTool: rcEmitArtifactResolutionTool,
|
|
1482
|
-
emitUnmappedFallbackTool: rcEmitUnmappedFallbackTool,
|
|
1483
|
-
readRawSpanTool: rcRawSpanTool,
|
|
1484
|
-
toolsetTools: rcToolset.toRuntimeTools(rcToolContext),
|
|
1485
|
-
}), join(transcriptDir, 'reconcile.jsonl'), 'reconcile', { onEntry: recordTranscriptEntry(join(transcriptDir, 'reconcile.jsonl')) }),
|
|
1581
|
+
buildToolSet: (_passNumber) => buildReconcileToolSetWithHeartbeat(),
|
|
1486
1582
|
getReconciliationActions: () => reconcileActions,
|
|
1487
|
-
|
|
1488
|
-
? ({ passNumber, stepIndex, stepBudget }) => {
|
|
1489
|
-
emitStageProgress('reconciliation', 85, `Reconciling results: pass ${passNumber} step ${stepIndex}/${stepBudget}`, { transient: true });
|
|
1490
|
-
void stage4.updateProgress(stepIndex / stepBudget, `Reconciling results · pass ${passNumber} step ${stepIndex}`);
|
|
1491
|
-
}
|
|
1492
|
-
: undefined,
|
|
1583
|
+
abortSignal: ctx?.abortSignal,
|
|
1493
1584
|
});
|
|
1494
1585
|
curatorReport = curatorOutcome.report;
|
|
1495
1586
|
curatorWarnings = curatorOutcome.warnings;
|
|
@@ -1512,31 +1603,13 @@ export class IngestBundleRunner {
|
|
|
1512
1603
|
canonicalPins: relevantCanonicalPins,
|
|
1513
1604
|
}),
|
|
1514
1605
|
buildUserPrompt: (idx, ev) => buildReconcileUserPrompt(idx, ev, undefined, reconcileNotes),
|
|
1515
|
-
buildToolSet: () =>
|
|
1516
|
-
loadSkillTool: rcLoadSkill,
|
|
1517
|
-
stageListTool: rcStageListTool,
|
|
1518
|
-
stageDiffTool: rcStageDiffTool,
|
|
1519
|
-
evictionListTool: rcEvictionListTool,
|
|
1520
|
-
emitConflictResolutionTool: rcEmitConflictResolutionTool,
|
|
1521
|
-
emitEvictionDecisionTool: rcEmitEvictionDecisionTool,
|
|
1522
|
-
emitArtifactResolutionTool: rcEmitArtifactResolutionTool,
|
|
1523
|
-
emitUnmappedFallbackTool: rcEmitUnmappedFallbackTool,
|
|
1524
|
-
readRawSpanTool: rcRawSpanTool,
|
|
1525
|
-
toolsetTools: rcToolset.toRuntimeTools(rcToolContext),
|
|
1526
|
-
}), join(transcriptDir, 'reconcile.jsonl'), 'reconcile', { onEntry: recordTranscriptEntry(join(transcriptDir, 'reconcile.jsonl')) }),
|
|
1606
|
+
buildToolSet: () => buildReconcileToolSetWithHeartbeat(),
|
|
1527
1607
|
modelRole: 'reconcile',
|
|
1528
1608
|
stepBudget: 60,
|
|
1529
1609
|
sourceKey: job.sourceKey,
|
|
1530
1610
|
jobId: job.jobId,
|
|
1531
1611
|
force: !!overrideReport,
|
|
1532
|
-
|
|
1533
|
-
? ({ stepIndex, stepBudget }) => {
|
|
1534
|
-
emitStageProgress('reconciliation', 85, `Reconciling results: step ${stepIndex}/${stepBudget}`, {
|
|
1535
|
-
transient: true,
|
|
1536
|
-
});
|
|
1537
|
-
void stage4.updateProgress(stepIndex / stepBudget, `Reconciling results · step ${stepIndex}`);
|
|
1538
|
-
}
|
|
1539
|
-
: undefined,
|
|
1612
|
+
abortSignal: ctx?.abortSignal,
|
|
1540
1613
|
});
|
|
1541
1614
|
}
|
|
1542
1615
|
await runTrace.event('debug', 'reconciliation', 'reconciliation_executed', {
|
|
@@ -1890,6 +1963,7 @@ export class IngestBundleRunner {
|
|
|
1890
1963
|
repairKind: 'final_artifact_gate',
|
|
1891
1964
|
maxAttempts: 1,
|
|
1892
1965
|
stepBudget: 16,
|
|
1966
|
+
abortSignal: ctx?.abortSignal,
|
|
1893
1967
|
});
|
|
1894
1968
|
isolatedDiffSummary.gateRepairAttempts += gateRepair.attempts;
|
|
1895
1969
|
if (gateRepair.status === 'failed') {
|
|
@@ -19,5 +19,6 @@ export interface ResolveTextualConflictInput {
|
|
|
19
19
|
reason: string;
|
|
20
20
|
maxAttempts?: number;
|
|
21
21
|
stepBudget?: number;
|
|
22
|
+
abortSignal?: AbortSignal;
|
|
22
23
|
}
|
|
23
24
|
export declare function resolveTextualConflict(input: ResolveTextualConflictInput): Promise<TextualConflictResolutionResult>;
|
|
@@ -165,6 +165,7 @@ export async function resolveTextualConflict(input) {
|
|
|
165
165
|
jobId: input.trace.context.jobId,
|
|
166
166
|
unitKey: input.unitKey,
|
|
167
167
|
},
|
|
168
|
+
abortSignal: input.abortSignal,
|
|
168
169
|
}));
|
|
169
170
|
if (result.stopReason === 'error') {
|
|
170
171
|
lastFailure = result.error?.message ?? 'resolver agent loop errored';
|
|
@@ -9,6 +9,7 @@ export interface RunIsolatedWorkUnitInput {
|
|
|
9
9
|
patchDir: string;
|
|
10
10
|
trace: IngestTraceWriter;
|
|
11
11
|
workUnit: WorkUnit;
|
|
12
|
+
abortSignal?: AbortSignal;
|
|
12
13
|
run(child: IngestSessionWorktree): Promise<WorkUnitOutcome>;
|
|
13
14
|
afterSuccess?(child: IngestSessionWorktree): Promise<void>;
|
|
14
15
|
}
|