@kaelio/ktx 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.9.0-py3-none-any.whl → kaelio_ktx-0.11.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/.tsbuildinfo +1 -1
- package/dist/clack.d.ts +6 -0
- package/dist/clack.js +17 -2
- package/dist/cli-program.d.ts +3 -0
- package/dist/cli-program.js +46 -2
- package/dist/cli-runtime.d.ts +5 -0
- package/dist/cli-runtime.js +50 -0
- package/dist/commands/setup-commands.js +2 -3
- package/dist/community-cta.d.ts +11 -0
- package/dist/community-cta.js +19 -0
- package/dist/connection.js +23 -1
- package/dist/connectors/bigquery/connector.d.ts +2 -5
- package/dist/connectors/bigquery/connector.js +2 -2
- package/dist/connectors/clickhouse/connector.d.ts +2 -5
- package/dist/connectors/clickhouse/connector.js +2 -2
- package/dist/connectors/mysql/connector.d.ts +7 -6
- package/dist/connectors/mysql/connector.js +25 -5
- package/dist/connectors/mysql/dialect.d.ts +1 -1
- package/dist/connectors/mysql/dialect.js +12 -2
- package/dist/connectors/postgres/connector.d.ts +2 -5
- package/dist/connectors/postgres/connector.js +2 -2
- package/dist/connectors/snowflake/connector.d.ts +2 -5
- package/dist/connectors/snowflake/connector.js +2 -2
- package/dist/connectors/sqlite/connector.d.ts +2 -5
- package/dist/connectors/sqlite/connector.js +2 -2
- package/dist/connectors/sqlserver/connector.d.ts +2 -5
- package/dist/connectors/sqlserver/connector.js +2 -2
- package/dist/context/connections/drivers.d.ts +0 -1
- package/dist/context/connections/drivers.js +0 -7
- package/dist/context/connections/query-executor.d.ts +2 -1
- package/dist/context/core/abort.d.ts +9 -0
- package/dist/context/core/abort.js +36 -0
- package/dist/context/core/git-env.d.ts +12 -1
- package/dist/context/core/git-env.js +17 -2
- package/dist/context/core/git.service.js +15 -7
- package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.d.ts +1 -0
- package/dist/context/ingest/adapters/historic-sql/query-history-filter-picker.js +6 -2
- package/dist/context/ingest/context-candidates/curator-pagination.service.d.ts +1 -5
- package/dist/context/ingest/context-candidates/curator-pagination.service.js +1 -3
- package/dist/context/ingest/context-evidence/sqlite-context-evidence-store.d.ts +1 -1
- package/dist/context/ingest/final-gate-repair.d.ts +1 -0
- package/dist/context/ingest/final-gate-repair.js +1 -0
- package/dist/context/ingest/ingest-bundle.runner.d.ts +3 -0
- package/dist/context/ingest/ingest-bundle.runner.js +127 -53
- package/dist/context/ingest/isolated-diff/textual-conflict-resolver.d.ts +1 -0
- package/dist/context/ingest/isolated-diff/textual-conflict-resolver.js +1 -0
- package/dist/context/ingest/isolated-diff/work-unit-executor.d.ts +1 -0
- package/dist/context/ingest/local-bundle-runtime.js +11 -4
- package/dist/context/ingest/local-ingest.d.ts +1 -0
- package/dist/context/ingest/local-ingest.js +13 -3
- package/dist/context/ingest/memory-flow/events.js +1 -1
- package/dist/context/ingest/memory-flow/schema.js +8 -3
- package/dist/context/ingest/memory-flow/types.d.ts +7 -3
- package/dist/context/ingest/ports.d.ts +3 -5
- package/dist/context/ingest/stages/stage-3-work-units.d.ts +1 -4
- package/dist/context/ingest/stages/stage-3-work-units.js +5 -1
- package/dist/context/ingest/stages/stage-4-reconciliation.d.ts +1 -4
- package/dist/context/ingest/stages/stage-4-reconciliation.js +1 -1
- package/dist/context/ingest/types.d.ts +1 -0
- package/dist/context/llm/ai-sdk-runtime.d.ts +3 -0
- package/dist/context/llm/ai-sdk-runtime.js +152 -16
- package/dist/context/llm/claude-code-runtime.d.ts +6 -4
- package/dist/context/llm/claude-code-runtime.js +127 -48
- package/dist/context/llm/codex-runtime.d.ts +3 -3
- package/dist/context/llm/codex-runtime.js +90 -47
- package/dist/context/llm/local-config.d.ts +15 -5
- package/dist/context/llm/local-config.js +6 -1
- package/dist/context/llm/rate-limit-governor.d.ts +103 -0
- package/dist/context/llm/rate-limit-governor.js +285 -0
- package/dist/context/llm/runtime-port.d.ts +3 -6
- package/dist/context/mcp/context-tools.js +43 -13
- package/dist/context/project/config.d.ts +12 -0
- package/dist/context/project/config.js +35 -0
- package/dist/context/scan/types.d.ts +15 -2
- package/dist/context/scan/types.js +12 -0
- package/dist/context/sl/description-normalization.js +4 -14
- package/dist/context/tools/context-candidate-mark.tool.d.ts +2 -2
- package/dist/context-build-view.d.ts +13 -0
- package/dist/context-build-view.js +60 -1
- package/dist/demo-metrics.d.ts +0 -2
- package/dist/demo-metrics.js +1 -11
- package/dist/ingest.d.ts +1 -0
- package/dist/ingest.js +32 -3
- package/dist/io/symbols.d.ts +2 -0
- package/dist/io/symbols.js +2 -0
- package/dist/io/tty.d.ts +9 -0
- package/dist/io/tty.js +5 -0
- package/dist/links.d.ts +1 -0
- package/dist/links.js +1 -0
- package/dist/memory-flow-hud.js +8 -16
- package/dist/public-ingest.js +50 -15
- package/dist/reveal-password-prompt.d.ts +24 -0
- package/dist/reveal-password-prompt.js +78 -0
- package/dist/scan.js +18 -2
- package/dist/setup-agents.js +1 -5
- package/dist/setup-databases.d.ts +1 -0
- package/dist/setup-databases.js +23 -3
- package/dist/setup-demo-tour.js +1 -0
- package/dist/setup-embeddings.js +1 -1
- package/dist/setup-models.d.ts +1 -14
- package/dist/setup-models.js +116 -340
- package/dist/setup-prompts.js +4 -7
- package/dist/setup-sources.js +7 -7
- package/dist/setup.d.ts +26 -1
- package/dist/setup.js +78 -7
- package/dist/sl.d.ts +2 -2
- package/dist/sl.js +20 -4
- package/dist/sql.js +18 -2
- package/dist/star-prompt/cache.d.ts +16 -0
- package/dist/star-prompt/cache.js +45 -0
- package/dist/star-prompt/star-count.d.ts +7 -0
- package/dist/star-prompt/star-count.js +66 -0
- package/dist/star-prompt/star-line.d.ts +12 -0
- package/dist/star-prompt/star-line.js +26 -0
- package/dist/telemetry/command-hook.d.ts +24 -0
- package/dist/telemetry/command-hook.js +37 -3
- package/dist/telemetry/emitter.d.ts +10 -0
- package/dist/telemetry/emitter.js +31 -0
- package/dist/telemetry/events.d.ts +24 -0
- package/dist/telemetry/events.js +15 -0
- package/dist/telemetry/exception.d.ts +18 -0
- package/dist/telemetry/exception.js +162 -0
- package/dist/telemetry/index.d.ts +4 -3
- package/dist/telemetry/index.js +3 -2
- package/dist/telemetry/redaction-secrets.d.ts +11 -0
- package/dist/telemetry/redaction-secrets.js +92 -0
- package/dist/update-check/cache.d.ts +21 -0
- package/dist/update-check/cache.js +38 -0
- package/dist/update-check/channel.d.ts +15 -0
- package/dist/update-check/channel.js +30 -0
- package/dist/update-check/registry.d.ts +1 -0
- package/dist/update-check/registry.js +45 -0
- package/dist/update-check/update-check.d.ts +43 -0
- package/dist/update-check/update-check.js +116 -0
- package/package.json +8 -1
- package/dist/context/connections/local-query-executor.d.ts +0 -6
- package/dist/context/connections/local-query-executor.js +0 -39
- package/dist/context/connections/postgres-query-executor.d.ts +0 -25
- package/dist/context/connections/postgres-query-executor.js +0 -53
- package/dist/context/connections/sqlite-query-executor.d.ts +0 -4
- package/dist/context/connections/sqlite-query-executor.js +0 -74
|
@@ -12,11 +12,6 @@ export interface KtxRuntimeToolDescriptor<TInput = unknown, TOutput = unknown> {
|
|
|
12
12
|
}
|
|
13
13
|
export type KtxRuntimeToolSet = Record<string, KtxRuntimeToolDescriptor>;
|
|
14
14
|
export type RunLoopStopReason = 'budget' | 'natural' | 'error';
|
|
15
|
-
/** @internal */
|
|
16
|
-
export interface RunLoopStepInfo {
|
|
17
|
-
stepIndex: number;
|
|
18
|
-
stepBudget: number;
|
|
19
|
-
}
|
|
20
15
|
export interface LlmTokenUsage {
|
|
21
16
|
inputTokens?: number;
|
|
22
17
|
outputTokens?: number;
|
|
@@ -40,7 +35,7 @@ export interface RunLoopParams {
|
|
|
40
35
|
toolSet: KtxRuntimeToolSet;
|
|
41
36
|
stepBudget: number;
|
|
42
37
|
telemetryTags: Record<string, string>;
|
|
43
|
-
|
|
38
|
+
abortSignal?: AbortSignal;
|
|
44
39
|
}
|
|
45
40
|
export interface RunLoopResult {
|
|
46
41
|
stopReason: RunLoopStopReason;
|
|
@@ -57,6 +52,7 @@ export interface KtxGenerateTextInput {
|
|
|
57
52
|
totalMs: number;
|
|
58
53
|
usage: LlmTokenUsage;
|
|
59
54
|
}) => void;
|
|
55
|
+
abortSignal?: AbortSignal;
|
|
60
56
|
}
|
|
61
57
|
export interface KtxGenerateObjectInput<TOutput, TSchema extends z.ZodType<TOutput>> {
|
|
62
58
|
role: KtxModelRole;
|
|
@@ -69,6 +65,7 @@ export interface KtxGenerateObjectInput<TOutput, TSchema extends z.ZodType<TOutp
|
|
|
69
65
|
totalMs: number;
|
|
70
66
|
usage: LlmTokenUsage;
|
|
71
67
|
}) => void;
|
|
68
|
+
abortSignal?: AbortSignal;
|
|
72
69
|
}
|
|
73
70
|
export interface KtxLlmRuntimePort {
|
|
74
71
|
generateText(input: KtxGenerateTextInput): Promise<string>;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { z } from 'zod';
|
|
3
|
-
import { emitTelemetryEvent, mcpTelemetrySampleRate, shouldEmitMcpTelemetry } from '../../telemetry/index.js';
|
|
3
|
+
import { emitTelemetryEvent, mcpTelemetrySampleRate, reportException, shouldEmitMcpTelemetry, } from '../../telemetry/index.js';
|
|
4
|
+
import { collectTelemetryRedactionSecrets } from '../../telemetry/redaction-secrets.js';
|
|
4
5
|
import { scrubErrorClass } from '../../telemetry/scrubber.js';
|
|
5
6
|
const connectionIdSchema = z.string().min(1);
|
|
6
7
|
const unknownRecordSchema = z.record(z.string(), z.unknown());
|
|
@@ -405,12 +406,26 @@ function mcpProgressCallback(context) {
|
|
|
405
406
|
});
|
|
406
407
|
};
|
|
407
408
|
}
|
|
408
|
-
function registerParsedTool(server, name, config, schema, handler) {
|
|
409
|
+
function registerParsedTool(server, name, config, schema, handler, telemetry) {
|
|
409
410
|
server.registerTool(name, config, async (input, context) => {
|
|
410
411
|
try {
|
|
411
412
|
return await handler(schema.parse(input), context);
|
|
412
413
|
}
|
|
413
414
|
catch (error) {
|
|
415
|
+
if (telemetry?.io) {
|
|
416
|
+
await reportException({
|
|
417
|
+
error,
|
|
418
|
+
context: { source: `mcp:${name}`, handled: true, fatal: false },
|
|
419
|
+
projectDir: telemetry.projectDir,
|
|
420
|
+
io: telemetry.io,
|
|
421
|
+
redactionSecrets: await collectTelemetryRedactionSecrets({
|
|
422
|
+
projectDir: telemetry.projectDir,
|
|
423
|
+
includeLlm: true,
|
|
424
|
+
includeEmbeddings: true,
|
|
425
|
+
env: process.env,
|
|
426
|
+
}),
|
|
427
|
+
});
|
|
428
|
+
}
|
|
414
429
|
return jsonErrorToolResult(formatToolError(error));
|
|
415
430
|
}
|
|
416
431
|
});
|
|
@@ -452,6 +467,20 @@ function instrumentMcpServer(server, telemetry) {
|
|
|
452
467
|
return result;
|
|
453
468
|
}
|
|
454
469
|
catch (error) {
|
|
470
|
+
if (telemetry.io) {
|
|
471
|
+
await reportException({
|
|
472
|
+
error,
|
|
473
|
+
context: { source: `mcp:${name}`, handled: true, fatal: false },
|
|
474
|
+
projectDir: telemetry.projectDir,
|
|
475
|
+
io: telemetry.io,
|
|
476
|
+
redactionSecrets: await collectTelemetryRedactionSecrets({
|
|
477
|
+
projectDir: telemetry.projectDir,
|
|
478
|
+
includeLlm: true,
|
|
479
|
+
includeEmbeddings: true,
|
|
480
|
+
env: process.env,
|
|
481
|
+
}),
|
|
482
|
+
});
|
|
483
|
+
}
|
|
455
484
|
if (telemetry.io && telemetry.projectDir && shouldEmitMcpTelemetry()) {
|
|
456
485
|
const errorClass = scrubErrorClass(error);
|
|
457
486
|
await emitTelemetryEvent({
|
|
@@ -476,6 +505,7 @@ function instrumentMcpServer(server, telemetry) {
|
|
|
476
505
|
}
|
|
477
506
|
export function registerKtxContextTools(deps) {
|
|
478
507
|
const { ports, userContext } = deps;
|
|
508
|
+
const toolTelemetry = { projectDir: deps.projectDir, io: deps.io };
|
|
479
509
|
const server = instrumentMcpServer(deps.server, {
|
|
480
510
|
projectDir: deps.projectDir,
|
|
481
511
|
io: deps.io,
|
|
@@ -489,7 +519,7 @@ export function registerKtxContextTools(deps) {
|
|
|
489
519
|
inputSchema: connectionListSchema.shape,
|
|
490
520
|
outputSchema: connectionListOutputSchema,
|
|
491
521
|
annotations: toolAnnotations.connection_list,
|
|
492
|
-
}, connectionListSchema, async () => jsonToolResult({ connections: await connections.list() }));
|
|
522
|
+
}, connectionListSchema, async () => jsonToolResult({ connections: await connections.list() }), toolTelemetry);
|
|
493
523
|
}
|
|
494
524
|
if (ports.knowledge) {
|
|
495
525
|
const knowledge = ports.knowledge;
|
|
@@ -503,7 +533,7 @@ export function registerKtxContextTools(deps) {
|
|
|
503
533
|
userId: userContext.userId,
|
|
504
534
|
query: input.query,
|
|
505
535
|
limit: input.limit,
|
|
506
|
-
})));
|
|
536
|
+
})), toolTelemetry);
|
|
507
537
|
registerParsedTool(server, 'wiki_read', {
|
|
508
538
|
title: toolAnnotations.wiki_read.title,
|
|
509
539
|
description: toolDescriptions.wiki_read,
|
|
@@ -513,7 +543,7 @@ export function registerKtxContextTools(deps) {
|
|
|
513
543
|
}, knowledgeReadSchema, async (input) => {
|
|
514
544
|
const page = await knowledge.read({ userId: userContext.userId, key: input.key });
|
|
515
545
|
return page ? jsonToolResult(page) : jsonErrorToolResult(`Wiki page "${input.key}" was not found.`);
|
|
516
|
-
});
|
|
546
|
+
}, toolTelemetry);
|
|
517
547
|
}
|
|
518
548
|
if (ports.semanticLayer) {
|
|
519
549
|
const semanticLayer = ports.semanticLayer;
|
|
@@ -528,7 +558,7 @@ export function registerKtxContextTools(deps) {
|
|
|
528
558
|
return source
|
|
529
559
|
? jsonToolResult(source)
|
|
530
560
|
: jsonErrorToolResult(`Semantic-layer source "${input.sourceName}" was not found.`);
|
|
531
|
-
});
|
|
561
|
+
}, toolTelemetry);
|
|
532
562
|
registerParsedTool(server, 'sl_query', {
|
|
533
563
|
title: toolAnnotations.sl_query.title,
|
|
534
564
|
description: toolDescriptions.sl_query,
|
|
@@ -550,7 +580,7 @@ export function registerKtxContextTools(deps) {
|
|
|
550
580
|
},
|
|
551
581
|
}, onProgress ? { onProgress } : undefined);
|
|
552
582
|
return jsonToolResult(projectSlQueryResult(result, input.include));
|
|
553
|
-
});
|
|
583
|
+
}, toolTelemetry);
|
|
554
584
|
}
|
|
555
585
|
if (ports.entityDetails) {
|
|
556
586
|
const entityDetails = ports.entityDetails;
|
|
@@ -560,7 +590,7 @@ export function registerKtxContextTools(deps) {
|
|
|
560
590
|
inputSchema: entityDetailsSchema.shape,
|
|
561
591
|
outputSchema: entityDetailsOutputSchema,
|
|
562
592
|
annotations: toolAnnotations.entity_details,
|
|
563
|
-
}, entityDetailsSchema, async (input) => jsonToolResult(await entityDetails.read(input)));
|
|
593
|
+
}, entityDetailsSchema, async (input) => jsonToolResult(await entityDetails.read(input)), toolTelemetry);
|
|
564
594
|
}
|
|
565
595
|
if (ports.dictionarySearch) {
|
|
566
596
|
const dictionarySearch = ports.dictionarySearch;
|
|
@@ -570,7 +600,7 @@ export function registerKtxContextTools(deps) {
|
|
|
570
600
|
inputSchema: dictionarySearchSchema.shape,
|
|
571
601
|
outputSchema: dictionarySearchOutputSchema,
|
|
572
602
|
annotations: toolAnnotations.dictionary_search,
|
|
573
|
-
}, dictionarySearchSchema, async (input) => jsonToolResult(await dictionarySearch.search(input)));
|
|
603
|
+
}, dictionarySearchSchema, async (input) => jsonToolResult(await dictionarySearch.search(input)), toolTelemetry);
|
|
574
604
|
}
|
|
575
605
|
if (ports.discover) {
|
|
576
606
|
const discover = ports.discover;
|
|
@@ -580,7 +610,7 @@ export function registerKtxContextTools(deps) {
|
|
|
580
610
|
inputSchema: discoverDataSchema.shape,
|
|
581
611
|
outputSchema: discoverDataOutputSchema,
|
|
582
612
|
annotations: toolAnnotations.discover_data,
|
|
583
|
-
}, discoverDataSchema, async (input) => jsonToolResult({ refs: await discover.search(input) }));
|
|
613
|
+
}, discoverDataSchema, async (input) => jsonToolResult({ refs: await discover.search(input) }), toolTelemetry);
|
|
584
614
|
}
|
|
585
615
|
if (ports.sqlExecution) {
|
|
586
616
|
const sqlExecution = ports.sqlExecution;
|
|
@@ -597,7 +627,7 @@ export function registerKtxContextTools(deps) {
|
|
|
597
627
|
sql: input.sql,
|
|
598
628
|
maxRows: input.maxRows ?? 1000,
|
|
599
629
|
}, onProgress ? { onProgress } : undefined));
|
|
600
|
-
});
|
|
630
|
+
}, toolTelemetry);
|
|
601
631
|
}
|
|
602
632
|
if (ports.memoryIngest) {
|
|
603
633
|
const memoryIngest = ports.memoryIngest;
|
|
@@ -617,7 +647,7 @@ export function registerKtxContextTools(deps) {
|
|
|
617
647
|
sourceType: 'external_ingest',
|
|
618
648
|
};
|
|
619
649
|
return jsonToolResult(await memoryIngest.ingest(ingestInput));
|
|
620
|
-
});
|
|
650
|
+
}, toolTelemetry);
|
|
621
651
|
registerParsedTool(server, 'memory_ingest_status', {
|
|
622
652
|
title: toolAnnotations.memory_ingest_status.title,
|
|
623
653
|
description: toolDescriptions.memory_ingest_status,
|
|
@@ -627,6 +657,6 @@ export function registerKtxContextTools(deps) {
|
|
|
627
657
|
}, memoryIngestStatusSchema, async (input) => {
|
|
628
658
|
const status = await memoryIngest.status(input.runId);
|
|
629
659
|
return status ? jsonToolResult(status) : jsonErrorToolResult(`Memory ingest run "${input.runId}" was not found.`);
|
|
630
|
-
});
|
|
660
|
+
}, toolTelemetry);
|
|
631
661
|
}
|
|
632
662
|
}
|
|
@@ -397,6 +397,18 @@ declare const ktxProjectConfigSchema: z.ZodObject<{
|
|
|
397
397
|
continue: "continue";
|
|
398
398
|
}>>;
|
|
399
399
|
}, z.core.$strict>>;
|
|
400
|
+
rateLimit: z.ZodPrefault<z.ZodObject<{
|
|
401
|
+
enabled: z.ZodDefault<z.ZodBoolean>;
|
|
402
|
+
throttleThreshold: z.ZodDefault<z.ZodNumber>;
|
|
403
|
+
minConcurrencyUnderPressure: z.ZodDefault<z.ZodInt>;
|
|
404
|
+
maxWaitMs: z.ZodOptional<z.ZodInt>;
|
|
405
|
+
retry: z.ZodPrefault<z.ZodObject<{
|
|
406
|
+
maxAttempts: z.ZodDefault<z.ZodInt>;
|
|
407
|
+
baseDelayMs: z.ZodDefault<z.ZodInt>;
|
|
408
|
+
maxDelayMs: z.ZodDefault<z.ZodInt>;
|
|
409
|
+
jitter: z.ZodDefault<z.ZodBoolean>;
|
|
410
|
+
}, z.core.$strict>>;
|
|
411
|
+
}, z.core.$strict>>;
|
|
400
412
|
profile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"json">]>>;
|
|
401
413
|
}, z.core.$strict>>;
|
|
402
414
|
agent: z.ZodPrefault<z.ZodObject<{
|
|
@@ -86,6 +86,40 @@ const workUnitsSchema = z
|
|
|
86
86
|
.describe('Behavior when a work unit fails: "abort" stops the whole ingest run; "continue" records the failure and keeps going.'),
|
|
87
87
|
})
|
|
88
88
|
.describe('Concurrency and failure handling for ingest work units.');
|
|
89
|
+
const ingestRateLimitRetrySchema = z
|
|
90
|
+
.strictObject({
|
|
91
|
+
maxAttempts: z
|
|
92
|
+
.int()
|
|
93
|
+
.positive()
|
|
94
|
+
.default(6)
|
|
95
|
+
.describe('Maximum attempts for a single rate-limited LLM call before the failure surfaces, counting the first try. Also bounds how far opaque backoff grows for providers that do not expose a reset time.'),
|
|
96
|
+
baseDelayMs: z.int().positive().default(1_000).describe('Initial opaque retry delay in milliseconds.'),
|
|
97
|
+
maxDelayMs: z.int().positive().default(60_000).describe('Maximum opaque retry delay in milliseconds.'),
|
|
98
|
+
jitter: z.boolean().default(true).describe('When true, apply bounded jitter to opaque retry delays.'),
|
|
99
|
+
})
|
|
100
|
+
.describe('Retry policy for rate-limit responses that do not include a reset time or retry-after value.');
|
|
101
|
+
const ingestRateLimitSchema = z
|
|
102
|
+
.strictObject({
|
|
103
|
+
enabled: z.boolean().default(true).describe('Master switch for ingest LLM rate-limit pacing and visible waits.'),
|
|
104
|
+
throttleThreshold: z
|
|
105
|
+
.number()
|
|
106
|
+
.min(0)
|
|
107
|
+
.max(1)
|
|
108
|
+
.default(0.8)
|
|
109
|
+
.describe('Provider utilization at or above which ingest throttles new work-unit starts.'),
|
|
110
|
+
minConcurrencyUnderPressure: z
|
|
111
|
+
.int()
|
|
112
|
+
.positive()
|
|
113
|
+
.default(1)
|
|
114
|
+
.describe('Effective work-unit concurrency while a provider is under rate-limit pressure.'),
|
|
115
|
+
maxWaitMs: z
|
|
116
|
+
.int()
|
|
117
|
+
.positive()
|
|
118
|
+
.optional()
|
|
119
|
+
.describe('Optional cap on a single provider reset wait. Omit to wait indefinitely until the provider reset time.'),
|
|
120
|
+
retry: ingestRateLimitRetrySchema.prefault({}).describe('Opaque retry policy for providers without reset hints.'),
|
|
121
|
+
})
|
|
122
|
+
.describe('Rate-limit pacing and wait policy for ingest LLM calls.');
|
|
89
123
|
const ingestSchema = z
|
|
90
124
|
.strictObject({
|
|
91
125
|
adapters: z
|
|
@@ -96,6 +130,7 @@ const ingestSchema = z
|
|
|
96
130
|
.prefault({ backend: 'none' })
|
|
97
131
|
.describe('Embedding configuration used when ingest adapters need to embed documents.'),
|
|
98
132
|
workUnits: workUnitsSchema.prefault({}).describe('Concurrency and failure handling for ingest work units.'),
|
|
133
|
+
rateLimit: ingestRateLimitSchema.prefault({}).describe('LLM rate-limit pacing and visible-wait policy for ingest.'),
|
|
99
134
|
profile: z
|
|
100
135
|
.union([z.boolean(), z.literal('json')])
|
|
101
136
|
.default(false)
|
|
@@ -233,10 +233,23 @@ export interface KtxTableListEntry {
|
|
|
233
233
|
name: string;
|
|
234
234
|
kind: 'table' | 'view';
|
|
235
235
|
}
|
|
236
|
-
interface KtxConnectorTestResult {
|
|
236
|
+
export interface KtxConnectorTestResult {
|
|
237
237
|
success: boolean;
|
|
238
238
|
error?: string;
|
|
239
|
-
|
|
239
|
+
/**
|
|
240
|
+
* The original error thrown by the driver, preserved unflattened so the
|
|
241
|
+
* connection-test path can re-throw it. Keeping the real error object lets
|
|
242
|
+
* telemetry record the driver's actual error class (e.g. `ConnectionError`)
|
|
243
|
+
* and `.code` (e.g. `ELOGIN`) instead of collapsing every failure to `Error`.
|
|
244
|
+
*/
|
|
245
|
+
cause?: unknown;
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* Single source of truth for a failed connector test result. Captures the
|
|
249
|
+
* driver's message for display while preserving the original error as `cause`
|
|
250
|
+
* so callers can surface its real class and code.
|
|
251
|
+
*/
|
|
252
|
+
export declare function connectorTestFailure(error: unknown): KtxConnectorTestResult;
|
|
240
253
|
export interface KtxScanConnector {
|
|
241
254
|
id: string;
|
|
242
255
|
driver: KtxConnectionDriver;
|
|
@@ -11,3 +11,15 @@ export function createKtxConnectorCapabilities(capabilities = {}) {
|
|
|
11
11
|
estimatedRowCounts: capabilities.estimatedRowCounts ?? false,
|
|
12
12
|
};
|
|
13
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* Single source of truth for a failed connector test result. Captures the
|
|
16
|
+
* driver's message for display while preserving the original error as `cause`
|
|
17
|
+
* so callers can surface its real class and code.
|
|
18
|
+
*/
|
|
19
|
+
export function connectorTestFailure(error) {
|
|
20
|
+
return {
|
|
21
|
+
success: false,
|
|
22
|
+
error: error instanceof Error ? error.message : String(error),
|
|
23
|
+
cause: error,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
@@ -39,12 +39,6 @@ function humanizeIdentifier(value) {
|
|
|
39
39
|
.trim()
|
|
40
40
|
.toLowerCase();
|
|
41
41
|
}
|
|
42
|
-
function formatCount(count, singular, plural = `${singular}s`) {
|
|
43
|
-
if (count <= 0) {
|
|
44
|
-
return null;
|
|
45
|
-
}
|
|
46
|
-
return `${count} ${count === 1 ? singular : plural}`;
|
|
47
|
-
}
|
|
48
42
|
function sourceFallback(source, sourceName) {
|
|
49
43
|
const table = cleanText(source.table);
|
|
50
44
|
const sql = cleanText(source.sql);
|
|
@@ -54,14 +48,10 @@ function sourceFallback(source, sourceName) {
|
|
|
54
48
|
if (sql) {
|
|
55
49
|
return `Semantic-layer source for ${sourceName} backed by curated SQL.`;
|
|
56
50
|
}
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
].filter((item) => Boolean(item));
|
|
62
|
-
return counts.length > 0
|
|
63
|
-
? `Semantic-layer overlay for ${sourceName} defining ${counts.join(', ')}.`
|
|
64
|
-
: `Semantic-layer overlay for ${sourceName}.`;
|
|
51
|
+
// Measure/segment/column counts are rendered live from the body at list/read
|
|
52
|
+
// time, so baking them into stored prose freezes a derived value that drifts
|
|
53
|
+
// as the source later gains measures. Keep the auto fallback count-free.
|
|
54
|
+
return `Semantic-layer overlay for ${sourceName}.`;
|
|
65
55
|
}
|
|
66
56
|
function columnFallback(column, sourceName) {
|
|
67
57
|
const columnName = cleanText(column.name) ?? 'column';
|
|
@@ -7,8 +7,8 @@ declare const contextCandidateMarkInputSchema: z.ZodObject<{
|
|
|
7
7
|
status: z.ZodEnum<{
|
|
8
8
|
conflict: "conflict";
|
|
9
9
|
merged: "merged";
|
|
10
|
-
pending: "pending";
|
|
11
10
|
rejected: "rejected";
|
|
11
|
+
pending: "pending";
|
|
12
12
|
promoted: "promoted";
|
|
13
13
|
}>;
|
|
14
14
|
rejectionReason: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
@@ -30,8 +30,8 @@ export declare class ContextCandidateMarkTool extends BaseTool<typeof contextCan
|
|
|
30
30
|
status: z.ZodEnum<{
|
|
31
31
|
conflict: "conflict";
|
|
32
32
|
merged: "merged";
|
|
33
|
-
pending: "pending";
|
|
34
33
|
rejected: "rejected";
|
|
34
|
+
pending: "pending";
|
|
35
35
|
promoted: "promoted";
|
|
36
36
|
}>;
|
|
37
37
|
rejectionReason: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
@@ -2,6 +2,7 @@ import type { KtxCliIo } from './index.js';
|
|
|
2
2
|
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
|
|
3
3
|
import type { KtxPublicIngestArgs, KtxPublicIngestPlanTarget, KtxPublicIngestProject } from './public-ingest.js';
|
|
4
4
|
import { executePublicIngestTarget } from './public-ingest.js';
|
|
5
|
+
import { fetchGitHubStarCount as defaultFetchGitHubStarCount } from './star-prompt/star-count.js';
|
|
5
6
|
type PhaseKey = 'database-schema' | 'query-history' | 'source-ingest';
|
|
6
7
|
type PhaseStatus = 'queued' | 'running' | 'done' | 'failed' | 'skipped';
|
|
7
8
|
interface PhaseState {
|
|
@@ -32,6 +33,7 @@ export interface ContextBuildViewState {
|
|
|
32
33
|
frame: number;
|
|
33
34
|
startedAt: number | null;
|
|
34
35
|
totalElapsedMs: number;
|
|
36
|
+
starCount: number | null;
|
|
35
37
|
}
|
|
36
38
|
export interface ContextBuildArgs {
|
|
37
39
|
projectDir: string;
|
|
@@ -73,6 +75,8 @@ interface CompletedItemName {
|
|
|
73
75
|
interface ContextBuildRenderOptions {
|
|
74
76
|
styled?: boolean;
|
|
75
77
|
showHint?: boolean;
|
|
78
|
+
showStarPrompt?: boolean;
|
|
79
|
+
columns?: number;
|
|
76
80
|
hintText?: string;
|
|
77
81
|
projectDir?: string;
|
|
78
82
|
title?: string;
|
|
@@ -89,6 +93,14 @@ export interface ContextBuildDeps {
|
|
|
89
93
|
now?: () => number;
|
|
90
94
|
onSourceProgress?: (sources: ContextBuildSourceProgressUpdate[]) => void;
|
|
91
95
|
sourceProgressThrottleMs?: number;
|
|
96
|
+
fetchStarCount?: typeof defaultFetchGitHubStarCount;
|
|
97
|
+
starPromptEnv?: StarPromptEnv;
|
|
98
|
+
starPromptHomeDir?: string;
|
|
99
|
+
}
|
|
100
|
+
interface StarPromptEnv extends NodeJS.ProcessEnv {
|
|
101
|
+
CI?: string;
|
|
102
|
+
DO_NOT_TRACK?: string;
|
|
103
|
+
KTX_NO_STAR?: string;
|
|
92
104
|
}
|
|
93
105
|
export declare function renderContextBuildView(state: ContextBuildViewState, options?: ContextBuildRenderOptions): string;
|
|
94
106
|
/** @internal */
|
|
@@ -101,6 +113,7 @@ export declare function parseIngestSummary(output: string): string | null;
|
|
|
101
113
|
export declare function viewStateFromSourceProgress(sources: ContextBuildSourceProgressUpdate[], now: number, startedAtMs?: number): ContextBuildViewState;
|
|
102
114
|
export declare function createRepainter(io: KtxCliIo): {
|
|
103
115
|
paint(content: string): void;
|
|
116
|
+
columns(): number;
|
|
104
117
|
};
|
|
105
118
|
export declare function initViewState(targets: KtxPublicIngestPlanTarget[]): ContextBuildViewState;
|
|
106
119
|
export declare function runContextBuild(project: KtxPublicIngestProject, args: ContextBuildArgs, io: KtxCliIo, deps?: ContextBuildDeps): Promise<ContextBuildResult>;
|
|
@@ -2,6 +2,9 @@ import { buildPublicIngestPlan, executePublicIngestTarget, publicProgressMessage
|
|
|
2
2
|
import { createAggregateProgressPort } from './progress-port-adapter.js';
|
|
3
3
|
import { formatDuration } from './demo-metrics.js';
|
|
4
4
|
import { profileMark } from './startup-profile.js';
|
|
5
|
+
import { isFreshStarCountCache, readStarCountCache, writeStarCountCache, } from './star-prompt/cache.js';
|
|
6
|
+
import { fetchGitHubStarCount as defaultFetchGitHubStarCount } from './star-prompt/star-count.js';
|
|
7
|
+
import { renderStarPromptLine } from './star-prompt/star-line.js';
|
|
5
8
|
profileMark('module:context-build-view');
|
|
6
9
|
const SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
|
|
7
10
|
const ESC = String.fromCharCode(0x1b);
|
|
@@ -286,6 +289,13 @@ export function renderContextBuildView(state, options = {}) {
|
|
|
286
289
|
lines.push(styled ? green(summary) : summary);
|
|
287
290
|
lines.push('');
|
|
288
291
|
}
|
|
292
|
+
if (options.showStarPrompt && hasActive) {
|
|
293
|
+
const starPrompt = renderStarPromptLine({
|
|
294
|
+
count: state.starCount,
|
|
295
|
+
columns: options.columns ?? 80,
|
|
296
|
+
});
|
|
297
|
+
lines.push(styled ? dim(starPrompt) : starPrompt);
|
|
298
|
+
}
|
|
289
299
|
if (options.showHint && hasActive) {
|
|
290
300
|
const hintContent = options.hintText ?? 'Ctrl+C to stop';
|
|
291
301
|
const hint = ` ${hintContent}`;
|
|
@@ -423,6 +433,7 @@ export function viewStateFromSourceProgress(sources, now, startedAtMs) {
|
|
|
423
433
|
frame: 0,
|
|
424
434
|
startedAt: startedAtMs ?? null,
|
|
425
435
|
totalElapsedMs: startedAtMs ? now - startedAtMs : 0,
|
|
436
|
+
starCount: null,
|
|
426
437
|
};
|
|
427
438
|
}
|
|
428
439
|
// --- Repaint ---
|
|
@@ -465,6 +476,9 @@ export function createRepainter(io) {
|
|
|
465
476
|
hasPainted = true;
|
|
466
477
|
lastCursorUpRows = cursorUpRowsAfterWrite(content);
|
|
467
478
|
},
|
|
479
|
+
columns() {
|
|
480
|
+
return terminalColumns();
|
|
481
|
+
},
|
|
468
482
|
};
|
|
469
483
|
}
|
|
470
484
|
// --- Orchestration ---
|
|
@@ -616,12 +630,42 @@ export function initViewState(targets) {
|
|
|
616
630
|
frame: 0,
|
|
617
631
|
startedAt: null,
|
|
618
632
|
totalElapsedMs: 0,
|
|
633
|
+
starCount: null,
|
|
619
634
|
};
|
|
620
635
|
}
|
|
621
636
|
function formatProgressDetail(update, target) {
|
|
622
637
|
const percent = Math.max(0, Math.min(100, Math.round(update.percent)));
|
|
623
638
|
return `[${percent}%] ${publicProgressMessage(update.message, target)}`;
|
|
624
639
|
}
|
|
640
|
+
const STAR_COUNT_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
|
641
|
+
function envFlag(value) {
|
|
642
|
+
return value !== undefined && value !== '' && value !== '0' && value !== 'false';
|
|
643
|
+
}
|
|
644
|
+
function shouldSuppressStarPrompt(env) {
|
|
645
|
+
return envFlag(env.CI) || envFlag(env.DO_NOT_TRACK) || envFlag(env.KTX_NO_STAR);
|
|
646
|
+
}
|
|
647
|
+
function startStarPromptCountRefresh(input) {
|
|
648
|
+
const cached = readStarCountCache({ homeDir: input.homeDir });
|
|
649
|
+
if (cached) {
|
|
650
|
+
input.state.starCount = cached.count;
|
|
651
|
+
}
|
|
652
|
+
if (isFreshStarCountCache(cached, new Date(input.now()), STAR_COUNT_CACHE_TTL_MS)) {
|
|
653
|
+
return;
|
|
654
|
+
}
|
|
655
|
+
void input.fetchStarCount()
|
|
656
|
+
.then((count) => {
|
|
657
|
+
if (typeof count !== 'number' || !Number.isFinite(count)) {
|
|
658
|
+
return;
|
|
659
|
+
}
|
|
660
|
+
input.state.starCount = count;
|
|
661
|
+
input.paint();
|
|
662
|
+
void writeStarCountCache({
|
|
663
|
+
count,
|
|
664
|
+
fetchedAt: new Date(input.now()).toISOString(),
|
|
665
|
+
}, { homeDir: input.homeDir });
|
|
666
|
+
})
|
|
667
|
+
.catch(() => undefined);
|
|
668
|
+
}
|
|
625
669
|
export async function runContextBuild(project, args, io, deps = {}) {
|
|
626
670
|
const plan = buildPublicIngestPlan(project, {
|
|
627
671
|
projectDir: args.projectDir,
|
|
@@ -636,13 +680,28 @@ export async function runContextBuild(project, args, io, deps = {}) {
|
|
|
636
680
|
const nowFn = deps.now ?? (() => Date.now());
|
|
637
681
|
state.startedAt = nowFn();
|
|
638
682
|
const repainter = isTTY ? createRepainter(io) : null;
|
|
683
|
+
const starPromptEnabled = repainter !== null && !shouldSuppressStarPrompt(deps.starPromptEnv ?? process.env);
|
|
639
684
|
const viewOpts = {
|
|
640
685
|
styled: true,
|
|
641
686
|
projectDir: args.projectDir,
|
|
642
687
|
notices: plan.notices ?? [],
|
|
643
688
|
warnings: plan.warnings,
|
|
644
689
|
};
|
|
645
|
-
const paint = (hint) => repainter?.paint(renderContextBuildView(state, {
|
|
690
|
+
const paint = (hint) => repainter?.paint(renderContextBuildView(state, {
|
|
691
|
+
...viewOpts,
|
|
692
|
+
showHint: hint,
|
|
693
|
+
showStarPrompt: starPromptEnabled && hint,
|
|
694
|
+
columns: repainter.columns(),
|
|
695
|
+
}));
|
|
696
|
+
if (starPromptEnabled) {
|
|
697
|
+
startStarPromptCountRefresh({
|
|
698
|
+
fetchStarCount: deps.fetchStarCount ?? defaultFetchGitHubStarCount,
|
|
699
|
+
homeDir: deps.starPromptHomeDir,
|
|
700
|
+
now: nowFn,
|
|
701
|
+
paint: () => paint(true),
|
|
702
|
+
state,
|
|
703
|
+
});
|
|
704
|
+
}
|
|
646
705
|
paint(true);
|
|
647
706
|
let spinnerInterval = null;
|
|
648
707
|
if (repainter) {
|
package/dist/demo-metrics.d.ts
CHANGED
package/dist/demo-metrics.js
CHANGED
|
@@ -5,13 +5,6 @@ const DEFAULT_OUTPUT_PRICE_PER_MTOK_USD = 15;
|
|
|
5
5
|
function eventsOf(events, type) {
|
|
6
6
|
return events.filter((event) => event.type === type);
|
|
7
7
|
}
|
|
8
|
-
function maxAgentStep(events) {
|
|
9
|
-
const steps = eventsOf(events, 'work_unit_step');
|
|
10
|
-
const started = eventsOf(events, 'work_unit_started');
|
|
11
|
-
const stepIndex = steps.reduce((max, event) => Math.max(max, event.stepIndex), 0);
|
|
12
|
-
const stepBudget = Math.max(0, ...steps.map((event) => event.stepBudget), ...started.map((event) => event.stepBudget));
|
|
13
|
-
return { step: stepIndex, budget: stepBudget };
|
|
14
|
-
}
|
|
15
8
|
function totalToolCalls(input) {
|
|
16
9
|
return input.details.transcripts.reduce((total, transcript) => total + transcript.toolCallCount, 0);
|
|
17
10
|
}
|
|
@@ -49,11 +42,10 @@ export function buildDemoMetrics(input, options = {}) {
|
|
|
49
42
|
const outputPrice = tuning.outputPricePerMTokUsd ?? DEFAULT_OUTPUT_PRICE_PER_MTOK_USD;
|
|
50
43
|
const nowMs = (options.now ?? Date.now)();
|
|
51
44
|
const elapsedMs = elapsedMsFromEvents(input.events, nowMs);
|
|
52
|
-
const { step, budget } = maxAgentStep(input.events);
|
|
53
45
|
const toolCalls = totalToolCalls(input);
|
|
54
46
|
const progress = workUnitProgress(input);
|
|
55
47
|
const finishedCount = eventsOf(input.events, 'work_unit_finished').length;
|
|
56
|
-
const stepDriver = Math.max(
|
|
48
|
+
const stepDriver = Math.max(toolCalls, finishedCount * 4);
|
|
57
49
|
const inputTokens = stepDriver * inputTokensPerStep;
|
|
58
50
|
const outputTokens = stepDriver * outputTokensPerStep;
|
|
59
51
|
const totalTokens = inputTokens + outputTokens;
|
|
@@ -63,8 +55,6 @@ export function buildDemoMetrics(input, options = {}) {
|
|
|
63
55
|
return {
|
|
64
56
|
elapsedMs,
|
|
65
57
|
etaMs: estimateEtaMs(elapsedMs, progress.finished, progress.total, input.status),
|
|
66
|
-
agentSteps: step,
|
|
67
|
-
agentStepBudget: budget,
|
|
68
58
|
toolCalls,
|
|
69
59
|
workUnitsStarted: progress.started,
|
|
70
60
|
workUnitsFinished: progress.finished,
|
package/dist/ingest.d.ts
CHANGED
|
@@ -58,6 +58,7 @@ export interface KtxIngestDeps {
|
|
|
58
58
|
readReportFile?: typeof readIngestReportSnapshotFile;
|
|
59
59
|
renderStoredMemoryFlow?: typeof renderMemoryFlowTui;
|
|
60
60
|
startLiveMemoryFlow?: typeof startLiveMemoryFlowTui;
|
|
61
|
+
abortSignal?: AbortSignal;
|
|
61
62
|
env?: NodeJS.ProcessEnv;
|
|
62
63
|
localIngestOptions?: Pick<RunLocalIngestOptions, 'agentRunner' | 'llmRuntime' | 'memoryModel' | 'semanticLayerCompute' | 'queryExecutor' | 'logger' | 'pullConfigOptions'>;
|
|
63
64
|
progress?: (update: KtxIngestProgressUpdate) => void;
|