@kaelio/ktx 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.2.0-py3-none-any.whl → kaelio_ktx-0.4.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/admin-reindex.js +10 -17
- package/dist/admin-reindex.test.js +1 -1
- package/dist/cli-program.test.js +0 -2
- package/dist/cli-project.d.ts +18 -0
- package/dist/cli-project.js +52 -0
- package/dist/cli-project.test.js +149 -0
- package/dist/cli-runtime.d.ts +0 -2
- package/dist/cli-runtime.js +2 -8
- package/dist/commands/runtime-commands.js +2 -2
- package/dist/context-build-view.js +1 -1
- package/dist/index.test.js +21 -25
- package/dist/ingest.js +9 -2
- package/dist/ingest.test.js +27 -3
- package/dist/managed-local-embeddings.d.ts +0 -2
- package/dist/managed-local-embeddings.js +2 -5
- package/dist/managed-local-embeddings.test.js +5 -8
- package/dist/managed-python-daemon.js +2 -2
- package/dist/managed-python-daemon.test.js +1 -1
- package/dist/managed-python-http.js +3 -3
- package/dist/managed-python-http.test.js +6 -6
- package/dist/print-command-tree.js +0 -2
- package/dist/public-ingest.d.ts +4 -2
- package/dist/public-ingest.js +9 -3
- package/dist/release-version.d.ts +1 -5
- package/dist/release-version.js +2 -39
- package/dist/runtime-requirements.js +1 -1
- package/dist/runtime.js +6 -6
- package/dist/runtime.test.js +7 -7
- package/dist/scan.js +7 -2
- package/dist/scan.test.js +1 -1
- package/dist/setup-embeddings.js +1 -1
- package/dist/setup-embeddings.test.js +2 -2
- package/dist/setup-runtime.test.js +1 -1
- package/node_modules/@ktx/context/dist/core/git.service.d.ts +1 -0
- package/node_modules/@ktx/context/dist/core/git.service.js +12 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.d.ts +2 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.js +18 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/local-ingest-acceptance.test.js +6 -6
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.d.ts +5 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.js +48 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.test.js +83 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/live-database/daemon-introspection.js +4 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/live-database/daemon-introspection.test.js +32 -0
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.d.ts +22 -0
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.js +95 -0
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.test.js +114 -0
- package/node_modules/@ktx/context/dist/ingest/index.d.ts +1 -2
- package/node_modules/@ktx/context/dist/ingest/index.js +0 -1
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.d.ts +2 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.isolated-diff.test.js +166 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.js +235 -45
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.test.js +193 -38
- package/node_modules/@ktx/context/dist/ingest/local-bundle-ingest.test.js +22 -3
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.js +0 -4
- package/node_modules/@ktx/context/dist/ingest/local-ingest.js +0 -7
- package/node_modules/@ktx/context/dist/ingest/local-stage-ingest.js +15 -5
- package/node_modules/@ktx/context/dist/ingest/local-stage-ingest.test.js +29 -0
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.d.ts +2 -2
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.js +1 -1
- package/node_modules/@ktx/context/dist/ingest/memory-flow/types.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/ports.d.ts +1 -20
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.d.ts +71 -0
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.js +27 -0
- package/node_modules/@ktx/context/dist/ingest/reports.d.ts +23 -5
- package/node_modules/@ktx/context/dist/ingest/reports.js +7 -24
- package/node_modules/@ktx/context/dist/ingest/types.d.ts +33 -0
- package/node_modules/@ktx/context/dist/llm/index.d.ts +1 -1
- package/node_modules/@ktx/context/dist/llm/index.js +1 -1
- package/node_modules/@ktx/context/dist/llm/local-config.d.ts +0 -1
- package/node_modules/@ktx/context/dist/llm/local-config.js +2 -12
- package/node_modules/@ktx/context/dist/llm/local-config.test.js +2 -23
- package/node_modules/@ktx/context/dist/package-exports.test.js +2 -2
- package/node_modules/@ktx/context/dist/project/config.d.ts +16 -0
- package/node_modules/@ktx/context/dist/project/driver-schemas.d.ts +8 -0
- package/node_modules/@ktx/context/dist/project/driver-schemas.js +4 -0
- package/node_modules/@ktx/context/dist/scan/enabled-tables.d.ts +3 -0
- package/node_modules/@ktx/context/dist/scan/enabled-tables.js +15 -0
- package/node_modules/@ktx/context/dist/scan/local-scan.d.ts +2 -4
- package/node_modules/@ktx/context/dist/scan/local-scan.js +2 -15
- package/package.json +1 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.d.ts +0 -4
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.js +0 -38
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.test.js +0 -63
- /package/{node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.test.d.ts → dist/cli-project.test.d.ts} +0 -0
package/assets/python/{kaelio_ktx-0.2.0-py3-none-any.whl → kaelio_ktx-0.4.0-py3-none-any.whl}
RENAMED
|
Binary file
|
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
"schemaVersion": 1,
|
|
3
3
|
"distributionName": "kaelio-ktx",
|
|
4
4
|
"normalizedName": "kaelio_ktx",
|
|
5
|
-
"version": "0.
|
|
5
|
+
"version": "0.4.0",
|
|
6
6
|
"wheel": {
|
|
7
|
-
"file": "kaelio_ktx-0.
|
|
8
|
-
"sha256": "
|
|
9
|
-
"bytes":
|
|
7
|
+
"file": "kaelio_ktx-0.4.0-py3-none-any.whl",
|
|
8
|
+
"sha256": "f969fd507d6ac483ae8ce2aab8257a4cb73dec0b3edcdef9be38dc133eb9a377",
|
|
9
|
+
"bytes": 80523
|
|
10
10
|
}
|
|
11
11
|
}
|
package/dist/admin-reindex.js
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
import { createLocalKtxEmbeddingProviderFromConfig, KtxIngestEmbeddingPortAdapter,
|
|
1
|
+
import { createLocalKtxEmbeddingProviderFromConfig, KtxIngestEmbeddingPortAdapter, } from '@ktx/context';
|
|
2
2
|
import { reindexLocalIndexes } from '@ktx/context/index-sync';
|
|
3
|
-
import { loadKtxProject } from '@ktx/context/project';
|
|
4
3
|
import { Option } from '@commander-js/extra-typings';
|
|
5
4
|
import { cancel, intro, log, note, outro } from '@clack/prompts';
|
|
5
|
+
import { loadKtxCliProject } from './cli-project.js';
|
|
6
6
|
import { resolveOutputMode } from './io/mode.js';
|
|
7
7
|
import { green, red, SYMBOLS } from './io/symbols.js';
|
|
8
|
-
import { ensureManagedLocalEmbeddingsDaemon } from './managed-local-embeddings.js';
|
|
9
8
|
export function registerAdminReindexCommand(admin, context) {
|
|
10
9
|
admin
|
|
11
10
|
.command('reindex')
|
|
@@ -25,22 +24,11 @@ export function registerAdminReindexCommand(admin, context) {
|
|
|
25
24
|
}, context.io));
|
|
26
25
|
});
|
|
27
26
|
}
|
|
28
|
-
|
|
27
|
+
function resolveReindexEmbeddingService(project) {
|
|
29
28
|
const config = project.config.ingest.embeddings;
|
|
30
29
|
if (config.backend === 'none') {
|
|
31
30
|
return null;
|
|
32
31
|
}
|
|
33
|
-
if (config.backend === 'sentence-transformers' &&
|
|
34
|
-
config.sentenceTransformers?.base_url === MANAGED_SENTENCE_TRANSFORMERS_BASE_URL) {
|
|
35
|
-
const daemon = await ensureManagedLocalEmbeddingsDaemon({
|
|
36
|
-
cliVersion: args.cliVersion,
|
|
37
|
-
projectDir: project.projectDir,
|
|
38
|
-
installPolicy: 'never',
|
|
39
|
-
io,
|
|
40
|
-
});
|
|
41
|
-
const provider = createLocalKtxEmbeddingProviderFromConfig(config, { env: { ...process.env, ...daemon.env } });
|
|
42
|
-
return provider ? new KtxIngestEmbeddingPortAdapter(provider) : null;
|
|
43
|
-
}
|
|
44
32
|
const provider = createLocalKtxEmbeddingProviderFromConfig(config);
|
|
45
33
|
return provider ? new KtxIngestEmbeddingPortAdapter(provider) : null;
|
|
46
34
|
}
|
|
@@ -143,8 +131,13 @@ function renderReindexPretty(summary, io) {
|
|
|
143
131
|
}
|
|
144
132
|
async function runKtxAdminReindex(args, io = process) {
|
|
145
133
|
try {
|
|
146
|
-
const project = await
|
|
147
|
-
|
|
134
|
+
const project = await loadKtxCliProject({
|
|
135
|
+
projectDir: args.projectDir,
|
|
136
|
+
cliVersion: args.cliVersion,
|
|
137
|
+
installPolicy: 'never',
|
|
138
|
+
io,
|
|
139
|
+
});
|
|
140
|
+
const embeddingService = resolveReindexEmbeddingService(project);
|
|
148
141
|
const summary = await reindexLocalIndexes(project, { force: args.force, embeddingService });
|
|
149
142
|
const mode = resolveOutputMode({ explicit: args.output, json: args.json, io });
|
|
150
143
|
if (!summary.embeddingsAvailable && mode === 'plain') {
|
package/dist/cli-program.test.js
CHANGED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project';
|
|
2
|
+
import type { KtxProjectConfig } from '@ktx/context/project';
|
|
3
|
+
import type { KtxCliIo } from './cli-runtime.js';
|
|
4
|
+
import { ensureManagedLocalEmbeddingsDaemon, type ManagedLocalEmbeddingsDaemon } from './managed-local-embeddings.js';
|
|
5
|
+
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
|
|
6
|
+
export interface LoadKtxCliProjectOptions {
|
|
7
|
+
projectDir: string;
|
|
8
|
+
cliVersion: string;
|
|
9
|
+
installPolicy: KtxManagedPythonInstallPolicy;
|
|
10
|
+
io: KtxCliIo;
|
|
11
|
+
}
|
|
12
|
+
export interface LoadKtxCliProjectDeps {
|
|
13
|
+
loadProject?: typeof loadKtxProject;
|
|
14
|
+
ensureLocalEmbeddings?: (options: Parameters<typeof ensureManagedLocalEmbeddingsDaemon>[0]) => Promise<ManagedLocalEmbeddingsDaemon>;
|
|
15
|
+
}
|
|
16
|
+
export declare function loadKtxCliProject(options: LoadKtxCliProjectOptions, deps?: LoadKtxCliProjectDeps): Promise<KtxLocalProject>;
|
|
17
|
+
export declare function projectNeedsManagedLocalEmbeddings(config: KtxProjectConfig): boolean;
|
|
18
|
+
export declare function substituteManagedLocalEmbeddingsUrl(config: KtxProjectConfig, baseUrl: string): KtxProjectConfig;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
|
|
2
|
+
import { loadKtxProject } from '@ktx/context/project';
|
|
3
|
+
import { ensureManagedLocalEmbeddingsDaemon, } from './managed-local-embeddings.js';
|
|
4
|
+
export async function loadKtxCliProject(options, deps = {}) {
|
|
5
|
+
const loadProject = deps.loadProject ?? loadKtxProject;
|
|
6
|
+
const ensureLocalEmbeddings = deps.ensureLocalEmbeddings ?? ensureManagedLocalEmbeddingsDaemon;
|
|
7
|
+
const project = await loadProject({ projectDir: options.projectDir });
|
|
8
|
+
if (!projectNeedsManagedLocalEmbeddings(project.config)) {
|
|
9
|
+
return project;
|
|
10
|
+
}
|
|
11
|
+
const daemon = await ensureLocalEmbeddings({
|
|
12
|
+
cliVersion: options.cliVersion,
|
|
13
|
+
projectDir: options.projectDir,
|
|
14
|
+
installPolicy: options.installPolicy,
|
|
15
|
+
io: options.io,
|
|
16
|
+
});
|
|
17
|
+
return {
|
|
18
|
+
...project,
|
|
19
|
+
config: substituteManagedLocalEmbeddingsUrl(project.config, daemon.baseUrl),
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
export function projectNeedsManagedLocalEmbeddings(config) {
|
|
23
|
+
return (embeddingUsesManagedSentinel(config.ingest.embeddings) ||
|
|
24
|
+
embeddingUsesManagedSentinel(config.scan.enrichment.embeddings));
|
|
25
|
+
}
|
|
26
|
+
export function substituteManagedLocalEmbeddingsUrl(config, baseUrl) {
|
|
27
|
+
const ingestEmbeddings = rewriteManagedEmbeddingConfig(config.ingest.embeddings, baseUrl);
|
|
28
|
+
const scanEnrichmentEmbeddings = rewriteManagedEmbeddingConfig(config.scan.enrichment.embeddings, baseUrl);
|
|
29
|
+
return {
|
|
30
|
+
...config,
|
|
31
|
+
ingest: { ...config.ingest, embeddings: ingestEmbeddings },
|
|
32
|
+
scan: {
|
|
33
|
+
...config.scan,
|
|
34
|
+
enrichment: { ...config.scan.enrichment, embeddings: scanEnrichmentEmbeddings },
|
|
35
|
+
},
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
function embeddingUsesManagedSentinel(embedding) {
|
|
39
|
+
return embedding?.sentenceTransformers?.base_url === MANAGED_SENTENCE_TRANSFORMERS_BASE_URL;
|
|
40
|
+
}
|
|
41
|
+
function rewriteManagedEmbeddingConfig(embedding, baseUrl) {
|
|
42
|
+
if (!embedding || !embeddingUsesManagedSentinel(embedding)) {
|
|
43
|
+
return embedding;
|
|
44
|
+
}
|
|
45
|
+
return {
|
|
46
|
+
...embedding,
|
|
47
|
+
sentenceTransformers: {
|
|
48
|
+
...embedding.sentenceTransformers,
|
|
49
|
+
base_url: baseUrl,
|
|
50
|
+
},
|
|
51
|
+
};
|
|
52
|
+
}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
2
|
+
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
|
|
3
|
+
import { buildDefaultKtxProjectConfig } from '@ktx/context/project';
|
|
4
|
+
import { loadKtxCliProject, projectNeedsManagedLocalEmbeddings, substituteManagedLocalEmbeddingsUrl, } from './cli-project.js';
|
|
5
|
+
const RESOLVED_BASE_URL = 'http://127.0.0.1:51234';
|
|
6
|
+
function makeIo() {
|
|
7
|
+
let stderr = '';
|
|
8
|
+
return {
|
|
9
|
+
io: {
|
|
10
|
+
stdout: { write: (_chunk) => { } },
|
|
11
|
+
stderr: {
|
|
12
|
+
write: (chunk) => {
|
|
13
|
+
stderr += chunk;
|
|
14
|
+
},
|
|
15
|
+
},
|
|
16
|
+
},
|
|
17
|
+
stderr: () => stderr,
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
function projectWithConfig(config) {
|
|
21
|
+
return {
|
|
22
|
+
projectDir: '/work/proj',
|
|
23
|
+
configPath: '/work/proj/ktx.yaml',
|
|
24
|
+
config,
|
|
25
|
+
coreConfig: {},
|
|
26
|
+
git: {},
|
|
27
|
+
fileStore: {},
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
function withManagedIngestEmbedding(config) {
|
|
31
|
+
return {
|
|
32
|
+
...config,
|
|
33
|
+
ingest: {
|
|
34
|
+
...config.ingest,
|
|
35
|
+
embeddings: {
|
|
36
|
+
backend: 'sentence-transformers',
|
|
37
|
+
model: 'all-MiniLM-L6-v2',
|
|
38
|
+
dimensions: 384,
|
|
39
|
+
sentenceTransformers: { base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, pathPrefix: '' },
|
|
40
|
+
},
|
|
41
|
+
},
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
function withManagedScanEnrichmentEmbedding(config) {
|
|
45
|
+
return {
|
|
46
|
+
...config,
|
|
47
|
+
scan: {
|
|
48
|
+
...config.scan,
|
|
49
|
+
enrichment: {
|
|
50
|
+
...config.scan.enrichment,
|
|
51
|
+
embeddings: {
|
|
52
|
+
backend: 'sentence-transformers',
|
|
53
|
+
model: 'all-MiniLM-L6-v2',
|
|
54
|
+
dimensions: 384,
|
|
55
|
+
sentenceTransformers: { base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, pathPrefix: '' },
|
|
56
|
+
},
|
|
57
|
+
},
|
|
58
|
+
},
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
const fakeDaemon = {
|
|
62
|
+
baseUrl: RESOLVED_BASE_URL,
|
|
63
|
+
stdoutLog: '/work/proj/.ktx/runtime/daemon.stdout.log',
|
|
64
|
+
stderrLog: '/work/proj/.ktx/runtime/daemon.stderr.log',
|
|
65
|
+
};
|
|
66
|
+
describe('projectNeedsManagedLocalEmbeddings', () => {
|
|
67
|
+
it('returns false when neither ingest nor scan embeddings reference the managed sentinel', () => {
|
|
68
|
+
expect(projectNeedsManagedLocalEmbeddings(buildDefaultKtxProjectConfig())).toBe(false);
|
|
69
|
+
});
|
|
70
|
+
it('returns true when ingest.embeddings uses the managed sentinel', () => {
|
|
71
|
+
expect(projectNeedsManagedLocalEmbeddings(withManagedIngestEmbedding(buildDefaultKtxProjectConfig()))).toBe(true);
|
|
72
|
+
});
|
|
73
|
+
it('returns true when scan.enrichment.embeddings uses the managed sentinel', () => {
|
|
74
|
+
expect(projectNeedsManagedLocalEmbeddings(withManagedScanEnrichmentEmbedding(buildDefaultKtxProjectConfig()))).toBe(true);
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
describe('substituteManagedLocalEmbeddingsUrl', () => {
|
|
78
|
+
it('rewrites the managed sentinel in both ingest.embeddings and scan.enrichment.embeddings', () => {
|
|
79
|
+
const config = withManagedScanEnrichmentEmbedding(withManagedIngestEmbedding(buildDefaultKtxProjectConfig()));
|
|
80
|
+
const resolved = substituteManagedLocalEmbeddingsUrl(config, RESOLVED_BASE_URL);
|
|
81
|
+
expect(resolved.ingest.embeddings.sentenceTransformers?.base_url).toBe(RESOLVED_BASE_URL);
|
|
82
|
+
expect(resolved.scan.enrichment.embeddings?.sentenceTransformers?.base_url).toBe(RESOLVED_BASE_URL);
|
|
83
|
+
});
|
|
84
|
+
it('returns the input unchanged when no sentinel is present', () => {
|
|
85
|
+
const config = buildDefaultKtxProjectConfig();
|
|
86
|
+
const resolved = substituteManagedLocalEmbeddingsUrl(config, RESOLVED_BASE_URL);
|
|
87
|
+
expect(resolved.ingest.embeddings).toEqual(config.ingest.embeddings);
|
|
88
|
+
expect(resolved.scan.enrichment.embeddings).toEqual(config.scan.enrichment.embeddings);
|
|
89
|
+
});
|
|
90
|
+
it('does not touch non-sentinel sentence-transformers URLs', () => {
|
|
91
|
+
const config = {
|
|
92
|
+
...buildDefaultKtxProjectConfig(),
|
|
93
|
+
ingest: {
|
|
94
|
+
...buildDefaultKtxProjectConfig().ingest,
|
|
95
|
+
embeddings: {
|
|
96
|
+
backend: 'sentence-transformers',
|
|
97
|
+
model: 'all-MiniLM-L6-v2',
|
|
98
|
+
dimensions: 384,
|
|
99
|
+
sentenceTransformers: { base_url: 'http://localhost:9999', pathPrefix: '' },
|
|
100
|
+
},
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
const resolved = substituteManagedLocalEmbeddingsUrl(config, RESOLVED_BASE_URL);
|
|
104
|
+
expect(resolved.ingest.embeddings.sentenceTransformers?.base_url).toBe('http://localhost:9999');
|
|
105
|
+
});
|
|
106
|
+
});
|
|
107
|
+
describe('loadKtxCliProject', () => {
|
|
108
|
+
it('returns the project unchanged and does not start the daemon when no sentinel is present', async () => {
|
|
109
|
+
const io = makeIo();
|
|
110
|
+
const project = projectWithConfig(buildDefaultKtxProjectConfig());
|
|
111
|
+
const loadProject = vi.fn(async () => project);
|
|
112
|
+
const ensureLocalEmbeddings = vi.fn(async () => fakeDaemon);
|
|
113
|
+
const result = await loadKtxCliProject({ projectDir: '/work/proj', cliVersion: '0.2.0', installPolicy: 'never', io: io.io }, { loadProject, ensureLocalEmbeddings });
|
|
114
|
+
expect(result).toBe(project);
|
|
115
|
+
expect(ensureLocalEmbeddings).not.toHaveBeenCalled();
|
|
116
|
+
});
|
|
117
|
+
it('starts the daemon and substitutes the resolved URL when ingest.embeddings uses the sentinel', async () => {
|
|
118
|
+
const io = makeIo();
|
|
119
|
+
const project = projectWithConfig(withManagedIngestEmbedding(buildDefaultKtxProjectConfig()));
|
|
120
|
+
const loadProject = vi.fn(async () => project);
|
|
121
|
+
const ensureLocalEmbeddings = vi.fn(async () => fakeDaemon);
|
|
122
|
+
const result = await loadKtxCliProject({ projectDir: '/work/proj', cliVersion: '0.2.0', installPolicy: 'never', io: io.io }, { loadProject, ensureLocalEmbeddings });
|
|
123
|
+
expect(ensureLocalEmbeddings).toHaveBeenCalledWith({
|
|
124
|
+
cliVersion: '0.2.0',
|
|
125
|
+
projectDir: '/work/proj',
|
|
126
|
+
installPolicy: 'never',
|
|
127
|
+
io: io.io,
|
|
128
|
+
});
|
|
129
|
+
expect(result.config.ingest.embeddings.sentenceTransformers?.base_url).toBe(RESOLVED_BASE_URL);
|
|
130
|
+
});
|
|
131
|
+
it('does not mutate process.env', async () => {
|
|
132
|
+
const io = makeIo();
|
|
133
|
+
const before = process.env.KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL;
|
|
134
|
+
delete process.env.KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL;
|
|
135
|
+
try {
|
|
136
|
+
const project = projectWithConfig(withManagedIngestEmbedding(buildDefaultKtxProjectConfig()));
|
|
137
|
+
await loadKtxCliProject({ projectDir: '/work/proj', cliVersion: '0.2.0', installPolicy: 'never', io: io.io }, { loadProject: vi.fn(async () => project), ensureLocalEmbeddings: vi.fn(async () => fakeDaemon) });
|
|
138
|
+
expect(process.env.KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL).toBeUndefined();
|
|
139
|
+
}
|
|
140
|
+
finally {
|
|
141
|
+
if (before === undefined) {
|
|
142
|
+
delete process.env.KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL;
|
|
143
|
+
}
|
|
144
|
+
else {
|
|
145
|
+
process.env.KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL = before;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
});
|
|
149
|
+
});
|
package/dist/cli-runtime.d.ts
CHANGED
|
@@ -11,8 +11,6 @@ import type { KtxTextIngestArgs } from './text-ingest.js';
|
|
|
11
11
|
export interface KtxCliPackageInfo {
|
|
12
12
|
name: string;
|
|
13
13
|
version: string;
|
|
14
|
-
packageVersion: string;
|
|
15
|
-
runtimeVersion: string;
|
|
16
14
|
contextPackageName: '@ktx/context';
|
|
17
15
|
}
|
|
18
16
|
export interface KtxCliIo {
|
package/dist/cli-runtime.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createRequire } from 'node:module';
|
|
2
2
|
import { profileMark, profileSpan } from './startup-profile.js';
|
|
3
|
-
import {
|
|
3
|
+
import { assertCliVersion } from './release-version.js';
|
|
4
4
|
profileMark('module:cli-runtime');
|
|
5
5
|
const requirePackageJson = createRequire(import.meta.url);
|
|
6
6
|
export function getKtxCliPackageInfo() {
|
|
@@ -15,15 +15,9 @@ export function packageInfoFromJson(packageJson) {
|
|
|
15
15
|
typeof packageJson.version !== 'string') {
|
|
16
16
|
throw new Error('Invalid KTX CLI package metadata');
|
|
17
17
|
}
|
|
18
|
-
const runtimeVersion = resolveKtxRuntimeVersion({
|
|
19
|
-
packageName: packageJson.name,
|
|
20
|
-
packageVersion: packageJson.version,
|
|
21
|
-
});
|
|
22
18
|
return {
|
|
23
19
|
name: packageJson.name,
|
|
24
|
-
version:
|
|
25
|
-
packageVersion: packageJson.version,
|
|
26
|
-
runtimeVersion,
|
|
20
|
+
version: assertCliVersion(packageJson.version, `${packageJson.name}/package.json`),
|
|
27
21
|
contextPackageName: '@ktx/context',
|
|
28
22
|
};
|
|
29
23
|
}
|
|
@@ -30,7 +30,7 @@ export function registerRuntimeCommands(program, context) {
|
|
|
30
30
|
});
|
|
31
31
|
runtime
|
|
32
32
|
.command('start')
|
|
33
|
-
.description('Start the KTX
|
|
33
|
+
.description('Start the KTX daemon')
|
|
34
34
|
.addOption(createRuntimeFeatureOption())
|
|
35
35
|
.option('--force', 'Restart even when a matching daemon is already running', false)
|
|
36
36
|
.action(async (options, command) => {
|
|
@@ -44,7 +44,7 @@ export function registerRuntimeCommands(program, context) {
|
|
|
44
44
|
});
|
|
45
45
|
runtime
|
|
46
46
|
.command('stop')
|
|
47
|
-
.description('Stop the KTX
|
|
47
|
+
.description('Stop the KTX daemon')
|
|
48
48
|
.option('--all', 'Stop all KTX daemon processes recorded or discoverable on this machine', false)
|
|
49
49
|
.action(async (options, command) => {
|
|
50
50
|
await runRuntimeArgs(context, {
|
|
@@ -532,7 +532,7 @@ function failedStepDetail(result) {
|
|
|
532
532
|
return result.steps.find((step) => step.status === 'failed')?.detail ?? null;
|
|
533
533
|
}
|
|
534
534
|
const INTERNAL_FAILURE_LINE_RE = /^(Report|Run|Job|Status|Adapter|Connection|Sync|Mode|Dry run|Diff|Tasks|Work units|Failed tasks|Saved memory|Provenance rows):\s*/;
|
|
535
|
-
const ACTIONABLE_FAILURE_LINE_RE = /^(Missing bundled Python runtime manifest|KTX Python runtime is required|KTX
|
|
535
|
+
const ACTIONABLE_FAILURE_LINE_RE = /^(Missing bundled Python runtime manifest|KTX Python runtime is required|KTX daemon HTTP|Error:|Failed\b|Could not\b|Cannot\b)/;
|
|
536
536
|
function trimErrorPrefix(line) {
|
|
537
537
|
return line.replace(/^Error:\s*/, '');
|
|
538
538
|
}
|
package/dist/index.test.js
CHANGED
|
@@ -31,9 +31,7 @@ describe('getKtxCliPackageInfo', () => {
|
|
|
31
31
|
it('identifies the CLI package and its context dependency', () => {
|
|
32
32
|
expect(getKtxCliPackageInfo()).toEqual({
|
|
33
33
|
name: '@ktx/cli',
|
|
34
|
-
version: '0.
|
|
35
|
-
packageVersion: '0.0.0-private',
|
|
36
|
-
runtimeVersion: '0.1.0-rc.1',
|
|
34
|
+
version: '0.0.0-private',
|
|
37
35
|
contextPackageName: '@ktx/context',
|
|
38
36
|
});
|
|
39
37
|
});
|
|
@@ -51,8 +49,6 @@ describe('getKtxCliPackageInfo', () => {
|
|
|
51
49
|
})).toEqual({
|
|
52
50
|
name: '@kaelio/ktx',
|
|
53
51
|
version: '0.1.0',
|
|
54
|
-
packageVersion: '0.1.0',
|
|
55
|
-
runtimeVersion: '0.1.0',
|
|
56
52
|
contextPackageName: '@ktx/context',
|
|
57
53
|
});
|
|
58
54
|
});
|
|
@@ -90,7 +86,7 @@ describe('runKtxCli', () => {
|
|
|
90
86
|
it('prints version information', async () => {
|
|
91
87
|
const testIo = makeIo();
|
|
92
88
|
await expect(runKtxCli(['--version'], testIo.io)).resolves.toBe(0);
|
|
93
|
-
expect(testIo.stdout()).toBe('@ktx/cli 0.
|
|
89
|
+
expect(testIo.stdout()).toBe('@ktx/cli 0.0.0-private\n');
|
|
94
90
|
expect(testIo.stderr()).toBe('');
|
|
95
91
|
});
|
|
96
92
|
it('prints the public command surface in root help', async () => {
|
|
@@ -192,7 +188,7 @@ describe('runKtxCli', () => {
|
|
|
192
188
|
await expect(runKtxCli(['--project-dir', tempDir, 'sl', '--query', 'revenue'], unknownIo.io, { sl })).resolves.toBe(1);
|
|
193
189
|
expect(unknownIo.stderr()).toContain("unknown option '--query'");
|
|
194
190
|
});
|
|
195
|
-
it('routes runtime management commands with the
|
|
191
|
+
it('routes runtime management commands with the CLI package version', async () => {
|
|
196
192
|
const runtime = vi.fn(async () => 0);
|
|
197
193
|
const installIo = makeIo();
|
|
198
194
|
const startIo = makeIo();
|
|
@@ -210,32 +206,32 @@ describe('runKtxCli', () => {
|
|
|
210
206
|
await expect(runKtxCli(['admin', 'runtime', 'prune', '--dry-run'], pruneIo.io, { runtime })).resolves.toBe(1);
|
|
211
207
|
expect(runtime).toHaveBeenNthCalledWith(1, {
|
|
212
208
|
command: 'install',
|
|
213
|
-
cliVersion: '0.
|
|
209
|
+
cliVersion: '0.0.0-private',
|
|
214
210
|
feature: 'local-embeddings',
|
|
215
211
|
force: true,
|
|
216
212
|
}, installIo.io);
|
|
217
213
|
expect(runtime).toHaveBeenNthCalledWith(2, {
|
|
218
214
|
command: 'start',
|
|
219
|
-
cliVersion: '0.
|
|
215
|
+
cliVersion: '0.0.0-private',
|
|
220
216
|
projectDir: expect.any(String),
|
|
221
217
|
feature: 'local-embeddings',
|
|
222
218
|
force: true,
|
|
223
219
|
}, startIo.io);
|
|
224
220
|
expect(runtime).toHaveBeenNthCalledWith(3, {
|
|
225
221
|
command: 'stop',
|
|
226
|
-
cliVersion: '0.
|
|
222
|
+
cliVersion: '0.0.0-private',
|
|
227
223
|
projectDir: expect.any(String),
|
|
228
224
|
all: false,
|
|
229
225
|
}, stopIo.io);
|
|
230
226
|
expect(runtime).toHaveBeenNthCalledWith(4, {
|
|
231
227
|
command: 'stop',
|
|
232
|
-
cliVersion: '0.
|
|
228
|
+
cliVersion: '0.0.0-private',
|
|
233
229
|
projectDir: expect.any(String),
|
|
234
230
|
all: true,
|
|
235
231
|
}, stopAllIo.io);
|
|
236
232
|
expect(runtime).toHaveBeenNthCalledWith(5, {
|
|
237
233
|
command: 'status',
|
|
238
|
-
cliVersion: '0.
|
|
234
|
+
cliVersion: '0.0.0-private',
|
|
239
235
|
json: true,
|
|
240
236
|
}, statusIo.io);
|
|
241
237
|
expect(runtime).toHaveBeenCalledTimes(5);
|
|
@@ -282,7 +278,7 @@ describe('runKtxCli', () => {
|
|
|
282
278
|
expect(sl).toHaveBeenLastCalledWith(expect.objectContaining({
|
|
283
279
|
command: 'query',
|
|
284
280
|
projectDir: tempDir,
|
|
285
|
-
cliVersion: '0.
|
|
281
|
+
cliVersion: '0.0.0-private',
|
|
286
282
|
runtimeInstallPolicy: 'prompt',
|
|
287
283
|
query: expect.objectContaining({ measures: ['orders.order_count'], dimensions: [] }),
|
|
288
284
|
}), promptIo.io);
|
|
@@ -291,13 +287,13 @@ describe('runKtxCli', () => {
|
|
|
291
287
|
sl,
|
|
292
288
|
})).resolves.toBe(0);
|
|
293
289
|
expect(sl).toHaveBeenLastCalledWith(expect.objectContaining({
|
|
294
|
-
cliVersion: '0.
|
|
290
|
+
cliVersion: '0.0.0-private',
|
|
295
291
|
runtimeInstallPolicy: 'auto',
|
|
296
292
|
}), autoIo.io);
|
|
297
293
|
const noInputIo = makeIo();
|
|
298
294
|
await expect(runKtxCli(['--project-dir', tempDir, 'sl', 'query', '--measure', 'orders.order_count', '--no-input'], noInputIo.io, { sl })).resolves.toBe(0);
|
|
299
295
|
expect(sl).toHaveBeenLastCalledWith(expect.objectContaining({
|
|
300
|
-
cliVersion: '0.
|
|
296
|
+
cliVersion: '0.0.0-private',
|
|
301
297
|
runtimeInstallPolicy: 'never',
|
|
302
298
|
}), noInputIo.io);
|
|
303
299
|
});
|
|
@@ -407,7 +403,7 @@ describe('runKtxCli', () => {
|
|
|
407
403
|
skipAgents: false,
|
|
408
404
|
inputMode: 'auto',
|
|
409
405
|
yes: false,
|
|
410
|
-
cliVersion: '0.
|
|
406
|
+
cliVersion: '0.0.0-private',
|
|
411
407
|
skipLlm: false,
|
|
412
408
|
skipEmbeddings: false,
|
|
413
409
|
databaseSchemas: [],
|
|
@@ -516,7 +512,7 @@ describe('runKtxCli', () => {
|
|
|
516
512
|
inputMode: 'disabled',
|
|
517
513
|
depth: 'fast',
|
|
518
514
|
queryHistory: 'default',
|
|
519
|
-
cliVersion: '0.
|
|
515
|
+
cliVersion: '0.0.0-private',
|
|
520
516
|
runtimeInstallPolicy: 'never',
|
|
521
517
|
}, testIo.io);
|
|
522
518
|
expect(testIo.stderr()).toBe(`Project: ${tempDir}\n`);
|
|
@@ -535,7 +531,7 @@ describe('runKtxCli', () => {
|
|
|
535
531
|
inputMode: 'auto',
|
|
536
532
|
depth: 'deep',
|
|
537
533
|
queryHistory: 'default',
|
|
538
|
-
cliVersion: '0.
|
|
534
|
+
cliVersion: '0.0.0-private',
|
|
539
535
|
runtimeInstallPolicy: 'prompt',
|
|
540
536
|
}, testIo.io);
|
|
541
537
|
expect(testIo.stderr()).toBe('');
|
|
@@ -584,7 +580,7 @@ describe('runKtxCli', () => {
|
|
|
584
580
|
json: false,
|
|
585
581
|
inputMode: 'disabled',
|
|
586
582
|
queryHistory: 'default',
|
|
587
|
-
cliVersion: '0.
|
|
583
|
+
cliVersion: '0.0.0-private',
|
|
588
584
|
runtimeInstallPolicy: 'never',
|
|
589
585
|
}, testIo.io);
|
|
590
586
|
});
|
|
@@ -802,7 +798,7 @@ describe('runKtxCli', () => {
|
|
|
802
798
|
command: 'run',
|
|
803
799
|
projectDir: tempDir,
|
|
804
800
|
inputMode: 'disabled',
|
|
805
|
-
cliVersion: '0.
|
|
801
|
+
cliVersion: '0.0.0-private',
|
|
806
802
|
anthropicApiKeyEnv: 'ANTHROPIC_API_KEY', // pragma: allowlist secret
|
|
807
803
|
llmModel: 'claude-sonnet-4-6',
|
|
808
804
|
skipLlm: false,
|
|
@@ -829,7 +825,7 @@ describe('runKtxCli', () => {
|
|
|
829
825
|
command: 'run',
|
|
830
826
|
projectDir: tempDir,
|
|
831
827
|
inputMode: 'disabled',
|
|
832
|
-
cliVersion: '0.
|
|
828
|
+
cliVersion: '0.0.0-private',
|
|
833
829
|
llmBackend: 'vertex',
|
|
834
830
|
vertexProject: 'local-gcp-project',
|
|
835
831
|
vertexLocation: 'us-east5',
|
|
@@ -854,7 +850,7 @@ describe('runKtxCli', () => {
|
|
|
854
850
|
command: 'run',
|
|
855
851
|
projectDir: tempDir,
|
|
856
852
|
inputMode: 'disabled',
|
|
857
|
-
cliVersion: '0.
|
|
853
|
+
cliVersion: '0.0.0-private',
|
|
858
854
|
llmBackend: 'claude-code',
|
|
859
855
|
llmModel: 'opus',
|
|
860
856
|
skipLlm: false,
|
|
@@ -929,7 +925,7 @@ describe('runKtxCli', () => {
|
|
|
929
925
|
projectDir: '/tmp/project',
|
|
930
926
|
inputMode: 'disabled',
|
|
931
927
|
yes: true,
|
|
932
|
-
cliVersion: '0.
|
|
928
|
+
cliVersion: '0.0.0-private',
|
|
933
929
|
skipLlm: true,
|
|
934
930
|
skipEmbeddings: true,
|
|
935
931
|
databaseDrivers: ['postgres'],
|
|
@@ -1147,7 +1143,7 @@ describe('runKtxCli', () => {
|
|
|
1147
1143
|
queryFile: '/tmp/query.json',
|
|
1148
1144
|
execute: false,
|
|
1149
1145
|
format: 'json',
|
|
1150
|
-
cliVersion: '0.
|
|
1146
|
+
cliVersion: '0.0.0-private',
|
|
1151
1147
|
runtimeInstallPolicy: 'auto',
|
|
1152
1148
|
}, autoIo.io);
|
|
1153
1149
|
expect(sl).toHaveBeenNthCalledWith(2, {
|
|
@@ -1157,7 +1153,7 @@ describe('runKtxCli', () => {
|
|
|
1157
1153
|
queryFile: '/tmp/query.json',
|
|
1158
1154
|
execute: false,
|
|
1159
1155
|
format: 'json',
|
|
1160
|
-
cliVersion: '0.
|
|
1156
|
+
cliVersion: '0.0.0-private',
|
|
1161
1157
|
runtimeInstallPolicy: 'never',
|
|
1162
1158
|
}, neverIo.io);
|
|
1163
1159
|
expect(conflictIo.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input');
|
package/dist/ingest.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { buildMemoryFlowViewModel, createMemoryFlowLiveBuffer, formatMemoryFlowFinalSummary, getLatestLocalIngestStatus, getLocalIngestStatus, ingestReportToMemoryFlowReplay, renderMemoryFlowReplay, runLocalIngest, runLocalMetabaseIngest, savedMemoryCountsForReport, sanitizeMemoryFlowError, } from '@ktx/context/ingest';
|
|
2
|
-
import {
|
|
2
|
+
import { loadKtxCliProject } from './cli-project.js';
|
|
3
3
|
import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js';
|
|
4
4
|
import { readIngestReportSnapshotFile } from './ingest-report-file.js';
|
|
5
5
|
import { createCliOperationalLogger } from './io/logger.js';
|
|
@@ -470,7 +470,14 @@ async function writeReportRecord(report, outputMode, io, options = {}) {
|
|
|
470
470
|
}
|
|
471
471
|
export async function runKtxIngest(args, io = process, deps = {}) {
|
|
472
472
|
try {
|
|
473
|
-
const
|
|
473
|
+
const cliVersion = args.command === 'run' ? args.cliVersion : undefined;
|
|
474
|
+
const runtimeInstallPolicy = args.command === 'run' ? args.runtimeInstallPolicy : undefined;
|
|
475
|
+
const project = await loadKtxCliProject({
|
|
476
|
+
projectDir: args.projectDir,
|
|
477
|
+
cliVersion: cliVersion ?? '0.0.0-private',
|
|
478
|
+
installPolicy: runtimeInstallPolicy ?? 'never',
|
|
479
|
+
io,
|
|
480
|
+
});
|
|
474
481
|
const env = deps.env ?? process.env;
|
|
475
482
|
if (args.command === 'run') {
|
|
476
483
|
const ingestProject = args.allowImplicitAdapter && !project.config.ingest.adapters.includes(args.adapter)
|
package/dist/ingest.test.js
CHANGED
|
@@ -807,9 +807,16 @@ describe('runKtxIngest', () => {
|
|
|
807
807
|
sourceKey: 'historic-sql',
|
|
808
808
|
body: {
|
|
809
809
|
workUnits: [],
|
|
810
|
-
|
|
810
|
+
finalization: {
|
|
811
811
|
sourceKey: 'historic-sql',
|
|
812
812
|
status: 'success',
|
|
813
|
+
commitSha: 'finalization-sha',
|
|
814
|
+
touchedPaths: ['semantic-layer/warehouse/_schema/public.yaml', 'wiki/global/historic-sql-orders.md'],
|
|
815
|
+
declaredTouchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
|
|
816
|
+
derivedTouchedSources: [{ connectionId: 'warehouse', sourceName: 'orders' }],
|
|
817
|
+
declaredChangedWikiPageKeys: ['historic-sql-orders'],
|
|
818
|
+
derivedChangedWikiPageKeys: ['historic-sql-orders'],
|
|
819
|
+
mismatches: [],
|
|
813
820
|
result: {
|
|
814
821
|
tableUsageMerged: 56,
|
|
815
822
|
staleTablesMarked: 1,
|
|
@@ -819,7 +826,24 @@ describe('runKtxIngest', () => {
|
|
|
819
826
|
},
|
|
820
827
|
errors: [],
|
|
821
828
|
warnings: [],
|
|
822
|
-
|
|
829
|
+
actions: [
|
|
830
|
+
...Array.from({ length: 57 }, (_, index) => ({
|
|
831
|
+
target: 'sl',
|
|
832
|
+
type: 'updated',
|
|
833
|
+
key: `orders-${index}`,
|
|
834
|
+
detail: 'Merged usage',
|
|
835
|
+
targetConnectionId: 'warehouse',
|
|
836
|
+
rawPaths: ['tables/public/orders.json'],
|
|
837
|
+
})),
|
|
838
|
+
...Array.from({ length: 35 }, (_, index) => ({
|
|
839
|
+
target: 'wiki',
|
|
840
|
+
type: 'updated',
|
|
841
|
+
key: `historic-sql-orders-${index}`,
|
|
842
|
+
detail: 'Projected pattern',
|
|
843
|
+
rawPaths: ['patterns/orders.json'],
|
|
844
|
+
})),
|
|
845
|
+
],
|
|
846
|
+
provenanceExclusions: [],
|
|
823
847
|
},
|
|
824
848
|
},
|
|
825
849
|
}),
|
|
@@ -1048,7 +1072,7 @@ describe('runKtxIngest', () => {
|
|
|
1048
1072
|
}),
|
|
1049
1073
|
}));
|
|
1050
1074
|
});
|
|
1051
|
-
it('passes
|
|
1075
|
+
it('passes KTX daemon options to adapters and pull-config options when no explicit daemon URL is set', async () => {
|
|
1052
1076
|
const projectDir = join(tempDir, 'managed-daemon-ingest-project');
|
|
1053
1077
|
await initKtxProject({ projectDir });
|
|
1054
1078
|
await writeWarehouseConfig(projectDir);
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV } from '@ktx/context';
|
|
2
1
|
import type { KtxProjectEmbeddingConfig } from '@ktx/context/project';
|
|
3
2
|
import type { KtxEmbeddingConfig } from '@ktx/llm';
|
|
4
3
|
import type { KtxCliIo } from './cli-runtime.js';
|
|
@@ -8,7 +7,6 @@ export interface ManagedLocalEmbeddingsDaemon {
|
|
|
8
7
|
baseUrl: string;
|
|
9
8
|
stdoutLog: string;
|
|
10
9
|
stderrLog: string;
|
|
11
|
-
env: Record<typeof MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV, string>;
|
|
12
10
|
}
|
|
13
11
|
export interface ManagedLocalEmbeddingsOptions {
|
|
14
12
|
cliVersion: string;
|