@kaelio/ktx 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.3.0-py3-none-any.whl → kaelio_ktx-0.4.1-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/admin-reindex.js +9 -14
- package/dist/admin-reindex.test.js +4 -1
- package/dist/cli-project.d.ts +4 -10
- package/dist/cli-project.js +5 -49
- package/dist/cli-project.test.js +4 -131
- package/dist/commands/knowledge-commands.js +2 -0
- package/dist/commands/sl-commands.js +2 -0
- package/dist/embedding-resolution.d.ts +36 -0
- package/dist/embedding-resolution.js +63 -0
- package/dist/embedding-resolution.test.d.ts +1 -0
- package/dist/embedding-resolution.test.js +132 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/index.test.js +36 -33
- package/dist/ingest.js +12 -9
- package/dist/knowledge.d.ts +7 -2
- package/dist/knowledge.js +21 -7
- package/dist/knowledge.test.js +53 -9
- package/dist/managed-local-embeddings.d.ts +7 -6
- package/dist/managed-local-embeddings.js +19 -13
- package/dist/managed-local-embeddings.test.js +87 -18
- package/dist/mcp-server-factory.js +11 -0
- package/dist/public-ingest.js +2 -8
- package/dist/runtime-requirements.js +1 -2
- package/dist/runtime-requirements.test.js +1 -2
- package/dist/scan.js +8 -4
- package/dist/setup-embeddings.js +6 -2
- package/dist/setup-embeddings.test.js +2 -5
- package/dist/setup-runtime.test.js +1 -3
- package/dist/sl.d.ts +6 -4
- package/dist/sl.js +23 -9
- package/dist/sl.test.js +77 -6
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.d.ts +2 -0
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.js +2 -2
- package/node_modules/@ktx/context/dist/ingest/local-ingest.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/local-ingest.js +2 -0
- package/node_modules/@ktx/context/dist/llm/index.d.ts +1 -1
- package/node_modules/@ktx/context/dist/llm/index.js +1 -1
- package/node_modules/@ktx/context/dist/llm/local-config.d.ts +0 -1
- package/node_modules/@ktx/context/dist/llm/local-config.js +1 -2
- package/node_modules/@ktx/context/dist/llm/local-config.test.js +3 -3
- package/node_modules/@ktx/context/dist/mcp/local-project-ports.d.ts +2 -2
- package/node_modules/@ktx/context/dist/mcp/local-project-ports.js +2 -5
- package/node_modules/@ktx/context/dist/mcp/local-project-ports.test.js +14 -10
- package/node_modules/@ktx/context/dist/package-exports.test.js +0 -1
- package/node_modules/@ktx/context/dist/project/config.js +1 -1
- package/node_modules/@ktx/context/dist/scan/local-enrichment.test.js +4 -5
- package/node_modules/@ktx/context/dist/scan/local-scan.d.ts +3 -1
- package/node_modules/@ktx/context/dist/scan/local-scan.js +3 -2
- package/package.json +1 -1
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { describe, expect, it, vi } from 'vitest';
|
|
2
|
-
import {
|
|
3
|
-
import { ensureManagedLocalEmbeddingsDaemon, managedLocalEmbeddingHealthConfig, managedLocalEmbeddingProjectConfig, } from './managed-local-embeddings.js';
|
|
2
|
+
import { ensureManagedLocalEmbeddingsDaemon, managedLocalEmbeddingHealthConfig, tryUseManagedLocalEmbeddingsDaemon, } from './managed-local-embeddings.js';
|
|
4
3
|
function makeIo() {
|
|
5
4
|
let stdout = '';
|
|
6
5
|
let stderr = '';
|
|
@@ -84,22 +83,6 @@ function daemonResult(status = 'reused') {
|
|
|
84
83
|
},
|
|
85
84
|
};
|
|
86
85
|
}
|
|
87
|
-
describe('managedLocalEmbeddingProjectConfig', () => {
|
|
88
|
-
it('uses a stable managed runtime marker instead of a random daemon port', () => {
|
|
89
|
-
expect(managedLocalEmbeddingProjectConfig({
|
|
90
|
-
model: 'all-MiniLM-L6-v2',
|
|
91
|
-
dimensions: 384,
|
|
92
|
-
})).toEqual({
|
|
93
|
-
backend: 'sentence-transformers',
|
|
94
|
-
model: 'all-MiniLM-L6-v2',
|
|
95
|
-
dimensions: 384,
|
|
96
|
-
sentenceTransformers: {
|
|
97
|
-
base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
|
|
98
|
-
pathPrefix: '',
|
|
99
|
-
},
|
|
100
|
-
});
|
|
101
|
-
});
|
|
102
|
-
});
|
|
103
86
|
describe('managedLocalEmbeddingHealthConfig', () => {
|
|
104
87
|
it('uses the active KTX daemon URL for the immediate health check', () => {
|
|
105
88
|
expect(managedLocalEmbeddingHealthConfig({
|
|
@@ -158,3 +141,89 @@ describe('ensureManagedLocalEmbeddingsDaemon', () => {
|
|
|
158
141
|
expect(io.stderr()).toContain('Using KTX daemon: http://127.0.0.1:61234');
|
|
159
142
|
});
|
|
160
143
|
});
|
|
144
|
+
describe('tryUseManagedLocalEmbeddingsDaemon', () => {
|
|
145
|
+
it('returns the daemon when one is running and healthy', async () => {
|
|
146
|
+
const readStatus = vi.fn(async () => ({
|
|
147
|
+
kind: 'running',
|
|
148
|
+
detail: 'ok',
|
|
149
|
+
layout: {},
|
|
150
|
+
state: {
|
|
151
|
+
schemaVersion: 1,
|
|
152
|
+
pid: 123,
|
|
153
|
+
host: '127.0.0.1',
|
|
154
|
+
port: 4321,
|
|
155
|
+
version: '0.5.0',
|
|
156
|
+
features: ['local-embeddings'],
|
|
157
|
+
startedAt: '2026-05-21T00:00:00Z',
|
|
158
|
+
stdoutLog: '/tmp/stdout.log',
|
|
159
|
+
stderrLog: '/tmp/stderr.log',
|
|
160
|
+
},
|
|
161
|
+
baseUrl: 'http://127.0.0.1:4321',
|
|
162
|
+
}));
|
|
163
|
+
const result = await tryUseManagedLocalEmbeddingsDaemon({
|
|
164
|
+
cliVersion: '0.5.0',
|
|
165
|
+
projectDir: '/work/proj',
|
|
166
|
+
readStatus,
|
|
167
|
+
});
|
|
168
|
+
expect(result).toEqual({
|
|
169
|
+
baseUrl: 'http://127.0.0.1:4321',
|
|
170
|
+
stdoutLog: '/tmp/stdout.log',
|
|
171
|
+
stderrLog: '/tmp/stderr.log',
|
|
172
|
+
});
|
|
173
|
+
expect(readStatus).toHaveBeenCalledWith({
|
|
174
|
+
cliVersion: '0.5.0',
|
|
175
|
+
projectDir: '/work/proj',
|
|
176
|
+
});
|
|
177
|
+
});
|
|
178
|
+
it('returns null when no daemon state exists', async () => {
|
|
179
|
+
const readStatus = vi.fn(async () => ({
|
|
180
|
+
kind: 'stopped',
|
|
181
|
+
detail: 'no state',
|
|
182
|
+
layout: {},
|
|
183
|
+
}));
|
|
184
|
+
const result = await tryUseManagedLocalEmbeddingsDaemon({
|
|
185
|
+
cliVersion: '0.5.0',
|
|
186
|
+
projectDir: '/work/proj',
|
|
187
|
+
readStatus,
|
|
188
|
+
});
|
|
189
|
+
expect(result).toBeNull();
|
|
190
|
+
});
|
|
191
|
+
it('returns null when daemon is stale', async () => {
|
|
192
|
+
const readStatus = vi.fn(async () => ({
|
|
193
|
+
kind: 'stale',
|
|
194
|
+
detail: 'process gone',
|
|
195
|
+
layout: {},
|
|
196
|
+
}));
|
|
197
|
+
const result = await tryUseManagedLocalEmbeddingsDaemon({
|
|
198
|
+
cliVersion: '0.5.0',
|
|
199
|
+
projectDir: '/work/proj',
|
|
200
|
+
readStatus,
|
|
201
|
+
});
|
|
202
|
+
expect(result).toBeNull();
|
|
203
|
+
});
|
|
204
|
+
it('rejects daemons that do not advertise local-embeddings', async () => {
|
|
205
|
+
const readStatus = vi.fn(async () => ({
|
|
206
|
+
kind: 'running',
|
|
207
|
+
detail: 'ok',
|
|
208
|
+
layout: {},
|
|
209
|
+
state: {
|
|
210
|
+
schemaVersion: 1,
|
|
211
|
+
pid: 123,
|
|
212
|
+
host: '127.0.0.1',
|
|
213
|
+
port: 4321,
|
|
214
|
+
version: '0.5.0',
|
|
215
|
+
features: ['core'],
|
|
216
|
+
startedAt: '2026-05-21T00:00:00Z',
|
|
217
|
+
stdoutLog: '/tmp/stdout.log',
|
|
218
|
+
stderrLog: '/tmp/stderr.log',
|
|
219
|
+
},
|
|
220
|
+
baseUrl: 'http://127.0.0.1:4321',
|
|
221
|
+
}));
|
|
222
|
+
const result = await tryUseManagedLocalEmbeddingsDaemon({
|
|
223
|
+
cliVersion: '0.5.0',
|
|
224
|
+
projectDir: '/work/proj',
|
|
225
|
+
readStatus,
|
|
226
|
+
});
|
|
227
|
+
expect(result).toBeNull();
|
|
228
|
+
});
|
|
229
|
+
});
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import { KtxIngestEmbeddingPortAdapter } from '@ktx/context';
|
|
1
2
|
import { createDefaultKtxMcpServer, createLocalProjectMcpContextPorts } from '@ktx/context/mcp';
|
|
2
3
|
import { createLocalProjectMemoryIngest } from '@ktx/context/memory';
|
|
4
|
+
import { resolveProjectEmbeddingProvider } from './embedding-resolution.js';
|
|
3
5
|
import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js';
|
|
4
6
|
import { createKtxCliScanConnector } from './local-scan-connectors.js';
|
|
5
7
|
import { createManagedPythonSemanticLayerComputePort } from './managed-python-command.js';
|
|
@@ -24,10 +26,19 @@ export async function createKtxMcpServerFactory(input) {
|
|
|
24
26
|
installPolicy: 'auto',
|
|
25
27
|
io,
|
|
26
28
|
});
|
|
29
|
+
const resolution = await resolveProjectEmbeddingProvider(input.project, {
|
|
30
|
+
mode: 'use-if-running',
|
|
31
|
+
cliVersion: input.cliVersion,
|
|
32
|
+
io,
|
|
33
|
+
});
|
|
34
|
+
const embeddingService = resolution.kind === 'configured' || resolution.kind === 'managed-running' || resolution.kind === 'managed-started'
|
|
35
|
+
? new KtxIngestEmbeddingPortAdapter(resolution.provider)
|
|
36
|
+
: null;
|
|
27
37
|
const contextTools = createLocalProjectMcpContextPorts(input.project, {
|
|
28
38
|
semanticLayerCompute,
|
|
29
39
|
queryExecutor,
|
|
30
40
|
sqlAnalysis,
|
|
41
|
+
embeddingService,
|
|
31
42
|
localScan: {
|
|
32
43
|
createConnector: async (connectionId) => createKtxCliScanConnector(input.project, connectionId),
|
|
33
44
|
},
|
package/dist/public-ingest.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { loadKtxProject } from '@ktx/context/project';
|
|
2
2
|
import { databaseContextDepth, deepReadinessGaps, isDatabaseDriver, normalizeConnectionDriver, } from './ingest-depth.js';
|
|
3
3
|
import { ensureManagedPythonCommandRuntime, } from './managed-python-command.js';
|
|
4
4
|
import { publicIngestOutputLine } from './public-ingest-copy.js';
|
|
@@ -536,13 +536,7 @@ export async function executePublicIngestTarget(target, args, io, deps) {
|
|
|
536
536
|
return markTargetResult(target, args, exitCode === 0 ? 'done' : 'failed', 'source-ingest', capturedIngestIo ? capturedFailureMessage(capturedIngestIo.capturedOutput()) : undefined);
|
|
537
537
|
}
|
|
538
538
|
export async function runKtxPublicIngest(args, io, deps = {}) {
|
|
539
|
-
const loadProject = deps.loadProject ??
|
|
540
|
-
((options) => loadKtxCliProject({
|
|
541
|
-
projectDir: options.projectDir,
|
|
542
|
-
cliVersion: args.cliVersion ?? '0.0.0-private',
|
|
543
|
-
installPolicy: args.runtimeInstallPolicy ?? 'never',
|
|
544
|
-
io,
|
|
545
|
-
}));
|
|
539
|
+
const loadProject = deps.loadProject ?? ((options) => loadKtxProject({ projectDir: options.projectDir }));
|
|
546
540
|
const project = await loadProject({ projectDir: args.projectDir });
|
|
547
541
|
if (shouldUseForegroundContextBuildView(args, io)) {
|
|
548
542
|
const plan = buildPublicIngestPlan(project, args);
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
|
|
2
1
|
function normalizeDriver(driver) {
|
|
3
2
|
return String(driver ?? '').trim().toLowerCase();
|
|
4
3
|
}
|
|
@@ -22,7 +21,7 @@ function requiresManagedLocalEmbeddings(embeddings) {
|
|
|
22
21
|
return false;
|
|
23
22
|
}
|
|
24
23
|
const baseUrl = embeddings.sentenceTransformers?.base_url;
|
|
25
|
-
return baseUrl === undefined || baseUrl === ''
|
|
24
|
+
return baseUrl === undefined || baseUrl === '';
|
|
26
25
|
}
|
|
27
26
|
function uniqueRequirements(requirements) {
|
|
28
27
|
const seen = new Set();
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
|
|
2
1
|
import { buildDefaultKtxProjectConfig } from '@ktx/context/project';
|
|
3
2
|
import { describe, expect, it } from 'vitest';
|
|
4
3
|
import { resolveProjectRuntimeRequirements, resolvePublicIngestRuntimeRequirements, } from './runtime-requirements.js';
|
|
@@ -37,7 +36,7 @@ describe('runtime requirement detection', () => {
|
|
|
37
36
|
model: 'all-MiniLM-L6-v2',
|
|
38
37
|
dimensions: 384,
|
|
39
38
|
sentenceTransformers: {
|
|
40
|
-
base_url:
|
|
39
|
+
base_url: '',
|
|
41
40
|
},
|
|
42
41
|
},
|
|
43
42
|
},
|
package/dist/scan.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { runLocalScan, } from '@ktx/context/scan';
|
|
2
|
-
import {
|
|
2
|
+
import { loadKtxProject } from '@ktx/context/project';
|
|
3
|
+
import { resolveProjectEmbeddingProvider } from './embedding-resolution.js';
|
|
3
4
|
import { createKtxCliLocalIngestAdapters } from './local-adapters.js';
|
|
4
5
|
import { createKtxCliScanConnector } from './local-scan-connectors.js';
|
|
5
6
|
import { profileMark } from './startup-profile.js';
|
|
@@ -237,12 +238,14 @@ export function createCliScanProgress(io, state = { progress: 0, hasPendingTrans
|
|
|
237
238
|
}
|
|
238
239
|
export async function runKtxScan(args, io = process, deps = {}) {
|
|
239
240
|
try {
|
|
240
|
-
const project = await
|
|
241
|
-
|
|
242
|
-
|
|
241
|
+
const project = await loadKtxProject({ projectDir: args.projectDir });
|
|
242
|
+
const resolution = await resolveProjectEmbeddingProvider(project, {
|
|
243
|
+
mode: 'ensure',
|
|
243
244
|
installPolicy: args.runtimeInstallPolicy ?? 'never',
|
|
245
|
+
cliVersion: args.cliVersion ?? '0.0.0-private',
|
|
244
246
|
io,
|
|
245
247
|
});
|
|
248
|
+
const embeddingProvider = resolution.kind === 'disabled' || resolution.kind === 'managed-unavailable' ? null : resolution.provider;
|
|
246
249
|
const managedDaemon = managedDaemonOptionsForScanRun(args, deps.runtimeIo ?? io);
|
|
247
250
|
const connector = args.mode !== 'structural' || args.detectRelationships
|
|
248
251
|
? await createKtxCliScanConnector(project, args.connectionId)
|
|
@@ -259,6 +262,7 @@ export async function runKtxScan(args, io = process, deps = {}) {
|
|
|
259
262
|
trigger: 'cli',
|
|
260
263
|
databaseIntrospectionUrl: args.databaseIntrospectionUrl,
|
|
261
264
|
connector,
|
|
265
|
+
embeddingProvider,
|
|
262
266
|
adapters: (deps.createLocalIngestAdapters ?? createKtxCliLocalIngestAdapters)(project, {
|
|
263
267
|
...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}),
|
|
264
268
|
...(managedDaemon ? { managedDaemon } : {}),
|
package/dist/setup-embeddings.js
CHANGED
|
@@ -3,7 +3,7 @@ import { resolveKtxConfigReference } from '@ktx/context/core';
|
|
|
3
3
|
import { loadKtxProject, markKtxSetupStateStepComplete, readKtxSetupState, serializeKtxProjectConfig, } from '@ktx/context/project';
|
|
4
4
|
import { runKtxEmbeddingHealthCheck } from '@ktx/llm';
|
|
5
5
|
import { createStaticCliSpinner } from './clack.js';
|
|
6
|
-
import { ensureManagedLocalEmbeddingsDaemon, managedLocalEmbeddingHealthConfig,
|
|
6
|
+
import { ensureManagedLocalEmbeddingsDaemon, managedLocalEmbeddingHealthConfig, } from './managed-local-embeddings.js';
|
|
7
7
|
import { withTextInputNavigation } from './prompt-navigation.js';
|
|
8
8
|
import { envCredentialReference, writeProjectLocalSecretReference } from './setup-secrets.js';
|
|
9
9
|
import { createKtxSetupPromptAdapter, } from './setup-prompts.js';
|
|
@@ -332,7 +332,11 @@ export async function runKtxSetupEmbeddingsStep(args, io, deps = {}) {
|
|
|
332
332
|
if (health.ok) {
|
|
333
333
|
progress.succeed(`Embedding test passed (${model}, ${dimensions} dimensions)`);
|
|
334
334
|
await persistEmbeddingConfig(args.projectDir, selectedBackend === LOCAL_EMBEDDING_BACKEND
|
|
335
|
-
?
|
|
335
|
+
? {
|
|
336
|
+
backend: 'sentence-transformers',
|
|
337
|
+
model,
|
|
338
|
+
dimensions,
|
|
339
|
+
}
|
|
336
340
|
: buildProjectEmbeddingConfig({
|
|
337
341
|
backend: selectedBackend,
|
|
338
342
|
model,
|
|
@@ -44,9 +44,6 @@ function managedDaemon(baseUrl = 'http://127.0.0.1:61234', logs = {}) {
|
|
|
44
44
|
baseUrl,
|
|
45
45
|
stdoutLog: logs.stdoutLog ?? '/tmp/ktx-daemon.stdout.log',
|
|
46
46
|
stderrLog: logs.stderrLog ?? '/tmp/ktx-daemon.stderr.log',
|
|
47
|
-
env: {
|
|
48
|
-
KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL: baseUrl,
|
|
49
|
-
},
|
|
50
47
|
};
|
|
51
48
|
}
|
|
52
49
|
describe('setup embeddings step', () => {
|
|
@@ -142,8 +139,8 @@ describe('setup embeddings step', () => {
|
|
|
142
139
|
backend: 'sentence-transformers',
|
|
143
140
|
model: 'all-MiniLM-L6-v2',
|
|
144
141
|
dimensions: 384,
|
|
145
|
-
sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' },
|
|
146
142
|
});
|
|
143
|
+
expect(config.ingest.embeddings.sentenceTransformers).toBeUndefined();
|
|
147
144
|
expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings);
|
|
148
145
|
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
|
|
149
146
|
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('embeddings');
|
|
@@ -216,8 +213,8 @@ describe('setup embeddings step', () => {
|
|
|
216
213
|
backend: 'sentence-transformers',
|
|
217
214
|
model: 'all-MiniLM-L6-v2',
|
|
218
215
|
dimensions: 384,
|
|
219
|
-
sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' },
|
|
220
216
|
});
|
|
217
|
+
expect(config.ingest.embeddings.sentenceTransformers).toBeUndefined();
|
|
221
218
|
expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings);
|
|
222
219
|
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
|
|
223
220
|
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('embeddings');
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { mkdtemp, rm } from 'node:fs/promises';
|
|
2
2
|
import { tmpdir } from 'node:os';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
|
-
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
|
|
5
4
|
import { buildDefaultKtxProjectConfig, readKtxSetupState } from '@ktx/context/project';
|
|
6
5
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
7
6
|
import { runKtxSetupRuntimeStep } from './setup-runtime.js';
|
|
@@ -79,7 +78,6 @@ describe('runKtxSetupRuntimeStep', () => {
|
|
|
79
78
|
baseUrl: 'http://127.0.0.1:61234',
|
|
80
79
|
stdoutLog: join(tempDir, '.ktx', 'runtime', 'daemon.stdout.log'),
|
|
81
80
|
stderrLog: join(tempDir, '.ktx', 'runtime', 'daemon.stderr.log'),
|
|
82
|
-
env: { KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL: 'http://127.0.0.1:61234' },
|
|
83
81
|
}));
|
|
84
82
|
const config = {
|
|
85
83
|
...buildDefaultKtxProjectConfig(),
|
|
@@ -89,7 +87,7 @@ describe('runKtxSetupRuntimeStep', () => {
|
|
|
89
87
|
backend: 'sentence-transformers',
|
|
90
88
|
model: 'all-MiniLM-L6-v2',
|
|
91
89
|
dimensions: 384,
|
|
92
|
-
sentenceTransformers: { base_url:
|
|
90
|
+
sentenceTransformers: { base_url: '' },
|
|
93
91
|
},
|
|
94
92
|
},
|
|
95
93
|
};
|
package/dist/sl.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { type KtxSqlQueryExecutorPort } from '@ktx/context/connections';
|
|
2
|
-
import { createLocalKtxEmbeddingProviderFromConfig, type KtxEmbeddingPort } from '@ktx/context';
|
|
3
2
|
import type { KtxSemanticLayerComputePort } from '@ktx/context/daemon';
|
|
4
3
|
import { loadKtxProject } from '@ktx/context/project';
|
|
5
|
-
import { type SemanticLayerQueryInput } from '@ktx/context/sl';
|
|
4
|
+
import { searchLocalSlSources as defaultSearchLocalSlSources, type SemanticLayerQueryInput } from '@ktx/context/sl';
|
|
5
|
+
import { resolveProjectEmbeddingProvider } from './embedding-resolution.js';
|
|
6
6
|
import { type KtxManagedPythonInstallPolicy } from './managed-python-command.js';
|
|
7
7
|
type SlQueryFormat = 'json' | 'sql';
|
|
8
8
|
export type KtxSlArgs = {
|
|
@@ -11,6 +11,7 @@ export type KtxSlArgs = {
|
|
|
11
11
|
connectionId?: string;
|
|
12
12
|
output?: string;
|
|
13
13
|
json?: boolean;
|
|
14
|
+
cliVersion: string;
|
|
14
15
|
} | {
|
|
15
16
|
command: 'search';
|
|
16
17
|
projectDir: string;
|
|
@@ -19,6 +20,7 @@ export type KtxSlArgs = {
|
|
|
19
20
|
limit?: number;
|
|
20
21
|
output?: string;
|
|
21
22
|
json?: boolean;
|
|
23
|
+
cliVersion: string;
|
|
22
24
|
} | {
|
|
23
25
|
command: 'validate';
|
|
24
26
|
projectDir: string;
|
|
@@ -46,8 +48,8 @@ interface KtxSlIo {
|
|
|
46
48
|
}
|
|
47
49
|
interface KtxSlDeps {
|
|
48
50
|
loadProject?: typeof loadKtxProject;
|
|
49
|
-
|
|
50
|
-
|
|
51
|
+
resolveEmbeddingProvider?: typeof resolveProjectEmbeddingProvider;
|
|
52
|
+
searchLocalSlSources?: typeof defaultSearchLocalSlSources;
|
|
51
53
|
createSemanticLayerCompute?: () => KtxSemanticLayerComputePort;
|
|
52
54
|
createManagedSemanticLayerCompute?: (options: {
|
|
53
55
|
cliVersion: string;
|
package/dist/sl.js
CHANGED
|
@@ -1,17 +1,19 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises';
|
|
2
2
|
import { createDefaultLocalQueryExecutor } from '@ktx/context/connections';
|
|
3
|
-
import {
|
|
3
|
+
import { KtxIngestEmbeddingPortAdapter } from '@ktx/context';
|
|
4
4
|
import { loadKtxProject } from '@ktx/context/project';
|
|
5
|
-
import { compileLocalSlQuery, listLocalSlSources, readLocalSlSource, searchLocalSlSources, validateLocalSlSource, } from '@ktx/context/sl';
|
|
5
|
+
import { compileLocalSlQuery, listLocalSlSources, readLocalSlSource, searchLocalSlSources as defaultSearchLocalSlSources, validateLocalSlSource, } from '@ktx/context/sl';
|
|
6
|
+
import { resolveProjectEmbeddingProvider, } from './embedding-resolution.js';
|
|
6
7
|
import { createManagedPythonSemanticLayerComputePort, } from './managed-python-command.js';
|
|
7
8
|
import { profileMark } from './startup-profile.js';
|
|
8
9
|
profileMark('module:sl');
|
|
9
|
-
function
|
|
10
|
-
if ('
|
|
11
|
-
|
|
10
|
+
function resolutionToEmbeddingPort(resolution) {
|
|
11
|
+
if (resolution.kind === 'configured' ||
|
|
12
|
+
resolution.kind === 'managed-running' ||
|
|
13
|
+
resolution.kind === 'managed-started') {
|
|
14
|
+
return new KtxIngestEmbeddingPortAdapter(resolution.provider);
|
|
12
15
|
}
|
|
13
|
-
|
|
14
|
-
return provider ? new KtxIngestEmbeddingPortAdapter(provider) : null;
|
|
16
|
+
return null;
|
|
15
17
|
}
|
|
16
18
|
async function printSlSources(input) {
|
|
17
19
|
const { resolveOutputMode } = await import('./io/mode.js');
|
|
@@ -90,12 +92,24 @@ export async function runKtxSl(args, io = process, deps = {}) {
|
|
|
90
92
|
return 0;
|
|
91
93
|
}
|
|
92
94
|
if (args.command === 'search') {
|
|
93
|
-
const
|
|
95
|
+
const resolver = deps.resolveEmbeddingProvider ?? resolveProjectEmbeddingProvider;
|
|
96
|
+
const resolution = await resolver(project, {
|
|
97
|
+
mode: 'use-if-running',
|
|
98
|
+
cliVersion: args.cliVersion,
|
|
99
|
+
io,
|
|
100
|
+
});
|
|
101
|
+
const embeddingService = resolutionToEmbeddingPort(resolution);
|
|
102
|
+
const search = deps.searchLocalSlSources ?? defaultSearchLocalSlSources;
|
|
103
|
+
const sources = await search(project, {
|
|
94
104
|
connectionId: args.connectionId,
|
|
95
105
|
query: args.query,
|
|
96
|
-
embeddingService
|
|
106
|
+
embeddingService,
|
|
97
107
|
limit: args.limit,
|
|
98
108
|
});
|
|
109
|
+
if (sources.length === 0 && resolution.kind === 'managed-unavailable' && !args.json) {
|
|
110
|
+
const { SYMBOLS } = await import('./io/symbols.js');
|
|
111
|
+
io.stderr.write(`embeddings: unavailable ${SYMBOLS.emDash} ${resolution.reason}\n`);
|
|
112
|
+
}
|
|
99
113
|
await printSlSources({
|
|
100
114
|
rows: sources,
|
|
101
115
|
emptyMessage: `No semantic-layer sources matched "${args.query}" in ${project.projectDir}`,
|
package/dist/sl.test.js
CHANGED
|
@@ -55,10 +55,17 @@ describe('runKtxSl', () => {
|
|
|
55
55
|
await expect(runKtxSl({ command: 'validate', projectDir, connectionId: 'warehouse', sourceName: 'orders' }, validateIo.io)).resolves.toBe(0);
|
|
56
56
|
expect(validateIo.stdout()).toContain('Valid semantic-layer source: warehouse/orders');
|
|
57
57
|
const listIo = makeIo();
|
|
58
|
-
await expect(runKtxSl({ command: 'list', projectDir, connectionId: 'warehouse' }, listIo.io)).resolves.toBe(0);
|
|
58
|
+
await expect(runKtxSl({ command: 'list', projectDir, connectionId: 'warehouse', cliVersion: '0.0.0-test' }, listIo.io)).resolves.toBe(0);
|
|
59
59
|
expect(listIo.stdout()).toContain('warehouse\torders\tcolumns=1\tmeasures=0\tjoins=0');
|
|
60
60
|
const searchIo = makeIo();
|
|
61
|
-
await expect(runKtxSl({
|
|
61
|
+
await expect(runKtxSl({
|
|
62
|
+
command: 'search',
|
|
63
|
+
projectDir,
|
|
64
|
+
connectionId: 'warehouse',
|
|
65
|
+
query: 'order',
|
|
66
|
+
json: true,
|
|
67
|
+
cliVersion: '0.0.0-test',
|
|
68
|
+
}, searchIo.io)).resolves.toBe(0);
|
|
62
69
|
expect(JSON.parse(searchIo.stdout())).toMatchObject({
|
|
63
70
|
kind: 'list',
|
|
64
71
|
data: {
|
|
@@ -77,7 +84,14 @@ describe('runKtxSl', () => {
|
|
|
77
84
|
const projectDir = join(tempDir, 'rank-project');
|
|
78
85
|
await seedSlSource({ projectDir });
|
|
79
86
|
const searchIo = makeIo();
|
|
80
|
-
await expect(runKtxSl({
|
|
87
|
+
await expect(runKtxSl({
|
|
88
|
+
command: 'search',
|
|
89
|
+
projectDir,
|
|
90
|
+
connectionId: 'warehouse',
|
|
91
|
+
query: 'order',
|
|
92
|
+
output: 'pretty',
|
|
93
|
+
cliVersion: '0.0.0-test',
|
|
94
|
+
}, searchIo.io)).resolves.toBe(0);
|
|
81
95
|
const stdout = stripVTControlCharacters(searchIo.stdout());
|
|
82
96
|
expect(stdout).toMatch(/#1\s+orders/);
|
|
83
97
|
expect(stdout).not.toContain('%');
|
|
@@ -99,7 +113,14 @@ describe('runKtxSl', () => {
|
|
|
99
113
|
].join('\n'),
|
|
100
114
|
});
|
|
101
115
|
const listIo = makeIo();
|
|
102
|
-
await expect(runKtxSl({
|
|
116
|
+
await expect(runKtxSl({
|
|
117
|
+
command: 'search',
|
|
118
|
+
projectDir,
|
|
119
|
+
connectionId: 'warehouse',
|
|
120
|
+
query: 'paid',
|
|
121
|
+
json: true,
|
|
122
|
+
cliVersion: '0.0.0-test',
|
|
123
|
+
}, listIo.io)).resolves.toBe(0);
|
|
103
124
|
expect(JSON.parse(listIo.stdout())).toMatchObject({
|
|
104
125
|
kind: 'list',
|
|
105
126
|
data: {
|
|
@@ -425,7 +446,7 @@ joins: []
|
|
|
425
446
|
const projectDir = join(tempDir, 'project');
|
|
426
447
|
await seedSlSource({ projectDir });
|
|
427
448
|
const listIo = makeIo();
|
|
428
|
-
const code = await runKtxSl({ command: 'list', projectDir, connectionId: 'warehouse', output: 'json' }, listIo.io);
|
|
449
|
+
const code = await runKtxSl({ command: 'list', projectDir, connectionId: 'warehouse', output: 'json', cliVersion: '0.0.0-test' }, listIo.io);
|
|
429
450
|
expect(code).toBe(0);
|
|
430
451
|
expect(listIo.stderr()).toBe('');
|
|
431
452
|
const parsed = JSON.parse(listIo.stdout());
|
|
@@ -447,11 +468,61 @@ joins: []
|
|
|
447
468
|
joinCount: 0,
|
|
448
469
|
});
|
|
449
470
|
});
|
|
471
|
+
it('search prints embeddings status when results are empty', async () => {
|
|
472
|
+
const stderr = [];
|
|
473
|
+
const io = {
|
|
474
|
+
stdout: { write: (_chunk) => { } },
|
|
475
|
+
stderr: {
|
|
476
|
+
write: (chunk) => {
|
|
477
|
+
stderr.push(chunk);
|
|
478
|
+
},
|
|
479
|
+
},
|
|
480
|
+
};
|
|
481
|
+
const projectDir = join(tempDir, 'empty-status');
|
|
482
|
+
const project = await initKtxProject({ projectDir });
|
|
483
|
+
await expect(runKtxSl({
|
|
484
|
+
command: 'search',
|
|
485
|
+
projectDir: project.projectDir,
|
|
486
|
+
query: 'nope',
|
|
487
|
+
cliVersion: '0.5.0',
|
|
488
|
+
}, io, {
|
|
489
|
+
loadProject: async () => project,
|
|
490
|
+
resolveEmbeddingProvider: async () => ({
|
|
491
|
+
kind: 'managed-unavailable',
|
|
492
|
+
reason: 'managed embeddings daemon is not running',
|
|
493
|
+
}),
|
|
494
|
+
searchLocalSlSources: async () => [],
|
|
495
|
+
})).resolves.toBe(0);
|
|
496
|
+
expect(stderr.join('')).toMatch(/embeddings: unavailable/);
|
|
497
|
+
expect(stderr.join('')).toMatch(/managed embeddings daemon is not running/);
|
|
498
|
+
});
|
|
499
|
+
it('passes a managed-daemon-backed embedding service into the search', async () => {
|
|
500
|
+
const projectDir = join(tempDir, 'resolver-project');
|
|
501
|
+
const project = await initKtxProject({ projectDir });
|
|
502
|
+
const search = vi.fn(async () => []);
|
|
503
|
+
const searchIo = makeIo();
|
|
504
|
+
await expect(runKtxSl({
|
|
505
|
+
command: 'search',
|
|
506
|
+
projectDir: project.projectDir,
|
|
507
|
+
query: 'income',
|
|
508
|
+
cliVersion: '0.5.0',
|
|
509
|
+
json: true,
|
|
510
|
+
}, searchIo.io, {
|
|
511
|
+
loadProject: async () => project,
|
|
512
|
+
resolveEmbeddingProvider: async () => ({
|
|
513
|
+
kind: 'managed-running',
|
|
514
|
+
provider: { id: 'fake' },
|
|
515
|
+
baseUrl: 'http://127.0.0.1:51234',
|
|
516
|
+
}),
|
|
517
|
+
searchLocalSlSources: search,
|
|
518
|
+
})).resolves.toBe(0);
|
|
519
|
+
expect(search).toHaveBeenCalledWith(project, expect.objectContaining({ embeddingService: expect.any(Object) }));
|
|
520
|
+
});
|
|
450
521
|
it('emits sl list with grouping and Clack-style framing when output=pretty', async () => {
|
|
451
522
|
const projectDir = join(tempDir, 'project');
|
|
452
523
|
await seedSlSource({ projectDir });
|
|
453
524
|
const listIo = makeIo();
|
|
454
|
-
const code = await runKtxSl({ command: 'list', projectDir, connectionId: 'warehouse', output: 'pretty' }, listIo.io);
|
|
525
|
+
const code = await runKtxSl({ command: 'list', projectDir, connectionId: 'warehouse', output: 'pretty', cliVersion: '0.0.0-test' }, listIo.io);
|
|
455
526
|
expect(code).toBe(0);
|
|
456
527
|
const stripAnsi = (s) => s.replace(/\[[0-9;]*m/g, '');
|
|
457
528
|
const out = stripAnsi(listIo.stdout());
|
|
@@ -2,6 +2,7 @@ import { type KtxSqlQueryExecutorPort } from '../connections/index.js';
|
|
|
2
2
|
import type { KtxLogger } from '../core/index.js';
|
|
3
3
|
import type { KtxSemanticLayerComputePort } from '../daemon/index.js';
|
|
4
4
|
import { createLocalKtxLlmRuntimeFromConfig, type AgentRunnerPort, type KtxLlmRuntimePort } from '../llm/index.js';
|
|
5
|
+
import type { KtxEmbeddingProvider } from '@ktx/llm';
|
|
5
6
|
import type { KtxLocalProject } from '../project/index.js';
|
|
6
7
|
import { SqliteContextEvidenceStore } from './context-evidence/index.js';
|
|
7
8
|
import { IngestBundleRunner } from './ingest-bundle.runner.js';
|
|
@@ -20,6 +21,7 @@ export interface CreateLocalBundleIngestRuntimeOptions {
|
|
|
20
21
|
queryExecutor?: KtxSqlQueryExecutorPort;
|
|
21
22
|
jobIdFactory?: () => string;
|
|
22
23
|
logger?: KtxLogger;
|
|
24
|
+
embeddingProvider?: KtxEmbeddingProvider | null;
|
|
23
25
|
}
|
|
24
26
|
export interface LocalBundleIngestRuntime {
|
|
25
27
|
runner: IngestBundleRunner;
|
|
@@ -4,7 +4,7 @@ import { fileURLToPath } from 'node:url';
|
|
|
4
4
|
import YAML from 'yaml';
|
|
5
5
|
import { localConnectionInfoFromConfig } from '../connections/index.js';
|
|
6
6
|
import { noopLogger, SessionWorktreeService } from '../core/index.js';
|
|
7
|
-
import { createRuntimeToolDescriptorFromAiTool,
|
|
7
|
+
import { createRuntimeToolDescriptorFromAiTool, createLocalKtxLlmRuntimeFromConfig, KtxIngestEmbeddingPortAdapter, RuntimeAgentRunner, } from '../llm/index.js';
|
|
8
8
|
import { ktxLocalStateDbPath } from '../project/index.js';
|
|
9
9
|
import { PromptService } from '../prompts/index.js';
|
|
10
10
|
import { SkillsRegistryService } from '../skills/index.js';
|
|
@@ -490,7 +490,7 @@ export function createLocalBundleIngestRuntime(options) {
|
|
|
490
490
|
mkdirSync(join(options.project.projectDir, '.ktx/cache/local-ingest'), { recursive: true });
|
|
491
491
|
const store = new SqliteBundleIngestStore({ dbPath });
|
|
492
492
|
const contextStore = new SqliteContextEvidenceStore({ dbPath });
|
|
493
|
-
const embeddingProvider =
|
|
493
|
+
const embeddingProvider = options.embeddingProvider ?? null;
|
|
494
494
|
const embedding = embeddingProvider ? new KtxIngestEmbeddingPortAdapter(embeddingProvider) : new NoopEmbeddingPort();
|
|
495
495
|
const connections = new LocalConnectionCatalog(options.project, options.queryExecutor);
|
|
496
496
|
const rootFileStore = options.project.fileStore;
|
|
@@ -24,6 +24,7 @@ export interface RunLocalIngestOptions {
|
|
|
24
24
|
semanticLayerCompute?: KtxSemanticLayerComputePort;
|
|
25
25
|
queryExecutor?: KtxSqlQueryExecutorPort;
|
|
26
26
|
logger?: KtxLogger;
|
|
27
|
+
embeddingProvider?: import('@ktx/llm').KtxEmbeddingProvider | null;
|
|
27
28
|
}
|
|
28
29
|
export interface LocalIngestMcpOptions extends Pick<RunLocalIngestOptions, 'agentRunner' | 'llmRuntime' | 'memoryModel' | 'semanticLayerCompute' | 'queryExecutor' | 'logger' | 'pullConfigOptions'> {
|
|
29
30
|
adapters?: SourceAdapter[];
|
|
@@ -100,6 +100,7 @@ export async function runLocalIngest(options) {
|
|
|
100
100
|
semanticLayerCompute: options.semanticLayerCompute,
|
|
101
101
|
queryExecutor: options.queryExecutor,
|
|
102
102
|
logger: options.logger,
|
|
103
|
+
embeddingProvider: options.embeddingProvider,
|
|
103
104
|
});
|
|
104
105
|
}
|
|
105
106
|
const result = await runtime.runner.run({
|
|
@@ -252,6 +253,7 @@ export async function runLocalMetabaseIngest(options) {
|
|
|
252
253
|
semanticLayerCompute: options.semanticLayerCompute,
|
|
253
254
|
queryExecutor: options.queryExecutor,
|
|
254
255
|
logger: options.logger,
|
|
256
|
+
embeddingProvider: options.embeddingProvider,
|
|
255
257
|
});
|
|
256
258
|
}
|
|
257
259
|
catch (error) {
|
|
@@ -10,4 +10,4 @@ export { RuntimeAgentRunner } from './runtime-port.js';
|
|
|
10
10
|
export { createAiSdkToolSet, createClaudeSdkTools, createRuntimeToolDescriptorFromAiTool, createRuntimeToolSetFromAiSdkTools, normalizeKtxRuntimeToolOutput, } from './runtime-tools.js';
|
|
11
11
|
export type { KtxLlmDebugProviderOptionsEntry, KtxLlmDebugRequest, KtxLlmDebugRequestRecorder, SummarizeKtxLlmDebugRequestInput, } from './debug-request-recorder.js';
|
|
12
12
|
export { createJsonlKtxLlmDebugRequestRecorder, summarizeKtxLlmDebugRequest, } from './debug-request-recorder.js';
|
|
13
|
-
export {
|
|
13
|
+
export { createLocalKtxEmbeddingProviderFromConfig, createLocalKtxLlmProviderFromConfig, createLocalKtxLlmRuntimeFromConfig, resolveLocalKtxEmbeddingConfig, resolveLocalKtxLlmConfig, } from './local-config.js';
|
|
@@ -7,4 +7,4 @@ export { generateKtxObject, generateKtxText } from './generation.js';
|
|
|
7
7
|
export { RuntimeAgentRunner } from './runtime-port.js';
|
|
8
8
|
export { createAiSdkToolSet, createClaudeSdkTools, createRuntimeToolDescriptorFromAiTool, createRuntimeToolSetFromAiSdkTools, normalizeKtxRuntimeToolOutput, } from './runtime-tools.js';
|
|
9
9
|
export { createJsonlKtxLlmDebugRequestRecorder, summarizeKtxLlmDebugRequest, } from './debug-request-recorder.js';
|
|
10
|
-
export {
|
|
10
|
+
export { createLocalKtxEmbeddingProviderFromConfig, createLocalKtxLlmProviderFromConfig, createLocalKtxLlmRuntimeFromConfig, resolveLocalKtxEmbeddingConfig, resolveLocalKtxLlmConfig, } from './local-config.js';
|
|
@@ -12,7 +12,6 @@ interface LocalConfigDeps {
|
|
|
12
12
|
llmProvider: KtxLlmProvider;
|
|
13
13
|
}) => KtxLlmRuntimePort;
|
|
14
14
|
}
|
|
15
|
-
export declare const MANAGED_SENTENCE_TRANSFORMERS_BASE_URL = "managed:local-embeddings";
|
|
16
15
|
export declare function resolveLocalKtxLlmConfig(config: KtxProjectLlmConfig, env: NodeJS.ProcessEnv): KtxLlmConfig | null;
|
|
17
16
|
export declare function createLocalKtxLlmProviderFromConfig(config: KtxProjectLlmConfig, deps?: LocalConfigDeps): KtxLlmProvider | null;
|
|
18
17
|
export declare function createLocalKtxLlmRuntimeFromConfig(config: KtxProjectLlmConfig, deps?: LocalConfigDeps): KtxLlmRuntimePort | null;
|
|
@@ -2,7 +2,6 @@ import { createKtxEmbeddingProvider, createKtxLlmProvider, } from '@ktx/llm';
|
|
|
2
2
|
import { resolveKtxConfigReference } from '../core/config-reference.js';
|
|
3
3
|
import { AiSdkKtxLlmRuntime } from './ai-sdk-runtime.js';
|
|
4
4
|
import { ClaudeCodeKtxLlmRuntime } from './claude-code-runtime.js';
|
|
5
|
-
export const MANAGED_SENTENCE_TRANSFORMERS_BASE_URL = 'managed:local-embeddings';
|
|
6
5
|
function resolveOptional(value, env) {
|
|
7
6
|
return resolveKtxConfigReference(value, env) || undefined;
|
|
8
7
|
}
|
|
@@ -99,7 +98,7 @@ export function resolveLocalKtxEmbeddingConfig(config, env) {
|
|
|
99
98
|
}
|
|
100
99
|
if (config.backend === 'sentence-transformers') {
|
|
101
100
|
const baseURL = config.sentenceTransformers?.base_url;
|
|
102
|
-
if (!baseURL
|
|
101
|
+
if (!baseURL) {
|
|
103
102
|
return null;
|
|
104
103
|
}
|
|
105
104
|
return {
|