@kaelio/ktx 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/python/{kaelio_ktx-0.2.0-py3-none-any.whl → kaelio_ktx-0.4.0-py3-none-any.whl} +0 -0
- package/assets/python/manifest.json +4 -4
- package/dist/admin-reindex.js +10 -17
- package/dist/admin-reindex.test.js +1 -1
- package/dist/cli-program.test.js +0 -2
- package/dist/cli-project.d.ts +18 -0
- package/dist/cli-project.js +52 -0
- package/dist/cli-project.test.js +149 -0
- package/dist/cli-runtime.d.ts +0 -2
- package/dist/cli-runtime.js +2 -8
- package/dist/commands/runtime-commands.js +2 -2
- package/dist/context-build-view.js +1 -1
- package/dist/index.test.js +21 -25
- package/dist/ingest.js +9 -2
- package/dist/ingest.test.js +27 -3
- package/dist/managed-local-embeddings.d.ts +0 -2
- package/dist/managed-local-embeddings.js +2 -5
- package/dist/managed-local-embeddings.test.js +5 -8
- package/dist/managed-python-daemon.js +2 -2
- package/dist/managed-python-daemon.test.js +1 -1
- package/dist/managed-python-http.js +3 -3
- package/dist/managed-python-http.test.js +6 -6
- package/dist/print-command-tree.js +0 -2
- package/dist/public-ingest.d.ts +4 -2
- package/dist/public-ingest.js +9 -3
- package/dist/release-version.d.ts +1 -5
- package/dist/release-version.js +2 -39
- package/dist/runtime-requirements.js +1 -1
- package/dist/runtime.js +6 -6
- package/dist/runtime.test.js +7 -7
- package/dist/scan.js +7 -2
- package/dist/scan.test.js +1 -1
- package/dist/setup-embeddings.js +1 -1
- package/dist/setup-embeddings.test.js +2 -2
- package/dist/setup-runtime.test.js +1 -1
- package/node_modules/@ktx/context/dist/core/git.service.d.ts +1 -0
- package/node_modules/@ktx/context/dist/core/git.service.js +12 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.d.ts +2 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.js +18 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/local-ingest-acceptance.test.js +6 -6
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.d.ts +5 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.js +48 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/projection.test.js +83 -0
- package/node_modules/@ktx/context/dist/ingest/adapters/live-database/daemon-introspection.js +4 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/live-database/daemon-introspection.test.js +32 -0
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.d.ts +22 -0
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.js +95 -0
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.test.d.ts +1 -0
- package/node_modules/@ktx/context/dist/ingest/finalization-scope.test.js +114 -0
- package/node_modules/@ktx/context/dist/ingest/index.d.ts +1 -2
- package/node_modules/@ktx/context/dist/ingest/index.js +0 -1
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.d.ts +2 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.isolated-diff.test.js +166 -0
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.js +235 -45
- package/node_modules/@ktx/context/dist/ingest/ingest-bundle.runner.test.js +193 -38
- package/node_modules/@ktx/context/dist/ingest/local-bundle-ingest.test.js +22 -3
- package/node_modules/@ktx/context/dist/ingest/local-bundle-runtime.js +0 -4
- package/node_modules/@ktx/context/dist/ingest/local-ingest.js +0 -7
- package/node_modules/@ktx/context/dist/ingest/local-stage-ingest.js +15 -5
- package/node_modules/@ktx/context/dist/ingest/local-stage-ingest.test.js +29 -0
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.d.ts +2 -2
- package/node_modules/@ktx/context/dist/ingest/memory-flow/schema.js +1 -1
- package/node_modules/@ktx/context/dist/ingest/memory-flow/types.d.ts +1 -1
- package/node_modules/@ktx/context/dist/ingest/ports.d.ts +1 -20
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.d.ts +71 -0
- package/node_modules/@ktx/context/dist/ingest/report-snapshot.js +27 -0
- package/node_modules/@ktx/context/dist/ingest/reports.d.ts +23 -5
- package/node_modules/@ktx/context/dist/ingest/reports.js +7 -24
- package/node_modules/@ktx/context/dist/ingest/types.d.ts +33 -0
- package/node_modules/@ktx/context/dist/llm/index.d.ts +1 -1
- package/node_modules/@ktx/context/dist/llm/index.js +1 -1
- package/node_modules/@ktx/context/dist/llm/local-config.d.ts +0 -1
- package/node_modules/@ktx/context/dist/llm/local-config.js +2 -12
- package/node_modules/@ktx/context/dist/llm/local-config.test.js +2 -23
- package/node_modules/@ktx/context/dist/package-exports.test.js +2 -2
- package/node_modules/@ktx/context/dist/project/config.d.ts +16 -0
- package/node_modules/@ktx/context/dist/project/driver-schemas.d.ts +8 -0
- package/node_modules/@ktx/context/dist/project/driver-schemas.js +4 -0
- package/node_modules/@ktx/context/dist/scan/enabled-tables.d.ts +3 -0
- package/node_modules/@ktx/context/dist/scan/enabled-tables.js +15 -0
- package/node_modules/@ktx/context/dist/scan/local-scan.d.ts +2 -4
- package/node_modules/@ktx/context/dist/scan/local-scan.js +2 -15
- package/package.json +1 -1
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.d.ts +0 -4
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.js +0 -38
- package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.test.js +0 -63
- /package/{node_modules/@ktx/context/dist/ingest/adapters/historic-sql/post-processor.test.d.ts → dist/cli-project.test.d.ts} +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL
|
|
1
|
+
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
|
|
2
2
|
import { ensureManagedPythonCommandRuntime, } from './managed-python-command.js';
|
|
3
3
|
import { startManagedPythonDaemon } from './managed-python-daemon.js';
|
|
4
4
|
export function managedLocalEmbeddingProjectConfig(input) {
|
|
@@ -39,13 +39,10 @@ export async function ensureManagedLocalEmbeddingsDaemon(options) {
|
|
|
39
39
|
force: false,
|
|
40
40
|
});
|
|
41
41
|
const verb = daemon.status === 'started' ? 'Started' : 'Using';
|
|
42
|
-
options.io.stderr.write(`${verb} KTX
|
|
42
|
+
options.io.stderr.write(`${verb} KTX daemon: ${daemon.baseUrl}\n`);
|
|
43
43
|
return {
|
|
44
44
|
baseUrl: daemon.baseUrl,
|
|
45
45
|
stdoutLog: daemon.state.stdoutLog,
|
|
46
46
|
stderrLog: daemon.state.stderrLog,
|
|
47
|
-
env: {
|
|
48
|
-
[MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: daemon.baseUrl,
|
|
49
|
-
},
|
|
50
47
|
};
|
|
51
48
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, expect, it, vi } from 'vitest';
|
|
2
|
-
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL
|
|
2
|
+
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
|
|
3
3
|
import { ensureManagedLocalEmbeddingsDaemon, managedLocalEmbeddingHealthConfig, managedLocalEmbeddingProjectConfig, } from './managed-local-embeddings.js';
|
|
4
4
|
function makeIo() {
|
|
5
5
|
let stdout = '';
|
|
@@ -101,7 +101,7 @@ describe('managedLocalEmbeddingProjectConfig', () => {
|
|
|
101
101
|
});
|
|
102
102
|
});
|
|
103
103
|
describe('managedLocalEmbeddingHealthConfig', () => {
|
|
104
|
-
it('uses the active
|
|
104
|
+
it('uses the active KTX daemon URL for the immediate health check', () => {
|
|
105
105
|
expect(managedLocalEmbeddingHealthConfig({
|
|
106
106
|
baseUrl: 'http://127.0.0.1:61234',
|
|
107
107
|
model: 'all-MiniLM-L6-v2',
|
|
@@ -115,7 +115,7 @@ describe('managedLocalEmbeddingHealthConfig', () => {
|
|
|
115
115
|
});
|
|
116
116
|
});
|
|
117
117
|
describe('ensureManagedLocalEmbeddingsDaemon', () => {
|
|
118
|
-
it('ensures the local-embeddings feature and starts the
|
|
118
|
+
it('ensures the local-embeddings feature and starts the KTX daemon', async () => {
|
|
119
119
|
const io = makeIo();
|
|
120
120
|
const ensureRuntime = vi.fn(async () => runtime());
|
|
121
121
|
const startDaemon = vi.fn(async () => daemonResult('started'));
|
|
@@ -130,9 +130,6 @@ describe('ensureManagedLocalEmbeddingsDaemon', () => {
|
|
|
130
130
|
baseUrl: 'http://127.0.0.1:61234',
|
|
131
131
|
stdoutLog: '/work/proj/.ktx/runtime/daemon.stdout.log',
|
|
132
132
|
stderrLog: '/work/proj/.ktx/runtime/daemon.stderr.log',
|
|
133
|
-
env: {
|
|
134
|
-
[MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: 'http://127.0.0.1:61234',
|
|
135
|
-
},
|
|
136
133
|
});
|
|
137
134
|
expect(ensureRuntime).toHaveBeenCalledWith({
|
|
138
135
|
cliVersion: '0.2.0',
|
|
@@ -146,7 +143,7 @@ describe('ensureManagedLocalEmbeddingsDaemon', () => {
|
|
|
146
143
|
features: ['local-embeddings'],
|
|
147
144
|
force: false,
|
|
148
145
|
});
|
|
149
|
-
expect(io.stderr()).toContain('Started KTX
|
|
146
|
+
expect(io.stderr()).toContain('Started KTX daemon: http://127.0.0.1:61234');
|
|
150
147
|
});
|
|
151
148
|
it('reuses an already running daemon without reporting a new start', async () => {
|
|
152
149
|
const io = makeIo();
|
|
@@ -158,6 +155,6 @@ describe('ensureManagedLocalEmbeddingsDaemon', () => {
|
|
|
158
155
|
ensureRuntime: vi.fn(async () => runtime()),
|
|
159
156
|
startDaemon: vi.fn(async () => daemonResult('reused')),
|
|
160
157
|
});
|
|
161
|
-
expect(io.stderr()).toContain('Using KTX
|
|
158
|
+
expect(io.stderr()).toContain('Using KTX daemon: http://127.0.0.1:61234');
|
|
162
159
|
});
|
|
163
160
|
});
|
|
@@ -173,7 +173,7 @@ async function waitForHealth(input) {
|
|
|
173
173
|
return;
|
|
174
174
|
}
|
|
175
175
|
lastDetail = finalHealth.detail;
|
|
176
|
-
throw new Error(`KTX
|
|
176
|
+
throw new Error(`KTX daemon failed to start: ${lastDetail}. stderr: ${input.state.stderrLog}`);
|
|
177
177
|
}
|
|
178
178
|
async function removeState(layout) {
|
|
179
179
|
await rm(layout.daemonStatePath, { force: true });
|
|
@@ -491,7 +491,7 @@ export async function startManagedPythonDaemon(options) {
|
|
|
491
491
|
});
|
|
492
492
|
child.unref();
|
|
493
493
|
if (!child.pid) {
|
|
494
|
-
throw new Error(`KTX
|
|
494
|
+
throw new Error(`KTX daemon did not report a pid. stderr: ${layout.daemonStderrPath}`);
|
|
495
495
|
}
|
|
496
496
|
const state = {
|
|
497
497
|
schemaVersion: 1,
|
|
@@ -100,7 +100,7 @@ function daemonOptionsBase(root) {
|
|
|
100
100
|
runtimeRoot: join(root, 'runtime'),
|
|
101
101
|
};
|
|
102
102
|
}
|
|
103
|
-
describe('
|
|
103
|
+
describe('KTX daemon lifecycle', () => {
|
|
104
104
|
let tempDir;
|
|
105
105
|
beforeEach(async () => {
|
|
106
106
|
tempDir = await mkdtemp(join(tmpdir(), 'ktx-managed-daemon-'));
|
|
@@ -11,7 +11,7 @@ function normalizedBaseUrl(baseUrl) {
|
|
|
11
11
|
function parseJsonObject(raw, path) {
|
|
12
12
|
const parsed = JSON.parse(raw);
|
|
13
13
|
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
14
|
-
throw new Error(`KTX
|
|
14
|
+
throw new Error(`KTX daemon HTTP ${path} returned non-object JSON`);
|
|
15
15
|
}
|
|
16
16
|
return parsed;
|
|
17
17
|
}
|
|
@@ -34,7 +34,7 @@ export async function postManagedDaemonJson(baseUrl, path, payload) {
|
|
|
34
34
|
const text = Buffer.concat(chunks).toString('utf8');
|
|
35
35
|
const statusCode = response.statusCode ?? 0;
|
|
36
36
|
if (statusCode < 200 || statusCode >= 300) {
|
|
37
|
-
reject(new Error(`KTX
|
|
37
|
+
reject(new Error(`KTX daemon HTTP ${path} failed with ${statusCode}: ${text}`));
|
|
38
38
|
return;
|
|
39
39
|
}
|
|
40
40
|
try {
|
|
@@ -70,7 +70,7 @@ export function createManagedPythonDaemonBaseUrlResolver(options) {
|
|
|
70
70
|
force: false,
|
|
71
71
|
});
|
|
72
72
|
const verb = daemon.status === 'started' ? 'Started' : 'Using existing';
|
|
73
|
-
options.io.stderr.write(`${verb} KTX
|
|
73
|
+
options.io.stderr.write(`${verb} KTX daemon: ${daemon.baseUrl}\n`);
|
|
74
74
|
cachedBaseUrl = daemon.baseUrl;
|
|
75
75
|
return cachedBaseUrl;
|
|
76
76
|
};
|
|
@@ -47,7 +47,7 @@ describe('createManagedPythonDaemonBaseUrlResolver', () => {
|
|
|
47
47
|
features: ['core'],
|
|
48
48
|
force: false,
|
|
49
49
|
});
|
|
50
|
-
expect(testIo.stderr()).toContain('Started KTX
|
|
50
|
+
expect(testIo.stderr()).toContain('Started KTX daemon: http://127.0.0.1:61234');
|
|
51
51
|
});
|
|
52
52
|
it('reports daemon reuse without reinstalling after the first resolved URL', async () => {
|
|
53
53
|
const testIo = io();
|
|
@@ -73,7 +73,7 @@ describe('createManagedPythonDaemonBaseUrlResolver', () => {
|
|
|
73
73
|
await expect(resolveBaseUrl()).resolves.toBe('http://127.0.0.1:61234');
|
|
74
74
|
expect(ensureRuntime).toHaveBeenCalledTimes(1);
|
|
75
75
|
expect(startDaemon).toHaveBeenCalledTimes(1);
|
|
76
|
-
expect(testIo.stderr()).toContain('Using existing KTX
|
|
76
|
+
expect(testIo.stderr()).toContain('Using existing KTX daemon: http://127.0.0.1:61234');
|
|
77
77
|
});
|
|
78
78
|
});
|
|
79
79
|
describe('createManagedDaemonHttpJsonRunner', () => {
|
|
@@ -87,8 +87,8 @@ describe('createManagedDaemonHttpJsonRunner', () => {
|
|
|
87
87
|
expect(postJson).toHaveBeenCalledWith('http://127.0.0.1:61234', '/sql/parse-table-identifier', { items: [] });
|
|
88
88
|
});
|
|
89
89
|
});
|
|
90
|
-
describe('
|
|
91
|
-
it('creates a Looker table parser backed by the
|
|
90
|
+
describe('KTX daemon ingest ports', () => {
|
|
91
|
+
it('creates a Looker table parser backed by the KTX daemon runner', async () => {
|
|
92
92
|
const requestJson = vi.fn(async () => ({
|
|
93
93
|
results: {
|
|
94
94
|
'model.explore': {
|
|
@@ -114,7 +114,7 @@ describe('managed daemon ingest ports', () => {
|
|
|
114
114
|
items: [{ key: 'model.explore', sql_table_name: 'public.orders', dialect: 'postgres' }],
|
|
115
115
|
});
|
|
116
116
|
});
|
|
117
|
-
it('creates a SQL analysis port backed by the
|
|
117
|
+
it('creates a SQL analysis port backed by the KTX daemon runner', async () => {
|
|
118
118
|
const requestJson = vi.fn(async () => ({
|
|
119
119
|
fingerprint: 'select-orders',
|
|
120
120
|
normalized_sql: 'SELECT * FROM public.orders WHERE id = ?',
|
|
@@ -134,7 +134,7 @@ describe('managed daemon ingest ports', () => {
|
|
|
134
134
|
dialect: 'postgres',
|
|
135
135
|
});
|
|
136
136
|
});
|
|
137
|
-
it('routes SQL batch analysis through the
|
|
137
|
+
it('routes SQL batch analysis through the KTX daemon runner', async () => {
|
|
138
138
|
const requestJson = vi.fn(async () => ({
|
|
139
139
|
results: {
|
|
140
140
|
orders: {
|
package/dist/public-ingest.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type KtxLocalProject
|
|
1
|
+
import { type KtxLocalProject } from '@ktx/context/project';
|
|
2
2
|
import type { KtxProgressPort } from '@ktx/context/scan';
|
|
3
3
|
import type { KtxCliIo } from './index.js';
|
|
4
4
|
import type { KtxIngestArgs, KtxIngestDeps, KtxIngestProgressUpdate } from './ingest.js';
|
|
@@ -68,7 +68,9 @@ export interface KtxPublicIngestTargetResult {
|
|
|
68
68
|
export type KtxPublicIngestProject = Pick<KtxLocalProject, 'projectDir' | 'config'>;
|
|
69
69
|
type KtxPublicIngestPhaseKey = 'database-schema' | 'query-history' | 'source-ingest';
|
|
70
70
|
export interface KtxPublicIngestDeps {
|
|
71
|
-
loadProject?: (options:
|
|
71
|
+
loadProject?: (options: {
|
|
72
|
+
projectDir: string;
|
|
73
|
+
}) => Promise<KtxPublicIngestProject>;
|
|
72
74
|
runScan?: (args: KtxScanArgs, io: KtxCliIo, deps?: KtxScanDeps) => Promise<number>;
|
|
73
75
|
runIngest?: (args: KtxIngestArgs, io: KtxCliIo, deps?: KtxIngestDeps) => Promise<number>;
|
|
74
76
|
runContextBuild?: (project: KtxPublicIngestProject, args: KtxPublicContextBuildArgs, io: KtxCliIo) => Promise<{
|
package/dist/public-ingest.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { loadKtxCliProject } from './cli-project.js';
|
|
2
2
|
import { databaseContextDepth, deepReadinessGaps, isDatabaseDriver, normalizeConnectionDriver, } from './ingest-depth.js';
|
|
3
3
|
import { ensureManagedPythonCommandRuntime, } from './managed-python-command.js';
|
|
4
4
|
import { publicIngestOutputLine } from './public-ingest-copy.js';
|
|
@@ -391,7 +391,7 @@ function createCapturedPublicIngestIo() {
|
|
|
391
391
|
};
|
|
392
392
|
}
|
|
393
393
|
const INTERNAL_STATUS_LINE_RE = /^(Report|Run|Job|Status|Adapter|Connection|Sync|Diff|Tasks|Work units|Failed tasks|Saved memory|Provenance rows):\s*/;
|
|
394
|
-
const ACTIONABLE_FAILURE_LINE_RE = /^(Missing bundled Python runtime manifest|KTX Python runtime is required|KTX
|
|
394
|
+
const ACTIONABLE_FAILURE_LINE_RE = /^(Missing bundled Python runtime manifest|KTX Python runtime is required|KTX daemon HTTP|Error:|Failed\b|Could not\b|Cannot\b)/;
|
|
395
395
|
const RUNTIME_BACKED_RETRY_LINE_RE = /^Then retry the runtime-backed KTX command\.?$/;
|
|
396
396
|
function trimErrorPrefix(line) {
|
|
397
397
|
return line.replace(/^Error:\s*/, '');
|
|
@@ -536,7 +536,13 @@ export async function executePublicIngestTarget(target, args, io, deps) {
|
|
|
536
536
|
return markTargetResult(target, args, exitCode === 0 ? 'done' : 'failed', 'source-ingest', capturedIngestIo ? capturedFailureMessage(capturedIngestIo.capturedOutput()) : undefined);
|
|
537
537
|
}
|
|
538
538
|
export async function runKtxPublicIngest(args, io, deps = {}) {
|
|
539
|
-
const loadProject = deps.loadProject ??
|
|
539
|
+
const loadProject = deps.loadProject ??
|
|
540
|
+
((options) => loadKtxCliProject({
|
|
541
|
+
projectDir: options.projectDir,
|
|
542
|
+
cliVersion: args.cliVersion ?? '0.0.0-private',
|
|
543
|
+
installPolicy: args.runtimeInstallPolicy ?? 'never',
|
|
544
|
+
io,
|
|
545
|
+
}));
|
|
540
546
|
const project = await loadProject({ projectDir: args.projectDir });
|
|
541
547
|
if (shouldUseForegroundContextBuildView(args, io)) {
|
|
542
548
|
const plan = buildPublicIngestPlan(project, args);
|
package/dist/release-version.js
CHANGED
|
@@ -1,44 +1,7 @@
|
|
|
1
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
2
|
-
import { dirname, join, parse } from 'node:path';
|
|
3
|
-
import { fileURLToPath } from 'node:url';
|
|
4
1
|
const semverPattern = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/;
|
|
5
|
-
function
|
|
6
|
-
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
7
|
-
}
|
|
8
|
-
function assertReleaseVersion(value, source) {
|
|
2
|
+
export function assertCliVersion(value, source) {
|
|
9
3
|
if (typeof value !== 'string' || !semverPattern.test(value)) {
|
|
10
|
-
throw new Error(`Invalid KTX
|
|
4
|
+
throw new Error(`Invalid KTX CLI version in ${source}`);
|
|
11
5
|
}
|
|
12
6
|
return value;
|
|
13
7
|
}
|
|
14
|
-
function findReleasePolicyPath(startDir) {
|
|
15
|
-
let current = startDir;
|
|
16
|
-
const root = parse(current).root;
|
|
17
|
-
while (true) {
|
|
18
|
-
const candidate = join(current, 'release-policy.json');
|
|
19
|
-
if (existsSync(candidate)) {
|
|
20
|
-
return candidate;
|
|
21
|
-
}
|
|
22
|
-
if (current === root) {
|
|
23
|
-
return undefined;
|
|
24
|
-
}
|
|
25
|
-
current = dirname(current);
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
function readSourceReleaseVersion(startDir = dirname(fileURLToPath(import.meta.url))) {
|
|
29
|
-
const policyPath = findReleasePolicyPath(startDir);
|
|
30
|
-
if (!policyPath) {
|
|
31
|
-
return undefined;
|
|
32
|
-
}
|
|
33
|
-
const policy = JSON.parse(readFileSync(policyPath, 'utf8'));
|
|
34
|
-
if (!isPlainObject(policy)) {
|
|
35
|
-
throw new Error(`Invalid KTX release policy: ${policyPath}`);
|
|
36
|
-
}
|
|
37
|
-
return assertReleaseVersion(policy.publicNpmPackageVersion, policyPath);
|
|
38
|
-
}
|
|
39
|
-
export function resolveKtxRuntimeVersion(input) {
|
|
40
|
-
if (input.packageName === '@kaelio/ktx') {
|
|
41
|
-
return assertReleaseVersion(input.packageVersion, `${input.packageName}/package.json`);
|
|
42
|
-
}
|
|
43
|
-
return readSourceReleaseVersion(input.startDir) ?? input.packageVersion;
|
|
44
|
-
}
|
|
@@ -45,7 +45,7 @@ export function resolveProjectRuntimeRequirements(config, options = {}) {
|
|
|
45
45
|
requirements.push({
|
|
46
46
|
feature: 'core',
|
|
47
47
|
reason: 'database-introspection',
|
|
48
|
-
detail: 'Database introspection fallback uses the
|
|
48
|
+
detail: 'Database introspection fallback uses the KTX daemon.',
|
|
49
49
|
});
|
|
50
50
|
}
|
|
51
51
|
for (const [connectionId, connection] of Object.entries(config.connections)) {
|
package/dist/runtime.js
CHANGED
|
@@ -15,7 +15,7 @@ function writeInstallResult(io, result) {
|
|
|
15
15
|
}
|
|
16
16
|
function writeDaemonStart(io, result) {
|
|
17
17
|
const verb = result.status === 'reused' ? 'Using existing' : 'Started';
|
|
18
|
-
io.stdout.write(`${verb} KTX
|
|
18
|
+
io.stdout.write(`${verb} KTX daemon\n`);
|
|
19
19
|
io.stdout.write(`url: ${result.baseUrl}\n`);
|
|
20
20
|
io.stdout.write(`pid: ${result.state.pid}\n`);
|
|
21
21
|
io.stdout.write(`version: ${result.state.version}\n`);
|
|
@@ -26,10 +26,10 @@ function writeDaemonStart(io, result) {
|
|
|
26
26
|
}
|
|
27
27
|
function writeDaemonStop(io, result) {
|
|
28
28
|
if (result.status === 'already-stopped') {
|
|
29
|
-
io.stdout.write('KTX
|
|
29
|
+
io.stdout.write('KTX daemon already stopped\n');
|
|
30
30
|
return;
|
|
31
31
|
}
|
|
32
|
-
io.stdout.write('Stopped KTX
|
|
32
|
+
io.stdout.write('Stopped KTX daemon\n');
|
|
33
33
|
io.stdout.write(`pid: ${result.state?.pid ?? 'unknown'}\n`);
|
|
34
34
|
io.stdout.write(`state: ${result.layout.daemonStatePath}\n`);
|
|
35
35
|
}
|
|
@@ -42,11 +42,11 @@ function writeDaemonStopAll(io, result) {
|
|
|
42
42
|
result.stale.length === 0 &&
|
|
43
43
|
result.failed.length === 0 &&
|
|
44
44
|
result.scanErrors.length === 0) {
|
|
45
|
-
io.stdout.write('No KTX
|
|
45
|
+
io.stdout.write('No KTX daemons found\n');
|
|
46
46
|
return 0;
|
|
47
47
|
}
|
|
48
48
|
if (failed === 0) {
|
|
49
|
-
io.stdout.write(`Stopped ${result.stopped.length} KTX
|
|
49
|
+
io.stdout.write(`Stopped ${result.stopped.length} KTX daemons\n`);
|
|
50
50
|
if (result.stale.length > 0) {
|
|
51
51
|
io.stdout.write(`Cleaned ${result.stale.length} stale daemon states\n`);
|
|
52
52
|
}
|
|
@@ -58,7 +58,7 @@ function writeDaemonStopAll(io, result) {
|
|
|
58
58
|
}
|
|
59
59
|
return 0;
|
|
60
60
|
}
|
|
61
|
-
io.stderr.write(`Stopped ${result.stopped.length} KTX
|
|
61
|
+
io.stderr.write(`Stopped ${result.stopped.length} KTX daemons; failed ${result.failed.length}${result.stale.length > 0 ? `; cleaned stale ${result.stale.length}` : ''}\n`);
|
|
62
62
|
for (const entry of result.failed) {
|
|
63
63
|
io.stderr.write(`pid: ${entry.pid} source: ${entry.source}${entry.url ? ` url: ${entry.url}` : ''}${entry.health ? ` health: ${entry.health}` : ''} detail: ${entry.detail}\n`);
|
|
64
64
|
}
|
package/dist/runtime.test.js
CHANGED
|
@@ -88,7 +88,7 @@ describe('runKtxRuntime', () => {
|
|
|
88
88
|
expect(io.stdout()).toContain('manifest: /runtime/0.2.0/manifest.json');
|
|
89
89
|
expect(io.stderr()).toBe('');
|
|
90
90
|
});
|
|
91
|
-
it('starts the
|
|
91
|
+
it('starts the KTX daemon and prints the base URL', async () => {
|
|
92
92
|
const io = makeIo();
|
|
93
93
|
const deps = {
|
|
94
94
|
startDaemon: vi.fn(async () => ({
|
|
@@ -131,13 +131,13 @@ describe('runKtxRuntime', () => {
|
|
|
131
131
|
features: ['local-embeddings'],
|
|
132
132
|
force: true,
|
|
133
133
|
});
|
|
134
|
-
expect(io.stdout()).toContain('Started KTX
|
|
134
|
+
expect(io.stdout()).toContain('Started KTX daemon');
|
|
135
135
|
expect(io.stdout()).toContain('url: http://127.0.0.1:61234');
|
|
136
136
|
expect(io.stdout()).toContain('pid: 4242');
|
|
137
137
|
expect(io.stdout()).toContain('features: core, local-embeddings');
|
|
138
138
|
expect(io.stdout()).toContain('stderr: /work/proj/.ktx/runtime/daemon.stderr.log');
|
|
139
139
|
});
|
|
140
|
-
it('stops the
|
|
140
|
+
it('stops the KTX daemon', async () => {
|
|
141
141
|
const io = makeIo();
|
|
142
142
|
const deps = {
|
|
143
143
|
stopDaemon: vi.fn(async () => ({
|
|
@@ -174,10 +174,10 @@ describe('runKtxRuntime', () => {
|
|
|
174
174
|
};
|
|
175
175
|
await expect(runKtxRuntime({ command: 'stop', cliVersion: '0.2.0', projectDir: '/work/proj', all: false }, io.io, deps)).resolves.toBe(0);
|
|
176
176
|
expect(deps.stopDaemon).toHaveBeenCalledWith({ cliVersion: '0.2.0', projectDir: '/work/proj' });
|
|
177
|
-
expect(io.stdout()).toContain('Stopped KTX
|
|
177
|
+
expect(io.stdout()).toContain('Stopped KTX daemon');
|
|
178
178
|
expect(io.stdout()).toContain('pid: 4242');
|
|
179
179
|
});
|
|
180
|
-
it('stops all discovered
|
|
180
|
+
it('stops all discovered KTX daemons and reports the summary', async () => {
|
|
181
181
|
const io = makeIo();
|
|
182
182
|
const deps = {
|
|
183
183
|
stopAllDaemons: vi.fn(async () => ({
|
|
@@ -192,7 +192,7 @@ describe('runKtxRuntime', () => {
|
|
|
192
192
|
};
|
|
193
193
|
await expect(runKtxRuntime({ command: 'stop', cliVersion: '0.2.0', projectDir: '/work/proj', all: true }, io.io, deps)).resolves.toBe(0);
|
|
194
194
|
expect(deps.stopAllDaemons).toHaveBeenCalledWith({ cliVersion: '0.2.0', projectDir: '/work/proj' });
|
|
195
|
-
expect(io.stdout()).toContain('Stopped 2 KTX
|
|
195
|
+
expect(io.stdout()).toContain('Stopped 2 KTX daemons');
|
|
196
196
|
expect(io.stdout()).toContain('pid: 4242 source: state url: http://127.0.0.1:61234');
|
|
197
197
|
expect(io.stdout()).toContain('pid: 5252 source: process url: http://127.0.0.1:8765');
|
|
198
198
|
});
|
|
@@ -215,7 +215,7 @@ describe('runKtxRuntime', () => {
|
|
|
215
215
|
})),
|
|
216
216
|
};
|
|
217
217
|
await expect(runKtxRuntime({ command: 'stop', cliVersion: '0.2.0', projectDir: '/work/proj', all: true }, io.io, deps)).resolves.toBe(1);
|
|
218
|
-
expect(io.stderr()).toContain('Stopped 0 KTX
|
|
218
|
+
expect(io.stderr()).toContain('Stopped 0 KTX daemons; failed 1');
|
|
219
219
|
expect(io.stderr()).toContain('pid: 4242 source: state url: http://127.0.0.1:61234');
|
|
220
220
|
expect(io.stderr()).toContain('process scan: ps failed');
|
|
221
221
|
});
|
package/dist/scan.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { loadKtxProject } from '@ktx/context/project';
|
|
2
1
|
import { runLocalScan, } from '@ktx/context/scan';
|
|
2
|
+
import { loadKtxCliProject } from './cli-project.js';
|
|
3
3
|
import { createKtxCliLocalIngestAdapters } from './local-adapters.js';
|
|
4
4
|
import { createKtxCliScanConnector } from './local-scan-connectors.js';
|
|
5
5
|
import { profileMark } from './startup-profile.js';
|
|
@@ -237,7 +237,12 @@ export function createCliScanProgress(io, state = { progress: 0, hasPendingTrans
|
|
|
237
237
|
}
|
|
238
238
|
export async function runKtxScan(args, io = process, deps = {}) {
|
|
239
239
|
try {
|
|
240
|
-
const project = await
|
|
240
|
+
const project = await loadKtxCliProject({
|
|
241
|
+
projectDir: args.projectDir,
|
|
242
|
+
cliVersion: args.cliVersion ?? '0.0.0-private',
|
|
243
|
+
installPolicy: args.runtimeInstallPolicy ?? 'never',
|
|
244
|
+
io,
|
|
245
|
+
});
|
|
241
246
|
const managedDaemon = managedDaemonOptionsForScanRun(args, deps.runtimeIo ?? io);
|
|
242
247
|
const connector = args.mode !== 'structural' || args.detectRelationships
|
|
243
248
|
? await createKtxCliScanConnector(project, args.connectionId)
|
package/dist/scan.test.js
CHANGED
|
@@ -294,7 +294,7 @@ describe('runKtxScan', () => {
|
|
|
294
294
|
expect(io.stdout()).not.toContain('+1');
|
|
295
295
|
expect(io.stdout()).not.toContain('/~');
|
|
296
296
|
});
|
|
297
|
-
it('passes
|
|
297
|
+
it('passes KTX daemon options to local ingest adapters when no explicit daemon URL is set', async () => {
|
|
298
298
|
await initKtxProject({ projectDir: tempDir });
|
|
299
299
|
const createLocalIngestAdapters = vi.fn(() => []);
|
|
300
300
|
const runLocalScan = vi.fn(async (_input) => ({
|
package/dist/setup-embeddings.js
CHANGED
|
@@ -214,7 +214,7 @@ function localEmbeddingSetupMessage(message, stderrTail = []) {
|
|
|
214
214
|
'The first run may download Python packages and the all-MiniLM-L6-v2 model.',
|
|
215
215
|
];
|
|
216
216
|
if (stderrTail.length > 0) {
|
|
217
|
-
lines.push('Recent
|
|
217
|
+
lines.push('Recent KTX daemon stderr:', ...stderrTail);
|
|
218
218
|
}
|
|
219
219
|
return lines.join('\n');
|
|
220
220
|
}
|
|
@@ -275,7 +275,7 @@ describe('setup embeddings step', () => {
|
|
|
275
275
|
healthCheck: vi.fn(async () => ({ ok: false, message: 'HTTP 500' })),
|
|
276
276
|
});
|
|
277
277
|
expect(result.status).toBe('failed');
|
|
278
|
-
expect(io.stderr()).toContain('Recent
|
|
278
|
+
expect(io.stderr()).toContain('Recent KTX daemon stderr:');
|
|
279
279
|
expect(io.stderr()).toContain('daemon traceback line 6');
|
|
280
280
|
expect(io.stderr()).toContain('daemon traceback line 45');
|
|
281
281
|
expect(io.stderr()).not.toContain('daemon traceback line 5');
|
|
@@ -296,7 +296,7 @@ describe('setup embeddings step', () => {
|
|
|
296
296
|
healthCheck: vi.fn(async () => ({ ok: false, message: 'HTTP 500' })),
|
|
297
297
|
});
|
|
298
298
|
expect(result.status).toBe('failed');
|
|
299
|
-
expect(io.stderr()).not.toContain('Recent
|
|
299
|
+
expect(io.stderr()).not.toContain('Recent KTX daemon stderr:');
|
|
300
300
|
});
|
|
301
301
|
it('uses fixed OpenAI defaults and only asks for credentials when OpenAI is selected', async () => {
|
|
302
302
|
const io = makeIo();
|
|
@@ -73,7 +73,7 @@ describe('runKtxSetupRuntimeStep', () => {
|
|
|
73
73
|
expect((await readKtxSetupState(tempDir)).completed_steps).not.toContain('runtime');
|
|
74
74
|
expect(io.stderr()).toContain('ktx admin runtime install --yes');
|
|
75
75
|
});
|
|
76
|
-
it('starts the
|
|
76
|
+
it('starts the KTX daemon for configured sentence-transformers embeddings', async () => {
|
|
77
77
|
const io = makeIo();
|
|
78
78
|
const ensureLocalEmbeddings = vi.fn(async () => ({
|
|
79
79
|
baseUrl: 'http://127.0.0.1:61234',
|
|
@@ -101,6 +101,7 @@ export declare class GitService {
|
|
|
101
101
|
status: 'A' | 'M' | 'D';
|
|
102
102
|
path: string;
|
|
103
103
|
}>>;
|
|
104
|
+
changedPaths(): Promise<string[]>;
|
|
104
105
|
/**
|
|
105
106
|
* List all paths under the working tree that match `pathSpec`, scoped to HEAD.
|
|
106
107
|
* Used for the reconciler's first-ever run when there's no watermark to diff from.
|
|
@@ -424,6 +424,18 @@ export class GitService {
|
|
|
424
424
|
}
|
|
425
425
|
return out;
|
|
426
426
|
}
|
|
427
|
+
async changedPaths() {
|
|
428
|
+
const raw = await this.git.raw(['status', '--porcelain=v1', '-z']);
|
|
429
|
+
const fields = raw.split('\0').filter(Boolean);
|
|
430
|
+
const paths = [];
|
|
431
|
+
for (const field of fields) {
|
|
432
|
+
const path = field.slice(3);
|
|
433
|
+
if (path.length > 0) {
|
|
434
|
+
paths.push(path);
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
return [...new Set(paths)].sort();
|
|
438
|
+
}
|
|
427
439
|
/**
|
|
428
440
|
* List all paths under the working tree that match `pathSpec`, scoped to HEAD.
|
|
429
441
|
* Used for the reconciler's first-ever run when there's no watermark to diff from.
|
package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/historic-sql.adapter.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ChunkResult, DiffSet, FetchContext, ScopeDescriptor, SourceAdapter } from '../../types.js';
|
|
1
|
+
import type { ChunkResult, DeterministicFinalizationContext, DiffSet, FetchContext, FinalizationResult, ScopeDescriptor, SourceAdapter } from '../../types.js';
|
|
2
2
|
import { type HistoricSqlSourceAdapterDeps } from './types.js';
|
|
3
3
|
export declare class HistoricSqlSourceAdapter implements SourceAdapter {
|
|
4
4
|
private readonly deps;
|
|
@@ -11,4 +11,5 @@ export declare class HistoricSqlSourceAdapter implements SourceAdapter {
|
|
|
11
11
|
fetch(pullConfig: unknown, stagedDir: string, ctx: FetchContext): Promise<void>;
|
|
12
12
|
chunk(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult>;
|
|
13
13
|
describeScope(stagedDir: string): Promise<ScopeDescriptor>;
|
|
14
|
+
finalize(ctx: DeterministicFinalizationContext): Promise<FinalizationResult>;
|
|
14
15
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { chunkHistoricSqlUnifiedStagedDir, describeHistoricSqlUnifiedScope } from './chunk-unified.js';
|
|
2
2
|
import { detectHistoricSqlStagedDir } from './detect.js';
|
|
3
|
+
import { projectHistoricSqlEvidence } from './projection.js';
|
|
3
4
|
import { stageHistoricSqlAggregatedSnapshot } from './stage-unified.js';
|
|
4
5
|
export class HistoricSqlSourceAdapter {
|
|
5
6
|
deps;
|
|
@@ -30,4 +31,21 @@ export class HistoricSqlSourceAdapter {
|
|
|
30
31
|
describeScope(stagedDir) {
|
|
31
32
|
return describeHistoricSqlUnifiedScope(stagedDir);
|
|
32
33
|
}
|
|
34
|
+
async finalize(ctx) {
|
|
35
|
+
const projection = await projectHistoricSqlEvidence({
|
|
36
|
+
workdir: ctx.workdir,
|
|
37
|
+
connectionId: ctx.connectionId,
|
|
38
|
+
syncId: ctx.syncId,
|
|
39
|
+
runId: ctx.runId,
|
|
40
|
+
overrideReplay: ctx.overrideReplay,
|
|
41
|
+
});
|
|
42
|
+
return {
|
|
43
|
+
result: projection,
|
|
44
|
+
warnings: projection.warnings,
|
|
45
|
+
errors: [],
|
|
46
|
+
touchedSources: projection.touchedSources,
|
|
47
|
+
changedWikiPageKeys: projection.changedWikiPageKeys,
|
|
48
|
+
actions: projection.actions,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
33
51
|
}
|
package/node_modules/@ktx/context/dist/ingest/adapters/historic-sql/local-ingest-acceptance.test.js
CHANGED
|
@@ -194,12 +194,12 @@ describe('historic-SQL local ingest retrieval acceptance', () => {
|
|
|
194
194
|
expect(result.result.failedWorkUnits).toEqual([]);
|
|
195
195
|
expect(result.result.workUnitCount).toBe(3);
|
|
196
196
|
expect(agentRunner.runLoop).toHaveBeenCalledTimes(3);
|
|
197
|
-
const
|
|
198
|
-
expect(
|
|
199
|
-
if (!
|
|
200
|
-
throw new Error('Expected historic-SQL
|
|
197
|
+
const finalization = result.report.body.finalization;
|
|
198
|
+
expect(finalization).toBeDefined();
|
|
199
|
+
if (!finalization) {
|
|
200
|
+
throw new Error('Expected historic-SQL finalization result');
|
|
201
201
|
}
|
|
202
|
-
expect(
|
|
202
|
+
expect(finalization).toMatchObject({
|
|
203
203
|
sourceKey: 'historic-sql',
|
|
204
204
|
status: 'success',
|
|
205
205
|
result: {
|
|
@@ -207,7 +207,7 @@ describe('historic-SQL local ingest retrieval acceptance', () => {
|
|
|
207
207
|
patternPagesWritten: 1,
|
|
208
208
|
},
|
|
209
209
|
});
|
|
210
|
-
expect(
|
|
210
|
+
expect(finalization.declaredTouchedSources).toEqual(expect.arrayContaining([
|
|
211
211
|
{ connectionId: 'warehouse', sourceName: 'customers' },
|
|
212
212
|
{ connectionId: 'warehouse', sourceName: 'orders' },
|
|
213
213
|
]));
|
|
@@ -1,8 +1,11 @@
|
|
|
1
|
+
import type { MemoryAction } from '../../../memory/index.js';
|
|
2
|
+
import type { FinalizationOverrideReplay } from '../../types.js';
|
|
1
3
|
export interface HistoricSqlProjectionInput {
|
|
2
4
|
workdir: string;
|
|
3
5
|
connectionId: string;
|
|
4
6
|
syncId: string;
|
|
5
7
|
runId: string;
|
|
8
|
+
overrideReplay?: FinalizationOverrideReplay;
|
|
6
9
|
}
|
|
7
10
|
export interface HistoricSqlProjectionResult {
|
|
8
11
|
tableUsageMerged: number;
|
|
@@ -14,6 +17,8 @@ export interface HistoricSqlProjectionResult {
|
|
|
14
17
|
connectionId: string;
|
|
15
18
|
sourceName: string;
|
|
16
19
|
}>;
|
|
20
|
+
changedWikiPageKeys: string[];
|
|
21
|
+
actions: MemoryAction[];
|
|
17
22
|
warnings: string[];
|
|
18
23
|
}
|
|
19
24
|
export declare function projectHistoricSqlEvidence(input: HistoricSqlProjectionInput): Promise<HistoricSqlProjectionResult>;
|