@knolo/cli 5.2.3 → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/bin/knolo.mjs +639 -156
- package/package.json +2 -2
- package/templates/icp-knowledge-canister/canisters/knolo-core-rust/Cargo.toml +1 -1
- package/templates/icp-knowledge-canister/canisters/knolo-core-rust/src/lib.rs +294 -3
- package/templates/icp-knowledge-canister/canisters/knolo-icp-canister/Cargo.lock +2 -2
- package/templates/icp-knowledge-canister/canisters/knolo-icp-canister/Cargo.toml +2 -2
package/bin/knolo.mjs
CHANGED
|
@@ -20,7 +20,17 @@ import path from 'node:path';
|
|
|
20
20
|
import process from 'node:process';
|
|
21
21
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
22
22
|
import { createRequire } from 'node:module';
|
|
23
|
-
import {
|
|
23
|
+
import {
|
|
24
|
+
isHubAddInvocation,
|
|
25
|
+
runHubAdd,
|
|
26
|
+
runHubInfo,
|
|
27
|
+
runHubLogin,
|
|
28
|
+
runHubLogout,
|
|
29
|
+
runHubPublish,
|
|
30
|
+
runHubSearch,
|
|
31
|
+
runHubWhoami,
|
|
32
|
+
runHubYank,
|
|
33
|
+
} from './registry/commands.mjs';
|
|
24
34
|
|
|
25
35
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
26
36
|
const require = createRequire(import.meta.url);
|
|
@@ -33,9 +43,45 @@ const DEFAULT_CONFIG = {
|
|
|
33
43
|
query: { topK: 5 },
|
|
34
44
|
};
|
|
35
45
|
const SUPPORTED_EXTENSIONS = new Set(['.md', '.txt', '.json']);
|
|
36
|
-
const SKIP_DIRS = new Set([
|
|
37
|
-
|
|
38
|
-
|
|
46
|
+
const SKIP_DIRS = new Set([
|
|
47
|
+
'node_modules',
|
|
48
|
+
'dist',
|
|
49
|
+
'.git',
|
|
50
|
+
'.dfx',
|
|
51
|
+
'.cargo-target',
|
|
52
|
+
]);
|
|
53
|
+
const SUBCOMMANDS = new Set([
|
|
54
|
+
'init',
|
|
55
|
+
'add',
|
|
56
|
+
'build',
|
|
57
|
+
'query',
|
|
58
|
+
'inspect',
|
|
59
|
+
'migrate',
|
|
60
|
+
'verify',
|
|
61
|
+
'explain',
|
|
62
|
+
'diff',
|
|
63
|
+
'dev',
|
|
64
|
+
'v5',
|
|
65
|
+
'search',
|
|
66
|
+
'info',
|
|
67
|
+
'login',
|
|
68
|
+
'whoami',
|
|
69
|
+
'logout',
|
|
70
|
+
'publish',
|
|
71
|
+
'yank',
|
|
72
|
+
'semantic:index',
|
|
73
|
+
'semantic:inspect',
|
|
74
|
+
'semantic:validate',
|
|
75
|
+
]);
|
|
76
|
+
const ICP_SUBCOMMANDS = new Set([
|
|
77
|
+
'init',
|
|
78
|
+
'build-pack',
|
|
79
|
+
'upload',
|
|
80
|
+
'query',
|
|
81
|
+
'health',
|
|
82
|
+
'info',
|
|
83
|
+
'clear',
|
|
84
|
+
]);
|
|
39
85
|
const ICP_TEMPLATE_CANDIDATES = [
|
|
40
86
|
path.resolve(__dirname, '../templates/icp-knowledge-canister'),
|
|
41
87
|
path.resolve(__dirname, '../../../examples/icp-knowledge-canister'),
|
|
@@ -47,7 +93,11 @@ function createError(message) {
|
|
|
47
93
|
|
|
48
94
|
async function tryImport(filePath) {
|
|
49
95
|
try {
|
|
50
|
-
if (
|
|
96
|
+
if (
|
|
97
|
+
filePath.startsWith('.') ||
|
|
98
|
+
filePath.startsWith('/') ||
|
|
99
|
+
/^[A-Za-z]:[\\/]/.test(filePath)
|
|
100
|
+
) {
|
|
51
101
|
return await import(pathToFileURL(filePath).href);
|
|
52
102
|
}
|
|
53
103
|
return await import(filePath);
|
|
@@ -66,9 +116,16 @@ async function loadCore() {
|
|
|
66
116
|
for (const candidate of candidates) {
|
|
67
117
|
const mod = await tryImport(candidate);
|
|
68
118
|
if (mod?.buildPack && mod?.mountPack && mod?.query) return mod;
|
|
69
|
-
if (
|
|
119
|
+
if (
|
|
120
|
+
mod?.default?.buildPack &&
|
|
121
|
+
mod?.default?.mountPack &&
|
|
122
|
+
mod?.default?.query
|
|
123
|
+
)
|
|
124
|
+
return mod.default;
|
|
70
125
|
}
|
|
71
|
-
throw createError(
|
|
126
|
+
throw createError(
|
|
127
|
+
'Could not load @knolo/core. Build packages/core first (npm run build --workspace @knolo/core).'
|
|
128
|
+
);
|
|
72
129
|
}
|
|
73
130
|
|
|
74
131
|
function printRootHelp() {
|
|
@@ -96,7 +153,7 @@ Commands:
|
|
|
96
153
|
explain <receipt> Verify and explain a query receipt
|
|
97
154
|
diff <pack-a> <pack-b> Compare pack identities and source/chunk counts
|
|
98
155
|
dev Watch config/sources and rebuild on change
|
|
99
|
-
v5 Inspect,
|
|
156
|
+
v5 Inspect, compress, decompress, and health-check a V5 Knowledge Image
|
|
100
157
|
icp Scaffold and operate an ICP-native knowledge canister
|
|
101
158
|
|
|
102
159
|
Global options:
|
|
@@ -110,26 +167,33 @@ function printCommandHelp(command) {
|
|
|
110
167
|
const help = {
|
|
111
168
|
init: 'Usage: knolo init',
|
|
112
169
|
add: 'Usage: knolo add <name> <path> | <publisher>/<slug>[@<version>] [--out <path>] [--force] [--json] [--registry <url>]',
|
|
113
|
-
search:
|
|
170
|
+
search:
|
|
171
|
+
'Usage: knolo search <query> [--format V4|V5] [--license <id>] [--official] [--agents] [--json] [--registry <url>]',
|
|
114
172
|
info: 'Usage: knolo info <publisher>/<slug> [--json] [--registry <url>]',
|
|
115
|
-
login:
|
|
173
|
+
login:
|
|
174
|
+
'Usage: knolo login [--token kno_…] [--stdin] [--registry <url>]\n\nStores the raw token locally. Hub write calls send Authorization: Bearer kno_….\nMint tokens at https://hub.knolo.dev/dashboard/tokens; login does not call POST /api/v1/tokens.',
|
|
116
175
|
whoami: 'Usage: knolo whoami',
|
|
117
176
|
logout: 'Usage: knolo logout',
|
|
118
|
-
publish:
|
|
177
|
+
publish:
|
|
178
|
+
'Usage: knolo publish <pack.knolo> --slug <slug> --version <version> --license <SPDX> [--publisher <handle>] [--readme <text>] [--sources <text>] [--intended-use <text>] [--json] [--registry <url>]\n\nUses the stored dashboard token as Authorization: Bearer kno_…. Hub issues a public Blob PUT grant; the CLI never needs PACKS_READ_WRITE_TOKEN.\nUploads go to the grant URL without the kno_ token. Complete sends the public Blob URL, then Hub verifies before release.',
|
|
119
179
|
yank: 'Usage: knolo yank <publisher>/<slug>@<version> [--json] [--registry <url>]\n\nUses the stored token as Authorization: Bearer kno_…. Yank is owner-only and leaves the public Blob in place.',
|
|
120
180
|
build: 'Usage: knolo build',
|
|
121
|
-
query:
|
|
181
|
+
query:
|
|
182
|
+
'Usage: knolo query <question> [--pack <path>] [--k <number>] [--receipt <file>] [--json]',
|
|
122
183
|
inspect: 'Usage: knolo inspect <pack.knolo>',
|
|
123
|
-
migrate:
|
|
184
|
+
migrate:
|
|
185
|
+
'Usage: knolo migrate <pack.knolo> --out <new-pack.knolo> [--to 4]',
|
|
124
186
|
verify: 'Usage: knolo verify <pack.knolo>',
|
|
125
187
|
explain: 'Usage: knolo explain <receipt.json> --pack <pack.knolo>',
|
|
126
188
|
diff: 'Usage: knolo diff <pack-a.knolo> <pack-b.knolo>',
|
|
127
189
|
dev: 'Usage: knolo dev',
|
|
128
|
-
v5: 'Usage: knolo v5 <info|health|studio> <image.v5> [--
|
|
190
|
+
v5: 'Usage: knolo v5 <info|health|studio|compress|decompress> <image.v5> [--out <file>] [--mode fast|balanced|max] [--attach-index]',
|
|
129
191
|
icp: 'Usage: knolo icp <command> [options]',
|
|
130
|
-
'semantic:index':
|
|
192
|
+
'semantic:index':
|
|
193
|
+
'Usage: knolo semantic:index --pack <path> [--out <path>] [--model <id>] [--endpoint <url>]',
|
|
131
194
|
'semantic:inspect': 'Usage: knolo semantic:inspect --sidecar <path>',
|
|
132
|
-
'semantic:validate':
|
|
195
|
+
'semantic:validate':
|
|
196
|
+
'Usage: knolo semantic:validate --pack <path> --sidecar <path> --model <id>',
|
|
133
197
|
};
|
|
134
198
|
console.log(help[command] ?? 'Unknown command.');
|
|
135
199
|
}
|
|
@@ -222,13 +286,16 @@ function parseArgv(argv) {
|
|
|
222
286
|
global.debug = true;
|
|
223
287
|
args.shift();
|
|
224
288
|
}
|
|
225
|
-
if (args[0] === '--help' || args[0] === '-h')
|
|
289
|
+
if (args[0] === '--help' || args[0] === '-h')
|
|
290
|
+
return { global, command: 'help', commandArgs: [] };
|
|
226
291
|
return { global, command: args[0] || 'help', commandArgs: args.slice(1) };
|
|
227
292
|
}
|
|
228
293
|
|
|
229
294
|
function readConfig(configPath = path.resolve(process.cwd(), CONFIG_FILE)) {
|
|
230
295
|
if (!existsSync(configPath)) {
|
|
231
|
-
throw createError(
|
|
296
|
+
throw createError(
|
|
297
|
+
`Missing config file at ${path.relative(process.cwd(), configPath)}. Run "knolo init" first.`
|
|
298
|
+
);
|
|
232
299
|
}
|
|
233
300
|
let parsed;
|
|
234
301
|
try {
|
|
@@ -241,12 +308,17 @@ function readConfig(configPath = path.resolve(process.cwd(), CONFIG_FILE)) {
|
|
|
241
308
|
}
|
|
242
309
|
|
|
243
310
|
function validateConfig(config) {
|
|
244
|
-
if (!config || typeof config !== 'object')
|
|
245
|
-
|
|
246
|
-
if (
|
|
311
|
+
if (!config || typeof config !== 'object')
|
|
312
|
+
throw createError(`${CONFIG_FILE} must be a JSON object.`);
|
|
313
|
+
if (config.version !== 1)
|
|
314
|
+
throw createError(`${CONFIG_FILE} must include "version": 1.`);
|
|
315
|
+
if (!Array.isArray(config.sources))
|
|
316
|
+
throw createError(`${CONFIG_FILE} must include a "sources" array.`);
|
|
247
317
|
for (const [i, s] of config.sources.entries()) {
|
|
248
|
-
if (!s?.name || typeof s.name !== 'string')
|
|
249
|
-
|
|
318
|
+
if (!s?.name || typeof s.name !== 'string')
|
|
319
|
+
throw createError(`sources[${i}].name must be a string.`);
|
|
320
|
+
if (!s?.path || typeof s.path !== 'string')
|
|
321
|
+
throw createError(`sources[${i}].path must be a string.`);
|
|
250
322
|
}
|
|
251
323
|
if (!config.output?.path || typeof config.output.path !== 'string') {
|
|
252
324
|
throw createError(`${CONFIG_FILE} must include output.path.`);
|
|
@@ -254,7 +326,10 @@ function validateConfig(config) {
|
|
|
254
326
|
}
|
|
255
327
|
|
|
256
328
|
function writeConfig(config) {
|
|
257
|
-
writeFileSync(
|
|
329
|
+
writeFileSync(
|
|
330
|
+
path.resolve(process.cwd(), CONFIG_FILE),
|
|
331
|
+
`${JSON.stringify(config, null, 2)}\n`
|
|
332
|
+
);
|
|
258
333
|
}
|
|
259
334
|
|
|
260
335
|
async function cmdInit() {
|
|
@@ -277,9 +352,15 @@ async function cmdInit() {
|
|
|
277
352
|
console.log('• kept existing docs/');
|
|
278
353
|
}
|
|
279
354
|
|
|
280
|
-
const visibleEntries = readdirSync(docsDir).filter(
|
|
355
|
+
const visibleEntries = readdirSync(docsDir).filter(
|
|
356
|
+
(name) => !name.startsWith('.')
|
|
357
|
+
);
|
|
281
358
|
if (visibleEntries.length === 0 && !existsSync(samplePath)) {
|
|
282
|
-
writeFileSync(
|
|
359
|
+
writeFileSync(
|
|
360
|
+
samplePath,
|
|
361
|
+
'# Hello from KnoLo\n\nThis is a starter document for KnoLo CLI demos.\n',
|
|
362
|
+
'utf8'
|
|
363
|
+
);
|
|
283
364
|
console.log('✔ created docs/hello.md');
|
|
284
365
|
} else {
|
|
285
366
|
console.log('• did not create sample doc because docs/ is not empty');
|
|
@@ -299,10 +380,14 @@ async function cmdAddLocal(args) {
|
|
|
299
380
|
if (!name || !sourcePath) throw createError('Usage: knolo add <name> <path>');
|
|
300
381
|
|
|
301
382
|
const resolved = path.resolve(process.cwd(), sourcePath);
|
|
302
|
-
if (!existsSync(resolved))
|
|
383
|
+
if (!existsSync(resolved))
|
|
384
|
+
throw createError(`Source path does not exist: ${sourcePath}`);
|
|
303
385
|
|
|
304
386
|
const config = readConfig();
|
|
305
|
-
const normalized =
|
|
387
|
+
const normalized =
|
|
388
|
+
sourcePath.startsWith('.') || path.isAbsolute(sourcePath)
|
|
389
|
+
? sourcePath
|
|
390
|
+
: `./${sourcePath}`;
|
|
306
391
|
const existing = config.sources.find((s) => s.name === name);
|
|
307
392
|
if (existing) {
|
|
308
393
|
existing.path = normalized;
|
|
@@ -317,15 +402,21 @@ async function cmdAddLocal(args) {
|
|
|
317
402
|
|
|
318
403
|
function resolveIcpTemplateDir() {
|
|
319
404
|
for (const candidate of ICP_TEMPLATE_CANDIDATES) {
|
|
320
|
-
if (existsSync(candidate) && statSync(candidate).isDirectory())
|
|
405
|
+
if (existsSync(candidate) && statSync(candidate).isDirectory())
|
|
406
|
+
return candidate;
|
|
321
407
|
}
|
|
322
|
-
throw createError(
|
|
408
|
+
throw createError(
|
|
409
|
+
'Bundled ICP template not found. Reinstall @knolo/cli or run from the Knolo repo checkout.'
|
|
410
|
+
);
|
|
323
411
|
}
|
|
324
412
|
|
|
325
413
|
function ensureDirectoryMissingOrEmpty(targetDir) {
|
|
326
414
|
if (!existsSync(targetDir)) return;
|
|
327
415
|
const stats = statSync(targetDir);
|
|
328
|
-
if (!stats.isDirectory())
|
|
416
|
+
if (!stats.isDirectory())
|
|
417
|
+
throw createError(
|
|
418
|
+
`Target path already exists and is not a directory: ${targetDir}`
|
|
419
|
+
);
|
|
329
420
|
const visibleEntries = readdirSync(targetDir);
|
|
330
421
|
if (visibleEntries.length > 0) {
|
|
331
422
|
throw createError(`Target directory is not empty: ${targetDir}`);
|
|
@@ -334,7 +425,9 @@ function ensureDirectoryMissingOrEmpty(targetDir) {
|
|
|
334
425
|
|
|
335
426
|
function copyDirectoryContents(sourceDir, targetDir) {
|
|
336
427
|
mkdirSync(targetDir, { recursive: true });
|
|
337
|
-
const entries = readdirSync(sourceDir, { withFileTypes: true }).sort((a, b) =>
|
|
428
|
+
const entries = readdirSync(sourceDir, { withFileTypes: true }).sort((a, b) =>
|
|
429
|
+
a.name.localeCompare(b.name)
|
|
430
|
+
);
|
|
338
431
|
for (const entry of entries) {
|
|
339
432
|
const sourcePath = path.join(sourceDir, entry.name);
|
|
340
433
|
const targetPath = path.join(targetDir, entry.name);
|
|
@@ -389,7 +482,9 @@ function parsePositiveInteger(value, flagName) {
|
|
|
389
482
|
}
|
|
390
483
|
|
|
391
484
|
function renderDidNat8Vec(bytes) {
|
|
392
|
-
return bytes.length
|
|
485
|
+
return bytes.length
|
|
486
|
+
? ` ${Array.from(bytes, (value) => `${value}`).join('; ')} `
|
|
487
|
+
: '';
|
|
393
488
|
}
|
|
394
489
|
|
|
395
490
|
function withArgumentFile(prefix, contents, run) {
|
|
@@ -440,7 +535,10 @@ async function walkDir(dir) {
|
|
|
440
535
|
if (entry.isDirectory()) {
|
|
441
536
|
if (SKIP_DIRS.has(entry.name)) continue;
|
|
442
537
|
out.push(...(await walkDir(full)));
|
|
443
|
-
} else if (
|
|
538
|
+
} else if (
|
|
539
|
+
entry.isFile() &&
|
|
540
|
+
SUPPORTED_EXTENSIONS.has(path.extname(entry.name).toLowerCase())
|
|
541
|
+
) {
|
|
444
542
|
out.push(full);
|
|
445
543
|
}
|
|
446
544
|
}
|
|
@@ -451,7 +549,10 @@ async function collectSourceFiles(sources) {
|
|
|
451
549
|
const files = [];
|
|
452
550
|
for (const source of sources) {
|
|
453
551
|
const resolved = path.resolve(process.cwd(), source.path);
|
|
454
|
-
if (!existsSync(resolved))
|
|
552
|
+
if (!existsSync(resolved))
|
|
553
|
+
throw createError(
|
|
554
|
+
`Configured source "${source.name}" not found at ${source.path}.`
|
|
555
|
+
);
|
|
455
556
|
const stats = await fs.lstat(resolved);
|
|
456
557
|
if (stats.isSymbolicLink()) continue;
|
|
457
558
|
if (stats.isFile()) {
|
|
@@ -459,9 +560,12 @@ async function collectSourceFiles(sources) {
|
|
|
459
560
|
files.push({ sourceName: source.name, absolutePath: resolved });
|
|
460
561
|
}
|
|
461
562
|
} else if (stats.isDirectory()) {
|
|
462
|
-
for (const file of await walkDir(resolved))
|
|
563
|
+
for (const file of await walkDir(resolved))
|
|
564
|
+
files.push({ sourceName: source.name, absolutePath: file });
|
|
463
565
|
} else {
|
|
464
|
-
throw createError(
|
|
566
|
+
throw createError(
|
|
567
|
+
`Configured source "${source.name}" must be a file or directory.`
|
|
568
|
+
);
|
|
465
569
|
}
|
|
466
570
|
}
|
|
467
571
|
files.sort((a, b) => a.absolutePath.localeCompare(b.absolutePath));
|
|
@@ -472,10 +576,13 @@ async function buildFromConfig(core, { silent = false } = {}) {
|
|
|
472
576
|
const start = Date.now();
|
|
473
577
|
const config = readConfig();
|
|
474
578
|
const files = await collectSourceFiles(config.sources);
|
|
475
|
-
if (files.length === 0)
|
|
579
|
+
if (files.length === 0)
|
|
580
|
+
throw createError('No supported files found (.md, .txt, .json).');
|
|
476
581
|
|
|
477
582
|
const docs = files.map((f) => {
|
|
478
|
-
const rel = path
|
|
583
|
+
const rel = path
|
|
584
|
+
.relative(process.cwd(), f.absolutePath)
|
|
585
|
+
.replace(/\\/g, '/');
|
|
479
586
|
return {
|
|
480
587
|
id: rel,
|
|
481
588
|
heading: path.basename(rel),
|
|
@@ -503,17 +610,21 @@ async function cmdBuild(core) {
|
|
|
503
610
|
|
|
504
611
|
async function collectNamedFiles(sourceName, inputPath) {
|
|
505
612
|
const resolved = path.resolve(process.cwd(), inputPath);
|
|
506
|
-
if (!existsSync(resolved))
|
|
613
|
+
if (!existsSync(resolved))
|
|
614
|
+
throw createError(`Source path not found: ${inputPath}`);
|
|
507
615
|
|
|
508
616
|
const stats = await fs.lstat(resolved);
|
|
509
617
|
if (stats.isSymbolicLink()) return [];
|
|
510
618
|
if (stats.isFile()) {
|
|
511
619
|
if (!SUPPORTED_EXTENSIONS.has(path.extname(resolved).toLowerCase())) {
|
|
512
|
-
throw createError(
|
|
620
|
+
throw createError(
|
|
621
|
+
`Unsupported file type for ${inputPath}. Expected .md, .txt, or .json.`
|
|
622
|
+
);
|
|
513
623
|
}
|
|
514
624
|
return [{ sourceName, absolutePath: resolved }];
|
|
515
625
|
}
|
|
516
|
-
if (!stats.isDirectory())
|
|
626
|
+
if (!stats.isDirectory())
|
|
627
|
+
throw createError(`Source path must be a file or directory: ${inputPath}`);
|
|
517
628
|
|
|
518
629
|
return (await walkDir(resolved))
|
|
519
630
|
.map((absolutePath) => ({ sourceName, absolutePath }))
|
|
@@ -548,8 +659,12 @@ async function cmdIcpInit(args) {
|
|
|
548
659
|
console.log(` cd ${relativeTarget}`);
|
|
549
660
|
console.log(' dfx start --background');
|
|
550
661
|
console.log(' dfx deploy');
|
|
551
|
-
console.log(
|
|
552
|
-
|
|
662
|
+
console.log(
|
|
663
|
+
' knolo icp build-pack ./knowledge --out ./dist/knowledge.knolo'
|
|
664
|
+
);
|
|
665
|
+
console.log(
|
|
666
|
+
' knolo icp upload ./dist/knowledge.knolo --canister knolo_knowledge'
|
|
667
|
+
);
|
|
553
668
|
console.log(' knolo icp query "alpha beta" --canister knolo_knowledge');
|
|
554
669
|
}
|
|
555
670
|
|
|
@@ -562,9 +677,12 @@ async function cmdIcpBuildPack(core, args) {
|
|
|
562
677
|
|
|
563
678
|
const resolvedInput = path.resolve(process.cwd(), docsDir);
|
|
564
679
|
const files = await collectNamedFiles('docs', docsDir);
|
|
565
|
-
if (!files.length)
|
|
680
|
+
if (!files.length)
|
|
681
|
+
throw createError('No supported files found (.md, .txt, .json).');
|
|
566
682
|
|
|
567
|
-
const rootDir = statSync(resolvedInput).isDirectory()
|
|
683
|
+
const rootDir = statSync(resolvedInput).isDirectory()
|
|
684
|
+
? resolvedInput
|
|
685
|
+
: path.dirname(resolvedInput);
|
|
568
686
|
const docs = buildDocsFromFiles(files, rootDir);
|
|
569
687
|
const bytes = await core.buildPack(docs, { format: 3 });
|
|
570
688
|
const outPath = path.resolve(process.cwd(), flags.out);
|
|
@@ -585,11 +703,16 @@ async function cmdIcpUpload(args) {
|
|
|
585
703
|
const { positional, flags } = parseFlagArgs(args);
|
|
586
704
|
const packFile = positional[0];
|
|
587
705
|
if (!packFile || !flags.canister) {
|
|
588
|
-
throw createError(
|
|
706
|
+
throw createError(
|
|
707
|
+
'Usage: knolo icp upload <packFile> --canister <name-or-id> [--label <text>]'
|
|
708
|
+
);
|
|
589
709
|
}
|
|
590
710
|
|
|
591
711
|
const packPath = path.resolve(process.cwd(), packFile);
|
|
592
|
-
if (!existsSync(packPath))
|
|
712
|
+
if (!existsSync(packPath))
|
|
713
|
+
throw createError(
|
|
714
|
+
`Pack file not found at ${path.relative(process.cwd(), packPath)}.`
|
|
715
|
+
);
|
|
593
716
|
|
|
594
717
|
const label = flags.label || defaultPackLabel(packPath);
|
|
595
718
|
const bytes = readFileSync(packPath);
|
|
@@ -597,7 +720,16 @@ async function cmdIcpUpload(args) {
|
|
|
597
720
|
|
|
598
721
|
withArgumentFile('knolo-icp-upload-', didArgs, (argumentFile) =>
|
|
599
722
|
runDfx(
|
|
600
|
-
[
|
|
723
|
+
[
|
|
724
|
+
'canister',
|
|
725
|
+
'call',
|
|
726
|
+
flags.canister,
|
|
727
|
+
'set_pack',
|
|
728
|
+
'--argument-file',
|
|
729
|
+
argumentFile,
|
|
730
|
+
'--output',
|
|
731
|
+
'json',
|
|
732
|
+
],
|
|
601
733
|
{ cwd: process.cwd() }
|
|
602
734
|
)
|
|
603
735
|
);
|
|
@@ -607,7 +739,9 @@ async function cmdIcpQuery(args) {
|
|
|
607
739
|
const { positional, flags } = parseFlagArgs(args);
|
|
608
740
|
const question = positional.join(' ').trim();
|
|
609
741
|
if (!question || !flags.canister) {
|
|
610
|
-
throw createError(
|
|
742
|
+
throw createError(
|
|
743
|
+
'Usage: knolo icp query <question> --canister <name-or-id> [--k <number>]'
|
|
744
|
+
);
|
|
611
745
|
}
|
|
612
746
|
|
|
613
747
|
const topK = flags.k === undefined ? 5 : parsePositiveInteger(flags.k, '--k');
|
|
@@ -615,7 +749,17 @@ async function cmdIcpQuery(args) {
|
|
|
615
749
|
|
|
616
750
|
withArgumentFile('knolo-icp-query-', didArgs, (argumentFile) =>
|
|
617
751
|
runDfx(
|
|
618
|
-
[
|
|
752
|
+
[
|
|
753
|
+
'canister',
|
|
754
|
+
'call',
|
|
755
|
+
flags.canister,
|
|
756
|
+
'search',
|
|
757
|
+
'--query',
|
|
758
|
+
'--argument-file',
|
|
759
|
+
argumentFile,
|
|
760
|
+
'--output',
|
|
761
|
+
'json',
|
|
762
|
+
],
|
|
619
763
|
{ cwd: process.cwd() }
|
|
620
764
|
)
|
|
621
765
|
);
|
|
@@ -623,33 +767,63 @@ async function cmdIcpQuery(args) {
|
|
|
623
767
|
|
|
624
768
|
async function cmdIcpHealth(args) {
|
|
625
769
|
const { flags } = parseFlagArgs(args);
|
|
626
|
-
if (!flags.canister)
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
770
|
+
if (!flags.canister)
|
|
771
|
+
throw createError('Usage: knolo icp health --canister <name-or-id>');
|
|
772
|
+
runDfx(
|
|
773
|
+
[
|
|
774
|
+
'canister',
|
|
775
|
+
'call',
|
|
776
|
+
flags.canister,
|
|
777
|
+
'health',
|
|
778
|
+
'--query',
|
|
779
|
+
'--output',
|
|
780
|
+
'json',
|
|
781
|
+
],
|
|
782
|
+
{
|
|
783
|
+
cwd: process.cwd(),
|
|
784
|
+
}
|
|
785
|
+
);
|
|
630
786
|
}
|
|
631
787
|
|
|
632
788
|
async function cmdIcpInfo(args) {
|
|
633
789
|
const { flags } = parseFlagArgs(args);
|
|
634
|
-
if (!flags.canister)
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
790
|
+
if (!flags.canister)
|
|
791
|
+
throw createError('Usage: knolo icp info --canister <name-or-id>');
|
|
792
|
+
runDfx(
|
|
793
|
+
[
|
|
794
|
+
'canister',
|
|
795
|
+
'call',
|
|
796
|
+
flags.canister,
|
|
797
|
+
'pack_info',
|
|
798
|
+
'--query',
|
|
799
|
+
'--output',
|
|
800
|
+
'json',
|
|
801
|
+
],
|
|
802
|
+
{
|
|
803
|
+
cwd: process.cwd(),
|
|
804
|
+
}
|
|
805
|
+
);
|
|
638
806
|
}
|
|
639
807
|
|
|
640
808
|
async function cmdIcpClear(args) {
|
|
641
809
|
const { flags } = parseFlagArgs(args);
|
|
642
|
-
if (!flags.canister)
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
810
|
+
if (!flags.canister)
|
|
811
|
+
throw createError('Usage: knolo icp clear --canister <name-or-id>');
|
|
812
|
+
runDfx(
|
|
813
|
+
['canister', 'call', flags.canister, 'clear_pack', '--output', 'json'],
|
|
814
|
+
{
|
|
815
|
+
cwd: process.cwd(),
|
|
816
|
+
}
|
|
817
|
+
);
|
|
646
818
|
}
|
|
647
819
|
|
|
648
820
|
async function cmdIcp(args) {
|
|
649
821
|
const [subcommand, ...rest] = args;
|
|
650
822
|
if (!subcommand) return printIcpHelp();
|
|
651
|
-
if (!ICP_SUBCOMMANDS.has(subcommand))
|
|
652
|
-
|
|
823
|
+
if (!ICP_SUBCOMMANDS.has(subcommand))
|
|
824
|
+
throw createError(`Unknown ICP command: ${subcommand}`);
|
|
825
|
+
if (rest.includes('--help') || rest.includes('-h'))
|
|
826
|
+
return printIcpCommandHelp(subcommand);
|
|
653
827
|
|
|
654
828
|
if (subcommand === 'init') return await cmdIcpInit(rest);
|
|
655
829
|
if (subcommand === 'upload') return await cmdIcpUpload(rest);
|
|
@@ -672,7 +846,8 @@ function parseQueryArgs(args) {
|
|
|
672
846
|
else if (arg === '--pack') opts.pack = args[++i];
|
|
673
847
|
else if (arg === '--k') opts.k = args[++i];
|
|
674
848
|
else if (arg === '--receipt') opts.receipt = args[++i];
|
|
675
|
-
else if (arg.startsWith('--'))
|
|
849
|
+
else if (arg.startsWith('--'))
|
|
850
|
+
throw createError(`Unknown flag for query: ${arg}`);
|
|
676
851
|
else positional.push(arg);
|
|
677
852
|
}
|
|
678
853
|
return { positional, opts };
|
|
@@ -681,21 +856,37 @@ function parseQueryArgs(args) {
|
|
|
681
856
|
async function cmdQuery(core, args) {
|
|
682
857
|
const { positional, opts } = parseQueryArgs(args);
|
|
683
858
|
const question = positional.join(' ').trim();
|
|
684
|
-
if (!question)
|
|
859
|
+
if (!question)
|
|
860
|
+
throw createError(
|
|
861
|
+
'Usage: knolo query <question> [--pack <path>] [--k <number>] [--json]'
|
|
862
|
+
);
|
|
685
863
|
|
|
686
|
-
const config = existsSync(path.resolve(process.cwd(), CONFIG_FILE))
|
|
864
|
+
const config = existsSync(path.resolve(process.cwd(), CONFIG_FILE))
|
|
865
|
+
? readConfig()
|
|
866
|
+
: DEFAULT_CONFIG;
|
|
687
867
|
const packPath = path.resolve(process.cwd(), opts.pack || config.output.path);
|
|
688
|
-
if (!existsSync(packPath))
|
|
868
|
+
if (!existsSync(packPath))
|
|
869
|
+
throw createError(
|
|
870
|
+
`Pack file not found at ${path.relative(process.cwd(), packPath)}.`
|
|
871
|
+
);
|
|
689
872
|
|
|
690
|
-
const topK =
|
|
691
|
-
|
|
873
|
+
const topK =
|
|
874
|
+
opts.k !== undefined ? Number(opts.k) : (config.query?.topK ?? 5);
|
|
875
|
+
if (!Number.isInteger(topK) || topK <= 0)
|
|
876
|
+
throw createError('--k must be a positive integer.');
|
|
692
877
|
|
|
693
878
|
const packBuffer = readFileSync(packPath);
|
|
694
879
|
const bytes = Uint8Array.from(packBuffer);
|
|
695
880
|
const kb = await mountPackFromBytes(core, bytes);
|
|
696
|
-
const receiptResult =
|
|
881
|
+
const receiptResult =
|
|
882
|
+
opts.receipt && core.queryWithReceipt
|
|
883
|
+
? core.queryWithReceipt(kb, question, { topK })
|
|
884
|
+
: null;
|
|
697
885
|
if (receiptResult && opts.receipt) {
|
|
698
|
-
writeFileSync(
|
|
886
|
+
writeFileSync(
|
|
887
|
+
path.resolve(process.cwd(), opts.receipt),
|
|
888
|
+
`${JSON.stringify(receiptResult.receipt, null, 2)}\n`
|
|
889
|
+
);
|
|
699
890
|
}
|
|
700
891
|
const rawHits = receiptResult?.hits ?? core.query(kb, question, { topK });
|
|
701
892
|
const hits = rawHits.map((hit) => ({
|
|
@@ -707,10 +898,28 @@ async function cmdQuery(core, args) {
|
|
|
707
898
|
|
|
708
899
|
if (opts.json) {
|
|
709
900
|
const patch = core.makeContextPatch(
|
|
710
|
-
hits.map((h, i) => ({
|
|
901
|
+
hits.map((h, i) => ({
|
|
902
|
+
blockId: i,
|
|
903
|
+
score: h.score,
|
|
904
|
+
text: h.snippet,
|
|
905
|
+
source: h.source,
|
|
906
|
+
})),
|
|
711
907
|
{ budget: 'small' }
|
|
712
908
|
);
|
|
713
|
-
console.log(
|
|
909
|
+
console.log(
|
|
910
|
+
JSON.stringify(
|
|
911
|
+
{
|
|
912
|
+
question,
|
|
913
|
+
packPath,
|
|
914
|
+
topK,
|
|
915
|
+
hits,
|
|
916
|
+
receipt: receiptResult?.receipt,
|
|
917
|
+
contextPatch: patch,
|
|
918
|
+
},
|
|
919
|
+
null,
|
|
920
|
+
2
|
|
921
|
+
)
|
|
922
|
+
);
|
|
714
923
|
return;
|
|
715
924
|
}
|
|
716
925
|
|
|
@@ -732,8 +941,10 @@ function parsePackCommandArgs(args, command) {
|
|
|
732
941
|
const flags = {};
|
|
733
942
|
for (let i = 0; i < args.length; i++) {
|
|
734
943
|
const arg = args[i];
|
|
735
|
-
if (arg === '--out' || arg === '--to' || arg === '--pack')
|
|
736
|
-
|
|
944
|
+
if (arg === '--out' || arg === '--to' || arg === '--pack')
|
|
945
|
+
flags[arg.slice(2)] = args[++i];
|
|
946
|
+
else if (arg.startsWith('--'))
|
|
947
|
+
throw createError(`Unknown flag for ${command}: ${arg}`);
|
|
737
948
|
else positional.push(arg);
|
|
738
949
|
}
|
|
739
950
|
return { positional, flags };
|
|
@@ -741,61 +952,166 @@ function parsePackCommandArgs(args, command) {
|
|
|
741
952
|
|
|
742
953
|
async function cmdInspect(core, args) {
|
|
743
954
|
const { positional } = parsePackCommandArgs(args, 'inspect');
|
|
744
|
-
const packPath = path.resolve(
|
|
745
|
-
|
|
955
|
+
const packPath = path.resolve(
|
|
956
|
+
process.cwd(),
|
|
957
|
+
positional[0] || 'dist/knowledge.knolo'
|
|
958
|
+
);
|
|
959
|
+
if (!existsSync(packPath))
|
|
960
|
+
throw createError(
|
|
961
|
+
`Pack file not found at ${path.relative(process.cwd(), packPath)}.`
|
|
962
|
+
);
|
|
746
963
|
const bytes = Uint8Array.from(readFileSync(packPath));
|
|
747
964
|
const pack = await mountPackFromBytes(core, bytes);
|
|
748
|
-
const result = {
|
|
749
|
-
|
|
965
|
+
const result = {
|
|
966
|
+
format:
|
|
967
|
+
pack.meta?.format ||
|
|
968
|
+
(pack.meta?.version === 4 ? 'v4' : `v${pack.meta?.version ?? 'unknown'}`),
|
|
969
|
+
version: pack.meta?.version,
|
|
970
|
+
stats: pack.meta?.stats,
|
|
971
|
+
blocks: pack.blocks.length,
|
|
972
|
+
semantic: Boolean(pack.semantic),
|
|
973
|
+
claims: Boolean(pack.claimGraph),
|
|
974
|
+
};
|
|
975
|
+
if (core.isPackV4?.(bytes.buffer))
|
|
976
|
+
result.container = core.inspectPackV4(bytes.buffer);
|
|
750
977
|
console.log(JSON.stringify(result, null, 2));
|
|
751
978
|
}
|
|
752
979
|
|
|
753
980
|
async function cmdVerify(core, args) {
|
|
754
981
|
const { positional } = parsePackCommandArgs(args, 'verify');
|
|
755
|
-
const packPath = path.resolve(
|
|
756
|
-
|
|
982
|
+
const packPath = path.resolve(
|
|
983
|
+
process.cwd(),
|
|
984
|
+
positional[0] || 'dist/knowledge.knolo'
|
|
985
|
+
);
|
|
986
|
+
if (!existsSync(packPath))
|
|
987
|
+
throw createError(
|
|
988
|
+
`Pack file not found at ${path.relative(process.cwd(), packPath)}.`
|
|
989
|
+
);
|
|
757
990
|
const bytes = Uint8Array.from(readFileSync(packPath));
|
|
758
991
|
const pack = await mountPackFromBytes(core, bytes);
|
|
759
|
-
console.log(
|
|
992
|
+
console.log(
|
|
993
|
+
JSON.stringify(
|
|
994
|
+
{
|
|
995
|
+
verified: true,
|
|
996
|
+
format:
|
|
997
|
+
pack.meta?.format ||
|
|
998
|
+
(pack.meta?.version === 4 ? 'v4' : `v${pack.meta?.version}`),
|
|
999
|
+
blocks: pack.blocks.length,
|
|
1000
|
+
terms: pack.meta?.stats?.terms ?? pack.lexicon.size,
|
|
1001
|
+
},
|
|
1002
|
+
null,
|
|
1003
|
+
2
|
|
1004
|
+
)
|
|
1005
|
+
);
|
|
760
1006
|
}
|
|
761
1007
|
|
|
762
1008
|
async function cmdMigrate(core, args) {
|
|
763
1009
|
const { positional, flags } = parsePackCommandArgs(args, 'migrate');
|
|
764
1010
|
const input = positional[0];
|
|
765
|
-
if (!input || !flags.out)
|
|
766
|
-
|
|
1011
|
+
if (!input || !flags.out)
|
|
1012
|
+
throw createError(
|
|
1013
|
+
'Usage: knolo migrate <pack.knolo> --out <new-pack.knolo> --to 4'
|
|
1014
|
+
);
|
|
1015
|
+
if (String(flags.to || '4') !== '4')
|
|
1016
|
+
throw createError('Only migration to format 4 is currently supported.');
|
|
767
1017
|
const inputPath = path.resolve(process.cwd(), input);
|
|
768
|
-
if (!existsSync(inputPath))
|
|
1018
|
+
if (!existsSync(inputPath))
|
|
1019
|
+
throw createError(
|
|
1020
|
+
`Pack file not found at ${path.relative(process.cwd(), inputPath)}.`
|
|
1021
|
+
);
|
|
769
1022
|
const bytes = Uint8Array.from(readFileSync(inputPath));
|
|
770
1023
|
const pack = await mountPackFromBytes(core, bytes);
|
|
771
|
-
if (core.isPackV4?.(bytes.buffer))
|
|
772
|
-
|
|
1024
|
+
if (core.isPackV4?.(bytes.buffer))
|
|
1025
|
+
throw createError('Input pack is already v4.');
|
|
1026
|
+
const docs = pack.blocks.map((text, i) => ({
|
|
1027
|
+
id: pack.docIds?.[i] ?? undefined,
|
|
1028
|
+
heading: pack.headings?.[i] ?? undefined,
|
|
1029
|
+
namespace: pack.namespaces?.[i] ?? undefined,
|
|
1030
|
+
text,
|
|
1031
|
+
}));
|
|
773
1032
|
const migrated = core.serializePackV4(pack, docs);
|
|
774
1033
|
const outPath = path.resolve(process.cwd(), flags.out);
|
|
775
1034
|
mkdirSync(path.dirname(outPath), { recursive: true });
|
|
776
1035
|
writeFileSync(outPath, Buffer.from(migrated));
|
|
777
|
-
console.log(
|
|
1036
|
+
console.log(
|
|
1037
|
+
`✔ migrated ${path.relative(process.cwd(), inputPath)} to ${path.relative(process.cwd(), outPath)}`
|
|
1038
|
+
);
|
|
778
1039
|
}
|
|
779
1040
|
|
|
780
1041
|
async function cmdExplain(core, args) {
|
|
781
1042
|
const { positional, flags } = parsePackCommandArgs(args, 'explain');
|
|
782
1043
|
const receiptPath = positional[0];
|
|
783
|
-
if (!receiptPath || !flags.out && !flags.pack)
|
|
1044
|
+
if (!receiptPath || (!flags.out && !flags.pack))
|
|
1045
|
+
throw createError(
|
|
1046
|
+
'Usage: knolo explain <receipt.json> --pack <pack.knolo>'
|
|
1047
|
+
);
|
|
784
1048
|
const packPath = path.resolve(process.cwd(), flags.pack);
|
|
785
|
-
const receipt = JSON.parse(
|
|
786
|
-
|
|
1049
|
+
const receipt = JSON.parse(
|
|
1050
|
+
readFileSync(path.resolve(process.cwd(), receiptPath), 'utf8')
|
|
1051
|
+
);
|
|
1052
|
+
const pack = await mountPackFromBytes(
|
|
1053
|
+
core,
|
|
1054
|
+
Uint8Array.from(readFileSync(packPath))
|
|
1055
|
+
);
|
|
787
1056
|
core.verifyReceipt(receipt, pack);
|
|
788
|
-
console.log(
|
|
1057
|
+
console.log(
|
|
1058
|
+
JSON.stringify(
|
|
1059
|
+
{
|
|
1060
|
+
verified: true,
|
|
1061
|
+
decision: receipt.decision,
|
|
1062
|
+
answerability: receipt.answerability,
|
|
1063
|
+
reasons: receipt.reasons,
|
|
1064
|
+
query: receipt.normalizedQuery,
|
|
1065
|
+
hits: receipt.hits.map((hit) => ({
|
|
1066
|
+
blockId: hit.blockId,
|
|
1067
|
+
score: hit.score,
|
|
1068
|
+
source: hit.source,
|
|
1069
|
+
spans: hit.spans,
|
|
1070
|
+
})),
|
|
1071
|
+
},
|
|
1072
|
+
null,
|
|
1073
|
+
2
|
|
1074
|
+
)
|
|
1075
|
+
);
|
|
789
1076
|
}
|
|
790
1077
|
|
|
791
1078
|
async function cmdDiff(core, args) {
|
|
792
1079
|
const { positional } = parsePackCommandArgs(args, 'diff');
|
|
793
|
-
if (positional.length < 2)
|
|
794
|
-
|
|
795
|
-
const
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
1080
|
+
if (positional.length < 2)
|
|
1081
|
+
throw createError('Usage: knolo diff <pack-a.knolo> <pack-b.knolo>');
|
|
1082
|
+
const load = async (file) => {
|
|
1083
|
+
const bytes = Uint8Array.from(
|
|
1084
|
+
readFileSync(path.resolve(process.cwd(), file))
|
|
1085
|
+
);
|
|
1086
|
+
return { bytes, pack: await mountPackFromBytes(core, bytes) };
|
|
1087
|
+
};
|
|
1088
|
+
const a = await load(positional[0]);
|
|
1089
|
+
const b = await load(positional[1]);
|
|
1090
|
+
const aDigest = core.packDigest(a.pack);
|
|
1091
|
+
const bDigest = core.packDigest(b.pack);
|
|
1092
|
+
const aSources = new Set(a.pack.docIds?.filter(Boolean) ?? []);
|
|
1093
|
+
const bSources = new Set(b.pack.docIds?.filter(Boolean) ?? []);
|
|
1094
|
+
console.log(
|
|
1095
|
+
JSON.stringify(
|
|
1096
|
+
{
|
|
1097
|
+
equal: aDigest === bDigest,
|
|
1098
|
+
a: {
|
|
1099
|
+
packDigest: aDigest,
|
|
1100
|
+
blocks: a.pack.blocks.length,
|
|
1101
|
+
terms: a.pack.lexicon.size,
|
|
1102
|
+
},
|
|
1103
|
+
b: {
|
|
1104
|
+
packDigest: bDigest,
|
|
1105
|
+
blocks: b.pack.blocks.length,
|
|
1106
|
+
terms: b.pack.lexicon.size,
|
|
1107
|
+
},
|
|
1108
|
+
addedSources: [...bSources].filter((id) => !aSources.has(id)).sort(),
|
|
1109
|
+
removedSources: [...aSources].filter((id) => !bSources.has(id)).sort(),
|
|
1110
|
+
},
|
|
1111
|
+
null,
|
|
1112
|
+
2
|
|
1113
|
+
)
|
|
1114
|
+
);
|
|
799
1115
|
}
|
|
800
1116
|
|
|
801
1117
|
function parseV5Args(args) {
|
|
@@ -803,8 +1119,19 @@ function parseV5Args(args) {
|
|
|
803
1119
|
const flags = {};
|
|
804
1120
|
for (let i = 0; i < args.length; i++) {
|
|
805
1121
|
const arg = args[i];
|
|
806
|
-
if (arg === '--
|
|
807
|
-
else if (
|
|
1122
|
+
if (arg === '--attach-index') flags.attachIndex = true;
|
|
1123
|
+
else if (
|
|
1124
|
+
arg === '--image' ||
|
|
1125
|
+
arg === '--index' ||
|
|
1126
|
+
arg === '--history' ||
|
|
1127
|
+
arg === '--run' ||
|
|
1128
|
+
arg === '--replay' ||
|
|
1129
|
+
arg === '--out' ||
|
|
1130
|
+
arg === '--mode'
|
|
1131
|
+
)
|
|
1132
|
+
flags[arg.slice(2)] = args[++i];
|
|
1133
|
+
else if (arg.startsWith('--'))
|
|
1134
|
+
throw createError(`Unknown flag for v5: ${arg}`);
|
|
808
1135
|
else positional.push(arg);
|
|
809
1136
|
}
|
|
810
1137
|
return { positional, flags };
|
|
@@ -812,24 +1139,106 @@ function parseV5Args(args) {
|
|
|
812
1139
|
|
|
813
1140
|
async function cmdV5(core, args) {
|
|
814
1141
|
const subcommand = args[0] || 'info';
|
|
815
|
-
if (
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
const
|
|
1142
|
+
if (
|
|
1143
|
+
!['info', 'health', 'studio', 'compress', 'decompress'].includes(subcommand)
|
|
1144
|
+
)
|
|
1145
|
+
throw createError(`Unknown V5 command: ${subcommand}`);
|
|
1146
|
+
const { positional, flags } = parseV5Args(
|
|
1147
|
+
args.slice(
|
|
1148
|
+
['info', 'health', 'studio', 'compress', 'decompress'].includes(args[0])
|
|
1149
|
+
? 1
|
|
1150
|
+
: 0
|
|
1151
|
+
)
|
|
1152
|
+
);
|
|
1153
|
+
const imagePath = path.resolve(
|
|
1154
|
+
process.cwd(),
|
|
1155
|
+
flags.image || positional[0] || 'dist/knowledge.v5'
|
|
1156
|
+
);
|
|
1157
|
+
if (!existsSync(imagePath))
|
|
1158
|
+
throw createError(
|
|
1159
|
+
`V5 image file not found at ${path.relative(process.cwd(), imagePath)}.`
|
|
1160
|
+
);
|
|
1161
|
+
if (subcommand === 'compress' || subcommand === 'decompress') {
|
|
1162
|
+
if (!flags.out)
|
|
1163
|
+
throw createError(
|
|
1164
|
+
`Usage: knolo v5 ${subcommand} <image.v5> --out <file>${subcommand === 'compress' ? ' [--mode fast|balanced|max] [--index]' : ''}`
|
|
1165
|
+
);
|
|
1166
|
+
const input = Uint8Array.from(readFileSync(imagePath));
|
|
1167
|
+
const result =
|
|
1168
|
+
subcommand === 'compress'
|
|
1169
|
+
? core.compressKnowledgeImageV5(input, {
|
|
1170
|
+
mode: flags.mode,
|
|
1171
|
+
index: flags.attachIndex,
|
|
1172
|
+
})
|
|
1173
|
+
: core.compressKnowledgeImageV5(input, {
|
|
1174
|
+
objects: false,
|
|
1175
|
+
events: false,
|
|
1176
|
+
});
|
|
1177
|
+
const outPath = path.resolve(process.cwd(), flags.out);
|
|
1178
|
+
mkdirSync(path.dirname(outPath), { recursive: true });
|
|
1179
|
+
writeFileSync(outPath, Buffer.from(result.bytes));
|
|
1180
|
+
const before = input.length;
|
|
1181
|
+
console.log(
|
|
1182
|
+
JSON.stringify(
|
|
1183
|
+
{
|
|
1184
|
+
operation: subcommand,
|
|
1185
|
+
input: path.relative(process.cwd(), imagePath),
|
|
1186
|
+
output: path.relative(process.cwd(), outPath),
|
|
1187
|
+
inputBytes: before,
|
|
1188
|
+
outputBytes: result.bytes.length,
|
|
1189
|
+
reduction: before ? 1 - result.bytes.length / before : 0,
|
|
1190
|
+
stateRoot: result.stateRoot,
|
|
1191
|
+
commitDigest: result.commitDigest,
|
|
1192
|
+
},
|
|
1193
|
+
null,
|
|
1194
|
+
2
|
|
1195
|
+
)
|
|
1196
|
+
);
|
|
1197
|
+
return;
|
|
1198
|
+
}
|
|
1199
|
+
const readOptional = (file, decode) =>
|
|
1200
|
+
file
|
|
1201
|
+
? decode(Uint8Array.from(readFileSync(path.resolve(process.cwd(), file))))
|
|
1202
|
+
: undefined;
|
|
820
1203
|
const input = {
|
|
821
1204
|
image: Uint8Array.from(readFileSync(imagePath)),
|
|
822
|
-
...(flags.index
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
1205
|
+
...(flags.index
|
|
1206
|
+
? {
|
|
1207
|
+
queryIndex: readOptional(
|
|
1208
|
+
flags.index,
|
|
1209
|
+
core.deserializeKnowledgeQueryIndexV1
|
|
1210
|
+
),
|
|
1211
|
+
}
|
|
1212
|
+
: {}),
|
|
1213
|
+
...(flags.history
|
|
1214
|
+
? {
|
|
1215
|
+
queryHistory: readOptional(
|
|
1216
|
+
flags.history,
|
|
1217
|
+
core.deserializeKnowledgeQueryHistoryV1
|
|
1218
|
+
),
|
|
1219
|
+
}
|
|
1220
|
+
: {}),
|
|
1221
|
+
...(flags.run
|
|
1222
|
+
? { run: readOptional(flags.run, core.deserializeKnowledgeRunV1) }
|
|
1223
|
+
: {}),
|
|
1224
|
+
...(flags.replay
|
|
1225
|
+
? {
|
|
1226
|
+
replayState: readOptional(
|
|
1227
|
+
flags.replay,
|
|
1228
|
+
core.deserializeKnowledgeSyncReplayStateV1
|
|
1229
|
+
),
|
|
1230
|
+
}
|
|
1231
|
+
: {}),
|
|
826
1232
|
};
|
|
827
1233
|
if (subcommand === 'studio') {
|
|
828
|
-
console.log(
|
|
1234
|
+
console.log(
|
|
1235
|
+
JSON.stringify(core.inspectKnowledgeStudioManagementV5(input), null, 2)
|
|
1236
|
+
);
|
|
829
1237
|
return;
|
|
830
1238
|
}
|
|
831
1239
|
const diagnostics = core.inspectKnowledgeRuntimeV5(input);
|
|
832
|
-
if (subcommand === 'health')
|
|
1240
|
+
if (subcommand === 'health')
|
|
1241
|
+
console.log(JSON.stringify({ healthy: true, ...diagnostics }, null, 2));
|
|
833
1242
|
else console.log(JSON.stringify(diagnostics, null, 2));
|
|
834
1243
|
}
|
|
835
1244
|
|
|
@@ -844,20 +1253,33 @@ function parseKeyValueArgs(args) {
|
|
|
844
1253
|
}
|
|
845
1254
|
|
|
846
1255
|
async function loadOllamaProvider() {
|
|
847
|
-
const mod = await tryImport(
|
|
1256
|
+
const mod = await tryImport(
|
|
1257
|
+
path.resolve(__dirname, '../../semantic-ollama/dist/index.js')
|
|
1258
|
+
);
|
|
848
1259
|
if (mod?.OllamaEmbeddingProvider) return mod.OllamaEmbeddingProvider;
|
|
849
1260
|
const pkg = await tryImport('@knolo/semantic-ollama');
|
|
850
1261
|
if (pkg?.OllamaEmbeddingProvider) return pkg.OllamaEmbeddingProvider;
|
|
851
|
-
throw createError(
|
|
1262
|
+
throw createError(
|
|
1263
|
+
'Could not load @knolo/semantic-ollama. Build packages/semantic-ollama first.'
|
|
1264
|
+
);
|
|
852
1265
|
}
|
|
853
1266
|
|
|
854
1267
|
async function cmdSemanticIndex(core, args) {
|
|
855
1268
|
const flags = parseKeyValueArgs(args);
|
|
856
|
-
const packPath = path.resolve(
|
|
857
|
-
|
|
1269
|
+
const packPath = path.resolve(
|
|
1270
|
+
process.cwd(),
|
|
1271
|
+
flags.pack || 'dist/knowledge.knolo'
|
|
1272
|
+
);
|
|
1273
|
+
const outPath = path.resolve(
|
|
1274
|
+
process.cwd(),
|
|
1275
|
+
flags.out || `${packPath}.semantic.json`
|
|
1276
|
+
);
|
|
858
1277
|
const modelId = flags.model || 'qwen3-embedding:4b';
|
|
859
1278
|
const endpoint = flags.endpoint || 'http://localhost:11434';
|
|
860
|
-
if (!existsSync(packPath))
|
|
1279
|
+
if (!existsSync(packPath))
|
|
1280
|
+
throw createError(
|
|
1281
|
+
`Pack file not found at ${path.relative(process.cwd(), packPath)}.`
|
|
1282
|
+
);
|
|
861
1283
|
|
|
862
1284
|
const bytes = Uint8Array.from(readFileSync(packPath));
|
|
863
1285
|
const pack = await mountPackFromBytes(core, bytes);
|
|
@@ -871,7 +1293,10 @@ async function cmdSemanticIndex(core, args) {
|
|
|
871
1293
|
dimension: vectors[0]?.length ?? 0,
|
|
872
1294
|
metric: 'cosine',
|
|
873
1295
|
createdAt: new Date().toISOString(),
|
|
874
|
-
blocks: vectors.map((vector, blockId) => ({
|
|
1296
|
+
blocks: vectors.map((vector, blockId) => ({
|
|
1297
|
+
blockId,
|
|
1298
|
+
vector: Array.from(core.normalizeVector(vector)),
|
|
1299
|
+
})),
|
|
875
1300
|
};
|
|
876
1301
|
writeFileSync(outPath, core.serializeSidecar(sidecar));
|
|
877
1302
|
console.log(`✔ wrote ${path.relative(process.cwd(), outPath)}`);
|
|
@@ -881,28 +1306,44 @@ async function cmdSemanticInspect(core, args) {
|
|
|
881
1306
|
const flags = parseKeyValueArgs(args);
|
|
882
1307
|
const sidecarPath = path.resolve(process.cwd(), flags.sidecar);
|
|
883
1308
|
const sidecar = core.parseSidecar(readFileSync(sidecarPath, 'utf8'));
|
|
884
|
-
console.log(
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
1309
|
+
console.log(
|
|
1310
|
+
JSON.stringify(
|
|
1311
|
+
{
|
|
1312
|
+
version: sidecar.version,
|
|
1313
|
+
packFingerprint: sidecar.packFingerprint,
|
|
1314
|
+
modelId: sidecar.modelId,
|
|
1315
|
+
dimension: sidecar.dimension,
|
|
1316
|
+
metric: sidecar.metric,
|
|
1317
|
+
createdAt: sidecar.createdAt,
|
|
1318
|
+
blocks: sidecar.blocks.length,
|
|
1319
|
+
},
|
|
1320
|
+
null,
|
|
1321
|
+
2
|
|
1322
|
+
)
|
|
1323
|
+
);
|
|
893
1324
|
}
|
|
894
1325
|
|
|
895
1326
|
async function cmdSemanticValidate(core, args) {
|
|
896
1327
|
const flags = parseKeyValueArgs(args);
|
|
897
|
-
const packPath = path.resolve(
|
|
1328
|
+
const packPath = path.resolve(
|
|
1329
|
+
process.cwd(),
|
|
1330
|
+
flags.pack || 'dist/knowledge.knolo'
|
|
1331
|
+
);
|
|
898
1332
|
const sidecarPath = path.resolve(process.cwd(), flags.sidecar);
|
|
899
1333
|
const modelId = flags.model;
|
|
900
1334
|
if (!modelId) throw createError('semantic:validate requires --model <id>.');
|
|
901
|
-
const pack = await mountPackFromBytes(
|
|
1335
|
+
const pack = await mountPackFromBytes(
|
|
1336
|
+
core,
|
|
1337
|
+
Uint8Array.from(readFileSync(packPath))
|
|
1338
|
+
);
|
|
902
1339
|
const sidecar = core.parseSidecar(readFileSync(sidecarPath, 'utf8'));
|
|
903
1340
|
core.validateSidecarForPack({ sidecar, pack, modelId });
|
|
904
|
-
if (sidecar.blocks.length !== pack.blocks.length)
|
|
905
|
-
|
|
1341
|
+
if (sidecar.blocks.length !== pack.blocks.length)
|
|
1342
|
+
throw createError(
|
|
1343
|
+
`Semantic block count mismatch: sidecar=${sidecar.blocks.length}, pack=${pack.blocks.length}`
|
|
1344
|
+
);
|
|
1345
|
+
if (sidecar.dimension <= 0)
|
|
1346
|
+
throw createError('Semantic sidecar dimension must be > 0.');
|
|
906
1347
|
console.log('✔ semantic sidecar validation passed');
|
|
907
1348
|
}
|
|
908
1349
|
|
|
@@ -951,7 +1392,10 @@ async function cmdDev(core) {
|
|
|
951
1392
|
};
|
|
952
1393
|
|
|
953
1394
|
const closers = [];
|
|
954
|
-
const configWatcher = watch(
|
|
1395
|
+
const configWatcher = watch(
|
|
1396
|
+
path.resolve(process.cwd(), CONFIG_FILE),
|
|
1397
|
+
schedule
|
|
1398
|
+
);
|
|
955
1399
|
closers.push(() => configWatcher.close());
|
|
956
1400
|
|
|
957
1401
|
const config = readConfig();
|
|
@@ -1084,14 +1528,18 @@ function loadAgentsFromDir(agentsDir, core) {
|
|
|
1084
1528
|
const entries = readdirSync(path.resolve(agentsDir), { withFileTypes: true })
|
|
1085
1529
|
.filter((e) => e.isFile())
|
|
1086
1530
|
.map((e) => e.name)
|
|
1087
|
-
.filter((name) =>
|
|
1531
|
+
.filter((name) =>
|
|
1532
|
+
['.json', '.yaml', '.yml'].includes(path.extname(name).toLowerCase())
|
|
1533
|
+
)
|
|
1088
1534
|
.sort((a, b) => a.localeCompare(b));
|
|
1089
1535
|
|
|
1090
1536
|
const loaded = entries.map((file) => {
|
|
1091
1537
|
const full = path.join(path.resolve(agentsDir), file);
|
|
1092
1538
|
const parsed = parseAgentFileContent(readFileSync(full, 'utf8'), full);
|
|
1093
|
-
const agent =
|
|
1094
|
-
|
|
1539
|
+
const agent =
|
|
1540
|
+
parsed?.agent && typeof parsed.agent === 'object' ? parsed.agent : parsed;
|
|
1541
|
+
if (typeof core.validateAgentDefinition === 'function')
|
|
1542
|
+
core.validateAgentDefinition(agent);
|
|
1095
1543
|
return { file, agent };
|
|
1096
1544
|
});
|
|
1097
1545
|
|
|
@@ -1103,26 +1551,41 @@ function loadAgentsFromDir(agentsDir, core) {
|
|
|
1103
1551
|
}
|
|
1104
1552
|
for (const [id, files] of dupes.entries()) {
|
|
1105
1553
|
if (files.length > 1) {
|
|
1106
|
-
throw createError(
|
|
1554
|
+
throw createError(
|
|
1555
|
+
`Duplicate agent id "${id}" found in files: ${files.sort((a, b) => a.localeCompare(b)).join(', ')}`
|
|
1556
|
+
);
|
|
1107
1557
|
}
|
|
1108
1558
|
}
|
|
1109
1559
|
|
|
1110
|
-
const agents = loaded
|
|
1560
|
+
const agents = loaded
|
|
1561
|
+
.map((x) => x.agent)
|
|
1562
|
+
.sort((a, b) => String(a.id).localeCompare(String(b.id)));
|
|
1111
1563
|
const registry = { version: 1, agents };
|
|
1112
|
-
if (typeof core.validateAgentRegistry === 'function')
|
|
1564
|
+
if (typeof core.validateAgentRegistry === 'function')
|
|
1565
|
+
core.validateAgentRegistry(registry);
|
|
1113
1566
|
return registry;
|
|
1114
1567
|
}
|
|
1115
1568
|
|
|
1116
1569
|
function loadEmbeddingsFromJson(filePath, expectedCount) {
|
|
1117
1570
|
const parsed = JSON.parse(readFileSync(filePath, 'utf8'));
|
|
1118
|
-
const vectors = Array.isArray(parsed?.embeddings)
|
|
1119
|
-
|
|
1120
|
-
|
|
1571
|
+
const vectors = Array.isArray(parsed?.embeddings)
|
|
1572
|
+
? parsed.embeddings
|
|
1573
|
+
: parsed;
|
|
1574
|
+
if (!Array.isArray(vectors))
|
|
1575
|
+
throw createError(
|
|
1576
|
+
'Embeddings JSON must be an array or { embeddings: [...] }.'
|
|
1577
|
+
);
|
|
1578
|
+
if (vectors.length !== expectedCount)
|
|
1579
|
+
throw createError(
|
|
1580
|
+
`Embeddings length mismatch: expected ${expectedCount}, got ${vectors.length}.`
|
|
1581
|
+
);
|
|
1121
1582
|
return vectors.map((entry, i) => {
|
|
1122
|
-
if (!Array.isArray(entry))
|
|
1583
|
+
if (!Array.isArray(entry))
|
|
1584
|
+
throw createError(`Embeddings[${i}] must be an array.`);
|
|
1123
1585
|
const vec = new Float32Array(entry.length);
|
|
1124
1586
|
for (let d = 0; d < entry.length; d++) {
|
|
1125
|
-
if (!Number.isFinite(entry[d]))
|
|
1587
|
+
if (!Number.isFinite(entry[d]))
|
|
1588
|
+
throw createError(`Embeddings[${i}][${d}] must be numeric.`);
|
|
1126
1589
|
vec[d] = entry[d];
|
|
1127
1590
|
}
|
|
1128
1591
|
return vec;
|
|
@@ -1147,7 +1610,9 @@ function parseDirectModeArgs(args) {
|
|
|
1147
1610
|
async function runDirectMode(core, firstArg, restArgs) {
|
|
1148
1611
|
const { positional, flags } = parseDirectModeArgs([firstArg, ...restArgs]);
|
|
1149
1612
|
if (flags.help) {
|
|
1150
|
-
console.log(
|
|
1613
|
+
console.log(
|
|
1614
|
+
'Usage: knolo <input.json> [output.knolo] [--agents ./agents] [--embeddings embeddings.json --model-id model]'
|
|
1615
|
+
);
|
|
1151
1616
|
return;
|
|
1152
1617
|
}
|
|
1153
1618
|
|
|
@@ -1155,12 +1620,18 @@ async function runDirectMode(core, firstArg, restArgs) {
|
|
|
1155
1620
|
const outFile = positional[1] || 'knowledge.knolo';
|
|
1156
1621
|
if (!inFile) throw createError('Usage: knolo <input.json> [output.knolo]');
|
|
1157
1622
|
|
|
1158
|
-
const docs = JSON.parse(
|
|
1159
|
-
|
|
1623
|
+
const docs = JSON.parse(
|
|
1624
|
+
readFileSync(path.resolve(process.cwd(), inFile), 'utf8')
|
|
1625
|
+
);
|
|
1626
|
+
if (!Array.isArray(docs))
|
|
1627
|
+
throw createError('Input JSON must be an array of docs.');
|
|
1160
1628
|
|
|
1161
1629
|
const options = {};
|
|
1162
1630
|
if (flags.embeddingsPath || flags.modelId) {
|
|
1163
|
-
if (!flags.embeddingsPath || !flags.modelId)
|
|
1631
|
+
if (!flags.embeddingsPath || !flags.modelId)
|
|
1632
|
+
throw createError(
|
|
1633
|
+
'Both --embeddings and --model-id are required together.'
|
|
1634
|
+
);
|
|
1164
1635
|
options.semantic = {
|
|
1165
1636
|
enabled: true,
|
|
1166
1637
|
modelId: flags.modelId,
|
|
@@ -1182,14 +1653,19 @@ async function main() {
|
|
|
1182
1653
|
try {
|
|
1183
1654
|
if (command === 'help') return printRootHelp();
|
|
1184
1655
|
if (command === 'icp') {
|
|
1185
|
-
if (
|
|
1656
|
+
if (
|
|
1657
|
+
commandArgs.length === 0 ||
|
|
1658
|
+
commandArgs[0] === '--help' ||
|
|
1659
|
+
commandArgs[0] === '-h'
|
|
1660
|
+
) {
|
|
1186
1661
|
return printIcpHelp();
|
|
1187
1662
|
}
|
|
1188
1663
|
return await cmdIcp(commandArgs);
|
|
1189
1664
|
}
|
|
1190
1665
|
|
|
1191
1666
|
if (SUBCOMMANDS.has(command)) {
|
|
1192
|
-
if (commandArgs.includes('--help') || commandArgs.includes('-h'))
|
|
1667
|
+
if (commandArgs.includes('--help') || commandArgs.includes('-h'))
|
|
1668
|
+
return printCommandHelp(command);
|
|
1193
1669
|
if (command === 'init') return await cmdInit();
|
|
1194
1670
|
if (command === 'add') return await cmdAdd(commandArgs);
|
|
1195
1671
|
if (command === 'search') return await runHubSearch(commandArgs);
|
|
@@ -1210,17 +1686,24 @@ async function main() {
|
|
|
1210
1686
|
if (command === 'diff') return await cmdDiff(core, commandArgs);
|
|
1211
1687
|
if (command === 'dev') return await cmdDev(core);
|
|
1212
1688
|
if (command === 'v5') return await cmdV5(core, commandArgs);
|
|
1213
|
-
if (command === 'semantic:index')
|
|
1214
|
-
|
|
1215
|
-
if (command === 'semantic:
|
|
1689
|
+
if (command === 'semantic:index')
|
|
1690
|
+
return await cmdSemanticIndex(core, commandArgs);
|
|
1691
|
+
if (command === 'semantic:inspect')
|
|
1692
|
+
return await cmdSemanticInspect(core, commandArgs);
|
|
1693
|
+
if (command === 'semantic:validate')
|
|
1694
|
+
return await cmdSemanticValidate(core, commandArgs);
|
|
1216
1695
|
}
|
|
1217
1696
|
|
|
1218
|
-
if (command.startsWith('-'))
|
|
1697
|
+
if (command.startsWith('-'))
|
|
1698
|
+
throw createError(`Unknown option: ${command}`);
|
|
1219
1699
|
const core = await loadCore();
|
|
1220
1700
|
return await runDirectMode(core, command, commandArgs);
|
|
1221
1701
|
} catch (error) {
|
|
1222
|
-
console.error(
|
|
1223
|
-
|
|
1702
|
+
console.error(
|
|
1703
|
+
`knolo: ${error instanceof Error ? error.message : String(error)}`
|
|
1704
|
+
);
|
|
1705
|
+
if (global.debug && error instanceof Error && error.stack)
|
|
1706
|
+
console.error(error.stack);
|
|
1224
1707
|
process.exit(1);
|
|
1225
1708
|
}
|
|
1226
1709
|
}
|