@bendyline/gezel-cli 1.0.6 → 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/app-author-command.js +464 -0
- package/dist/app-command.js +685 -0
- package/dist/bin/gezel-main.js +423 -25
- package/dist/handboek-export.js +713 -30
- package/dist/knowledge-command.js +351 -0
- package/dist/tui/index.js +372 -51
- package/package.json +10 -9
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/knowledge-command.ts
|
|
4
|
+
import { createHash as createHash2 } from "crypto";
|
|
5
|
+
import { mkdir as mkdir2, mkdtemp, readFile as readFile2, rm as rm2, stat, writeFile as writeFile2 } from "fs/promises";
|
|
6
|
+
import { homedir as homedir2, tmpdir } from "os";
|
|
7
|
+
import { basename as basename2, join as join2, resolve as resolve2 } from "path";
|
|
8
|
+
import { KnowledgeIdSchema, formatKnowledgeUri } from "@bendyline/gezel";
|
|
9
|
+
import {
|
|
10
|
+
CatalogHandle,
|
|
11
|
+
EmbedderUnavailableError,
|
|
12
|
+
GEZEL_MARKDOWN_CHUNKS_2,
|
|
13
|
+
KNOWLEDGE_EMBEDDING_PROFILES,
|
|
14
|
+
compileKnowledgeCatalog,
|
|
15
|
+
createProfileEmbedder,
|
|
16
|
+
extractGezkVerified,
|
|
17
|
+
knowledgeEmbeddingProfile,
|
|
18
|
+
loadMarkdownCatalog,
|
|
19
|
+
readGezkManifest,
|
|
20
|
+
signManifest,
|
|
21
|
+
validateExtractedCatalog
|
|
22
|
+
} from "@bendyline/gezel-knowledge";
|
|
23
|
+
|
|
24
|
+
// src/connection.ts
|
|
25
|
+
import { createHash, randomUUID } from "crypto";
|
|
26
|
+
import { chmod, mkdir, readFile, rm, writeFile } from "fs/promises";
|
|
27
|
+
import { homedir } from "os";
|
|
28
|
+
import { basename, dirname, join, resolve } from "path";
|
|
29
|
+
import { pickRandomNameWithGender } from "@bendyline/gezel";
|
|
30
|
+
import {
|
|
31
|
+
GezelSdkError,
|
|
32
|
+
authorizeLocal,
|
|
33
|
+
authorizeLocalOwner
|
|
34
|
+
} from "@bendyline/gezel-app-sdk";
|
|
35
|
+
import {
|
|
36
|
+
GezelApiError,
|
|
37
|
+
GezelClient,
|
|
38
|
+
createTrustingFetch,
|
|
39
|
+
discoverOrSpawn,
|
|
40
|
+
electronNativeBinCandidates,
|
|
41
|
+
isProcessAlive,
|
|
42
|
+
readRuntime,
|
|
43
|
+
readSystemServiceEndpoint,
|
|
44
|
+
resolveDaemonEntry,
|
|
45
|
+
stopOwnedDaemon,
|
|
46
|
+
systemSharedAssetsDir
|
|
47
|
+
} from "@bendyline/gezel-client/node";
|
|
48
|
+
import { gezelPaths } from "@bendyline/gezel/paths";
|
|
49
|
+
var CliError = class extends Error {
|
|
50
|
+
constructor(message) {
|
|
51
|
+
super(message);
|
|
52
|
+
this.name = "CliError";
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
// src/knowledge-command.ts
|
|
57
|
+
var CATALOG_JSON = "catalog.json";
|
|
58
|
+
var CONTENT_DIR = "content";
|
|
59
|
+
function hfCacheDir() {
|
|
60
|
+
if (process.env.GEZEL_HF_CACHE_DIR) return process.env.GEZEL_HF_CACHE_DIR;
|
|
61
|
+
const home = process.env.GEZEL_HOME ?? join2(homedir2(), ".gezel");
|
|
62
|
+
return join2(home, "engines", "hf-cache");
|
|
63
|
+
}
|
|
64
|
+
async function defaultCreateEmbedder(profileId) {
|
|
65
|
+
const profile = knowledgeEmbeddingProfile(profileId);
|
|
66
|
+
if (!profile) {
|
|
67
|
+
throw new CliError(
|
|
68
|
+
`unknown embedding profile '${profileId}' \u2014 registered: ${KNOWLEDGE_EMBEDDING_PROFILES.map((p) => p.id).join(", ")}`
|
|
69
|
+
);
|
|
70
|
+
}
|
|
71
|
+
try {
|
|
72
|
+
return await createProfileEmbedder(profile, { cacheDir: hfCacheDir() });
|
|
73
|
+
} catch (err) {
|
|
74
|
+
if (err instanceof EmbedderUnavailableError) throw new CliError(err.message);
|
|
75
|
+
throw err;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
async function runKnowledgeInit(dir) {
|
|
79
|
+
const root = resolve2(dir);
|
|
80
|
+
const configPath = join2(root, CATALOG_JSON);
|
|
81
|
+
const exists = await stat(configPath).then(
|
|
82
|
+
() => true,
|
|
83
|
+
() => false
|
|
84
|
+
);
|
|
85
|
+
if (exists) throw new CliError(`${configPath} already exists`);
|
|
86
|
+
await mkdir2(join2(root, CONTENT_DIR, "Getting Started"), { recursive: true });
|
|
87
|
+
const id = sanitizeCatalogId(basename2(root));
|
|
88
|
+
const config = {
|
|
89
|
+
id,
|
|
90
|
+
version: "1.0.0",
|
|
91
|
+
name: basename2(root),
|
|
92
|
+
description: "",
|
|
93
|
+
language: "en",
|
|
94
|
+
publisher: { id, name: basename2(root) },
|
|
95
|
+
license: { name: "All rights reserved", attributionRequired: false },
|
|
96
|
+
profile: "gezel-bge-small-en-v1.5@1"
|
|
97
|
+
};
|
|
98
|
+
await writeFile2(configPath, `${JSON.stringify(config, null, 2)}
|
|
99
|
+
`, "utf8");
|
|
100
|
+
await writeFile2(
|
|
101
|
+
join2(root, CONTENT_DIR, "Getting Started", "welcome.md"),
|
|
102
|
+
"# Welcome\n\nPut Markdown files under content/ \u2014 folders become the table of contents.\n",
|
|
103
|
+
"utf8"
|
|
104
|
+
);
|
|
105
|
+
console.log(`Initialized knowledge catalog at ${root}`);
|
|
106
|
+
console.log(` ${CATALOG_JSON} \u2014 identity, license, embedding profile`);
|
|
107
|
+
console.log(` ${CONTENT_DIR}/ \u2014 Markdown content (folders become topics)`);
|
|
108
|
+
console.log(`Build with: gezel knowledge build ${dir}`);
|
|
109
|
+
}
|
|
110
|
+
function sanitizeCatalogId(name) {
|
|
111
|
+
const slug = name.toLowerCase().normalize("NFKD").replace(/\p{Mark}/gu, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 63);
|
|
112
|
+
return KnowledgeIdSchema.safeParse(slug).success ? slug : "my-catalog";
|
|
113
|
+
}
|
|
114
|
+
async function runKnowledgeBuild(dir, opts, deps = {}) {
|
|
115
|
+
const root = resolve2(dir);
|
|
116
|
+
const config = await readCatalogConfig(root);
|
|
117
|
+
const contentDir = join2(root, CONTENT_DIR);
|
|
118
|
+
const hasContentDir = await stat(contentDir).then(
|
|
119
|
+
(s) => s.isDirectory(),
|
|
120
|
+
() => false
|
|
121
|
+
);
|
|
122
|
+
const source = await loadMarkdownCatalog(hasContentDir ? contentDir : root, {
|
|
123
|
+
language: config.language
|
|
124
|
+
});
|
|
125
|
+
const profileId = config.profile ?? "gezel-bge-small-en-v1.5@1";
|
|
126
|
+
const profile = knowledgeEmbeddingProfile(profileId);
|
|
127
|
+
if (!profile) {
|
|
128
|
+
throw new CliError(
|
|
129
|
+
`catalog.json names unknown profile '${profileId}' \u2014 registered: ${KNOWLEDGE_EMBEDDING_PROFILES.map((p) => p.id).join(", ")}`
|
|
130
|
+
);
|
|
131
|
+
}
|
|
132
|
+
console.log(
|
|
133
|
+
`Building ${config.id}@${config.version}: ${source.documents.length} documents, profile ${profileId}`
|
|
134
|
+
);
|
|
135
|
+
console.log("Loading the embedding model (first run downloads it)\u2026");
|
|
136
|
+
const embedder = await (deps.createEmbedder ?? defaultCreateEmbedder)(profileId);
|
|
137
|
+
const signKeyPem = opts.signKey ? await readFile2(resolve2(opts.signKey), "utf8") : null;
|
|
138
|
+
const outputPath = resolve2(opts.out ?? join2(root, `${config.id}-${config.version}.gezk`));
|
|
139
|
+
const workDir = await mkdtemp(join2(tmpdir(), "gezk-build-"));
|
|
140
|
+
try {
|
|
141
|
+
let lastPct = -1;
|
|
142
|
+
const report = await compileKnowledgeCatalog({
|
|
143
|
+
catalog: {
|
|
144
|
+
id: config.id,
|
|
145
|
+
version: config.version,
|
|
146
|
+
name: config.name,
|
|
147
|
+
...config.description ? { description: config.description } : {},
|
|
148
|
+
language: config.language,
|
|
149
|
+
publisher: config.publisher,
|
|
150
|
+
createdAt: config.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
151
|
+
license: config.license
|
|
152
|
+
},
|
|
153
|
+
topics: source.topics,
|
|
154
|
+
documents: (async function* () {
|
|
155
|
+
for (const doc of source.documents) yield doc;
|
|
156
|
+
})(),
|
|
157
|
+
outputPath,
|
|
158
|
+
embeddingProfile: embedder.profile,
|
|
159
|
+
chunkingProfile: GEZEL_MARKDOWN_CHUNKS_2,
|
|
160
|
+
embed: (texts) => embedder.embed(texts),
|
|
161
|
+
countTokens: (text) => embedder.countTokens(text),
|
|
162
|
+
workDir,
|
|
163
|
+
...signKeyPem ? { finalizeManifest: (manifest) => signManifest(manifest, signKeyPem) } : {},
|
|
164
|
+
onProgress: ({ done, total }) => {
|
|
165
|
+
if (!process.stderr.isTTY || total === 0) return;
|
|
166
|
+
const pct = Math.floor(done / total * 100);
|
|
167
|
+
if (pct !== lastPct) {
|
|
168
|
+
lastPct = pct;
|
|
169
|
+
process.stderr.write(`\rEmbedding chunks: ${done}/${total} (${pct}%)`);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
});
|
|
173
|
+
if (process.stderr.isTTY && lastPct >= 0) process.stderr.write("\n");
|
|
174
|
+
console.log(
|
|
175
|
+
`Wrote ${outputPath} \u2014 ${report.documents} documents, ${report.chunks} chunks, ${report.shards} shard${report.shards === 1 ? "" : "s"}, ${formatBytes(report.archiveBytes)}${report.manifest.signature ? `, signed (key ${report.manifest.signature.keyId})` : ""}`
|
|
176
|
+
);
|
|
177
|
+
} finally {
|
|
178
|
+
await embedder.dispose().catch(() => {
|
|
179
|
+
});
|
|
180
|
+
await rm2(workDir, { recursive: true, force: true });
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
async function readCatalogConfig(root) {
|
|
184
|
+
const configPath = join2(root, CATALOG_JSON);
|
|
185
|
+
let raw;
|
|
186
|
+
try {
|
|
187
|
+
raw = await readFile2(configPath, "utf8");
|
|
188
|
+
} catch {
|
|
189
|
+
throw new CliError(`no ${CATALOG_JSON} in ${root} \u2014 run 'gezel knowledge init ${root}' first`);
|
|
190
|
+
}
|
|
191
|
+
const parsed = JSON.parse(raw);
|
|
192
|
+
for (const field of ["id", "version", "name", "language", "publisher", "license"]) {
|
|
193
|
+
if (!parsed[field]) throw new CliError(`${CATALOG_JSON} is missing '${field}'`);
|
|
194
|
+
}
|
|
195
|
+
return parsed;
|
|
196
|
+
}
|
|
197
|
+
async function runKnowledgeValidate(path, opts) {
|
|
198
|
+
const { rootDir, cleanup } = await materializeCatalog(path);
|
|
199
|
+
try {
|
|
200
|
+
const report = await validateExtractedCatalog(rootDir, { deep: opts.deep });
|
|
201
|
+
for (const check of report.checks) {
|
|
202
|
+
const mark = check.ok ? "ok " : "FAIL";
|
|
203
|
+
console.log(` ${mark} ${check.name}${check.detail ? ` \u2014 ${check.detail}` : ""}`);
|
|
204
|
+
}
|
|
205
|
+
if (!report.ok) throw new CliError(`${basename2(path)} failed validation`);
|
|
206
|
+
console.log(
|
|
207
|
+
`${basename2(path)} is valid: ${report.manifest?.counts.documents} documents, ${report.manifest?.counts.chunks} chunks${opts.deep ? " (deep)" : ""}`
|
|
208
|
+
);
|
|
209
|
+
} finally {
|
|
210
|
+
await cleanup();
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
async function runKnowledgeInspect(path) {
|
|
214
|
+
const manifest = await manifestFor(resolve2(path));
|
|
215
|
+
const rows = [
|
|
216
|
+
["catalog", `${manifest.id}@${manifest.version} \u2014 ${manifest.name}`],
|
|
217
|
+
["publisher", `${manifest.publisher.name} (${manifest.publisher.id})`],
|
|
218
|
+
["language", manifest.language],
|
|
219
|
+
["license", manifest.license.name],
|
|
220
|
+
["created", manifest.createdAt],
|
|
221
|
+
[
|
|
222
|
+
"counts",
|
|
223
|
+
`${manifest.counts.documents} documents, ${manifest.counts.chunks} chunks, ${manifest.counts.shards} shard${manifest.counts.shards === 1 ? "" : "s"}`
|
|
224
|
+
],
|
|
225
|
+
["topics", manifest.topics.map((t) => t.name).join(", ")],
|
|
226
|
+
["profile", `${manifest.embedding.id} (${manifest.embedding.model.repo})`],
|
|
227
|
+
["chunking", manifest.chunking.id],
|
|
228
|
+
["signature", manifest.signature ? `ed25519, key ${manifest.signature.keyId}` : "unsigned"],
|
|
229
|
+
[
|
|
230
|
+
"files",
|
|
231
|
+
`${manifest.files.length} (${formatBytes(manifest.files.reduce((s, f) => s + f.sizeBytes, 0))} extracted)`
|
|
232
|
+
]
|
|
233
|
+
];
|
|
234
|
+
if (manifest.description) rows.splice(1, 0, ["description", manifest.description]);
|
|
235
|
+
const width = Math.max(...rows.map(([k]) => k.length));
|
|
236
|
+
for (const [key, value] of rows) console.log(`${key.padEnd(width)} ${value}`);
|
|
237
|
+
}
|
|
238
|
+
async function runKnowledgeSearch(path, query, opts, deps = {}) {
|
|
239
|
+
const limit = Math.min(50, Math.max(1, opts.limit ?? 10));
|
|
240
|
+
const { rootDir, cleanup } = await materializeCatalog(path);
|
|
241
|
+
try {
|
|
242
|
+
const manifest = JSON.parse(
|
|
243
|
+
await readFile2(join2(rootDir, "manifest.json"), "utf8")
|
|
244
|
+
);
|
|
245
|
+
const handle = CatalogHandle.open(rootDir);
|
|
246
|
+
try {
|
|
247
|
+
const docHits = handle.searchDocumentsFts(query, limit);
|
|
248
|
+
if (docHits.length > 0) {
|
|
249
|
+
console.log("Documents:");
|
|
250
|
+
for (const hit of docHits) {
|
|
251
|
+
const doc = handle.getDocument(hit.documentId);
|
|
252
|
+
console.log(
|
|
253
|
+
` ${doc?.title ?? hit.documentId} ${formatKnowledgeUri({ catalogId: manifest.id, documentId: hit.documentId })}`
|
|
254
|
+
);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
let chunkHits = handle.searchChunksFts(
|
|
258
|
+
query,
|
|
259
|
+
handle.shards.map((s) => s.id),
|
|
260
|
+
Math.ceil(limit / Math.max(1, handle.shards.length))
|
|
261
|
+
);
|
|
262
|
+
if (opts.semantic) {
|
|
263
|
+
const embedder = await (deps.createEmbedder ?? defaultCreateEmbedder)(
|
|
264
|
+
manifest.embedding.id
|
|
265
|
+
);
|
|
266
|
+
try {
|
|
267
|
+
const vector = await embedder.embedQuery(query);
|
|
268
|
+
chunkHits = [...handle.searchSemantic(vector, { finalK: limit }), ...chunkHits];
|
|
269
|
+
} finally {
|
|
270
|
+
await embedder.dispose().catch(() => {
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
const seen = /* @__PURE__ */ new Set();
|
|
275
|
+
const deduped = chunkHits.filter((h) => !seen.has(h.chunkUid) && seen.add(h.chunkUid));
|
|
276
|
+
if (deduped.length > 0) {
|
|
277
|
+
console.log(opts.semantic ? "Passages (semantic + full-text):" : "Passages (full-text):");
|
|
278
|
+
for (const hit of deduped.slice(0, limit)) {
|
|
279
|
+
const uri = formatKnowledgeUri({
|
|
280
|
+
catalogId: manifest.id,
|
|
281
|
+
documentId: hit.documentId,
|
|
282
|
+
fragment: { chunk: hit.chunkUid }
|
|
283
|
+
});
|
|
284
|
+
const score = hit.cosine !== void 0 ? ` (${hit.cosine.toFixed(3)})` : "";
|
|
285
|
+
console.log(` ${hit.title}${score} ${uri}`);
|
|
286
|
+
console.log(` ${excerpt(hit.text)}`);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
if (docHits.length === 0 && deduped.length === 0) {
|
|
290
|
+
console.log(`No results for "${query}".`);
|
|
291
|
+
process.exitCode = 1;
|
|
292
|
+
}
|
|
293
|
+
} finally {
|
|
294
|
+
handle.close();
|
|
295
|
+
}
|
|
296
|
+
} finally {
|
|
297
|
+
await cleanup();
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
async function materializeCatalog(path) {
|
|
301
|
+
const abs = resolve2(path);
|
|
302
|
+
const info = await stat(abs).catch(() => null);
|
|
303
|
+
if (!info) throw new CliError(`no such file or directory: ${abs}`);
|
|
304
|
+
if (info.isDirectory()) {
|
|
305
|
+
const hasManifest = await stat(join2(abs, "manifest.json")).then(
|
|
306
|
+
() => true,
|
|
307
|
+
() => false
|
|
308
|
+
);
|
|
309
|
+
if (!hasManifest) throw new CliError(`${abs} has no manifest.json \u2014 not an extracted catalog`);
|
|
310
|
+
return { rootDir: abs, cleanup: async () => {
|
|
311
|
+
} };
|
|
312
|
+
}
|
|
313
|
+
const dest = await mkdtemp(join2(tmpdir(), "gezk-cli-"));
|
|
314
|
+
try {
|
|
315
|
+
await extractGezkVerified(abs, dest);
|
|
316
|
+
} catch (err) {
|
|
317
|
+
await rm2(dest, { recursive: true, force: true });
|
|
318
|
+
throw new CliError(
|
|
319
|
+
`archive verification failed: ${err instanceof Error ? err.message : String(err)}`
|
|
320
|
+
);
|
|
321
|
+
}
|
|
322
|
+
return { rootDir: dest, cleanup: () => rm2(dest, { recursive: true, force: true }) };
|
|
323
|
+
}
|
|
324
|
+
async function manifestFor(abs) {
|
|
325
|
+
const info = await stat(abs).catch(() => null);
|
|
326
|
+
if (!info) throw new CliError(`no such file or directory: ${abs}`);
|
|
327
|
+
if (info.isDirectory()) {
|
|
328
|
+
return JSON.parse(await readFile2(join2(abs, "manifest.json"), "utf8"));
|
|
329
|
+
}
|
|
330
|
+
return readGezkManifest(abs);
|
|
331
|
+
}
|
|
332
|
+
function excerpt(text) {
|
|
333
|
+
const flat = text.replace(/\s+/g, " ").trim();
|
|
334
|
+
return flat.length > 160 ? `${flat.slice(0, 159)}\u2026` : flat;
|
|
335
|
+
}
|
|
336
|
+
function formatBytes(bytes) {
|
|
337
|
+
if (bytes >= 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
338
|
+
if (bytes >= 1024) return `${Math.round(bytes / 1024)} KB`;
|
|
339
|
+
return `${bytes} B`;
|
|
340
|
+
}
|
|
341
|
+
function contentSha256(bytes) {
|
|
342
|
+
return createHash2("sha256").update(bytes).digest("hex");
|
|
343
|
+
}
|
|
344
|
+
export {
|
|
345
|
+
contentSha256,
|
|
346
|
+
runKnowledgeBuild,
|
|
347
|
+
runKnowledgeInit,
|
|
348
|
+
runKnowledgeInspect,
|
|
349
|
+
runKnowledgeSearch,
|
|
350
|
+
runKnowledgeValidate
|
|
351
|
+
};
|