@bendyline/gezel-cli 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/knowledge-command.js +49 -9
- package/package.json +10 -10
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
// src/knowledge-command.ts
|
|
4
4
|
import { createHash as createHash2 } from "crypto";
|
|
5
|
+
import { existsSync } from "fs";
|
|
5
6
|
import { mkdir as mkdir2, mkdtemp, readFile as readFile2, rm as rm2, stat, writeFile as writeFile2 } from "fs/promises";
|
|
6
7
|
import { homedir as homedir2, tmpdir } from "os";
|
|
7
|
-
import { basename as basename2, dirname as dirname2, join as join2, resolve as resolve2 } from "path";
|
|
8
|
+
import { basename as basename2, dirname as dirname2, join as join2, relative, resolve as resolve2 } from "path";
|
|
8
9
|
import { KnowledgeIdSchema, formatKnowledgeUri } from "@bendyline/gezel";
|
|
9
10
|
import {
|
|
10
11
|
CatalogHandle,
|
|
@@ -13,10 +14,12 @@ import {
|
|
|
13
14
|
MARKDOWN_CHUNKS_2,
|
|
14
15
|
compileKnowledgeCatalog,
|
|
15
16
|
createProfileEmbedder,
|
|
17
|
+
detectTableOfContents,
|
|
16
18
|
extractGezkVerified,
|
|
17
19
|
knowledgeEmbeddingProfile,
|
|
18
20
|
loadMarkdownCatalog,
|
|
19
21
|
readGezkManifest,
|
|
22
|
+
readMkdocsDocsDir,
|
|
20
23
|
signManifest,
|
|
21
24
|
validateExtractedCatalog
|
|
22
25
|
} from "@bendyline/gezel-knowledge";
|
|
@@ -106,23 +109,59 @@ async function runKnowledgeInit(dir) {
|
|
|
106
109
|
console.log(`Initialized knowledge catalog at ${root}`);
|
|
107
110
|
console.log(` ${CATALOG_JSON} \u2014 identity, license, embedding profile`);
|
|
108
111
|
console.log(` ${CONTENT_DIR}/ \u2014 Markdown content (folders become topics)`);
|
|
112
|
+
console.log(
|
|
113
|
+
" An outline the tree already has is honored: SUMMARY.md, an mkdocs.yml nav, _toc.yml, or Hugo _index.md pages"
|
|
114
|
+
);
|
|
109
115
|
console.log(`Build with: gezel knowledge build ${dir}`);
|
|
110
116
|
}
|
|
111
117
|
function sanitizeCatalogId(name) {
|
|
112
118
|
const slug = name.toLowerCase().normalize("NFKD").replace(/\p{Mark}/gu, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 63);
|
|
113
119
|
return KnowledgeIdSchema.safeParse(slug).success ? slug : "my-catalog";
|
|
114
120
|
}
|
|
115
|
-
async function
|
|
116
|
-
|
|
117
|
-
const config = await readCatalogConfig(root);
|
|
118
|
-
const contentDir = join2(root, CONTENT_DIR);
|
|
119
|
-
const hasContentDir = await stat(contentDir).then(
|
|
121
|
+
async function isDirectory(path) {
|
|
122
|
+
return stat(path).then(
|
|
120
123
|
(s) => s.isDirectory(),
|
|
121
124
|
() => false
|
|
122
125
|
);
|
|
123
|
-
|
|
124
|
-
|
|
126
|
+
}
|
|
127
|
+
async function resolveContentRoot(root, config) {
|
|
128
|
+
if (config.content) {
|
|
129
|
+
const dir = resolve2(root, config.content);
|
|
130
|
+
if (!await isDirectory(dir)) {
|
|
131
|
+
throw new CliError(
|
|
132
|
+
`${CATALOG_JSON} names content '${config.content}', which is not a directory`
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
return dir;
|
|
136
|
+
}
|
|
137
|
+
const contentDir = join2(root, CONTENT_DIR);
|
|
138
|
+
if (await isDirectory(contentDir)) return contentDir;
|
|
139
|
+
if (config.toc === void 0 || config.toc.format === "mkdocs") {
|
|
140
|
+
const mkdocs = config.toc?.path ? resolve2(root, config.toc.path) : [join2(root, "mkdocs.yml"), join2(root, "mkdocs.yaml")].find((p) => existsSync(p));
|
|
141
|
+
if (mkdocs && existsSync(mkdocs)) {
|
|
142
|
+
const docsDir = await readMkdocsDocsDir(mkdocs);
|
|
143
|
+
if (await isDirectory(docsDir)) return docsDir;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return root;
|
|
147
|
+
}
|
|
148
|
+
function describeToc(toc, root) {
|
|
149
|
+
const source = toc.format === "folders" ? "folders" : toc.format === "hugo" ? "Hugo conventions (_index.md, weight)" : toc.format;
|
|
150
|
+
return toc.path ? `${source} (${relative(root, toc.path) || toc.path})` : source;
|
|
151
|
+
}
|
|
152
|
+
async function runKnowledgeBuild(dir, opts, deps = {}) {
|
|
153
|
+
const root = resolve2(dir);
|
|
154
|
+
const config = await readCatalogConfig(root);
|
|
155
|
+
const contentRoot = await resolveContentRoot(root, config);
|
|
156
|
+
const toc = config.toc ?? await detectTableOfContents(contentRoot, root);
|
|
157
|
+
const source = await loadMarkdownCatalog(contentRoot, {
|
|
158
|
+
language: config.language,
|
|
159
|
+
uri: { publisherId: config.publisher.id, catalogId: config.id },
|
|
160
|
+
...config.ignore ? { ignore: config.ignore } : {},
|
|
161
|
+
toc,
|
|
162
|
+
onWarning: (message) => console.warn(`warning: ${message}`)
|
|
125
163
|
});
|
|
164
|
+
console.log(`Table of contents: ${describeToc(source.toc, root)}`);
|
|
126
165
|
const profileId = config.profile ?? "bge-small-en-v1.5@1";
|
|
127
166
|
const profile = knowledgeEmbeddingProfile(profileId);
|
|
128
167
|
if (!profile) {
|
|
@@ -131,7 +170,7 @@ async function runKnowledgeBuild(dir, opts, deps = {}) {
|
|
|
131
170
|
);
|
|
132
171
|
}
|
|
133
172
|
console.log(
|
|
134
|
-
`Building ${config.id}@${config.version}: ${source.documents.length} documents, profile ${profileId}`
|
|
173
|
+
`Building ${config.id}@${config.version}: ${source.documents.length} documents, ${source.assets.length} assets, profile ${profileId}`
|
|
135
174
|
);
|
|
136
175
|
console.log("Loading the embedding model (first run downloads it)\u2026");
|
|
137
176
|
const embedder = await (deps.createEmbedder ?? defaultCreateEmbedder)(profileId);
|
|
@@ -161,6 +200,7 @@ async function runKnowledgeBuild(dir, opts, deps = {}) {
|
|
|
161
200
|
embed: (texts) => embedder.embed(texts),
|
|
162
201
|
countTokens: (text) => embedder.countTokens(text),
|
|
163
202
|
workDir,
|
|
203
|
+
assets: source.assets,
|
|
164
204
|
...signKeyPem ? { finalizeManifest: (manifest) => signManifest(manifest, signKeyPem) } : {},
|
|
165
205
|
onProgress: ({ done, total }) => {
|
|
166
206
|
if (!process.stderr.isTTY || total === 0) return;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bendyline/gezel-cli",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.1",
|
|
4
4
|
"description": "The `gezel` command-line tool.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"keywords": [
|
|
@@ -44,15 +44,15 @@
|
|
|
44
44
|
"!dist/**/*.map"
|
|
45
45
|
],
|
|
46
46
|
"dependencies": {
|
|
47
|
-
"@bendyline/gezel": "1.1.
|
|
48
|
-
"@bendyline/gezel-app-sdk": "1.0.
|
|
49
|
-
"@bendyline/gezel-catalog": "1.1.
|
|
50
|
-
"@bendyline/gezel-client": "1.1.
|
|
51
|
-
"@bendyline/gezel-knowledge": "1.1.
|
|
52
|
-
"@bendyline/gezel-service": "1.1.
|
|
53
|
-
"@bendyline/squisq": "2.11.
|
|
54
|
-
"@bendyline/squisq-formats": "2.6.
|
|
55
|
-
"@bendyline/squisq-react": "2.11.
|
|
47
|
+
"@bendyline/gezel": "1.1.1",
|
|
48
|
+
"@bendyline/gezel-app-sdk": "1.0.9",
|
|
49
|
+
"@bendyline/gezel-catalog": "1.1.1",
|
|
50
|
+
"@bendyline/gezel-client": "1.1.1",
|
|
51
|
+
"@bendyline/gezel-knowledge": "1.1.1",
|
|
52
|
+
"@bendyline/gezel-service": "1.1.1",
|
|
53
|
+
"@bendyline/squisq": "2.11.3",
|
|
54
|
+
"@bendyline/squisq-formats": "2.6.3",
|
|
55
|
+
"@bendyline/squisq-react": "2.11.3",
|
|
56
56
|
"commander": "^15.0.0",
|
|
57
57
|
"ink": "^7.1.1",
|
|
58
58
|
"ink-text-input": "^6.0.0",
|