deepskill 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.mjs +13 -0
- package/dist/cli.d.mts +1 -0
- package/dist/cli.mjs +676 -0
- package/package.json +50 -0
package/bin/cli.mjs
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import module from 'node:module';
|
|
4
|
+
|
|
5
|
+
if (module.enableCompileCache && !process.env.NODE_DISABLE_COMPILE_CACHE) {
|
|
6
|
+
try {
|
|
7
|
+
module.enableCompileCache();
|
|
8
|
+
} catch {
|
|
9
|
+
// Ignore errors
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
await import('../dist/cli.mjs');
|
package/dist/cli.d.mts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { };
|
package/dist/cli.mjs
ADDED
|
@@ -0,0 +1,676 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "fs";
|
|
3
|
+
import { basename, dirname, extname, join, relative } from "path";
|
|
4
|
+
import { fileURLToPath } from "url";
|
|
5
|
+
import pc from "picocolors";
|
|
6
|
+
import { create, insert, load, save, search } from "@orama/orama";
|
|
7
|
+
import { createHash } from "crypto";
|
|
8
|
+
import matter from "gray-matter";
|
|
9
|
+
//#region src/commands/init.ts
|
|
10
|
+
const SKILL_TEMPLATE = (name) => `---
|
|
11
|
+
name: ${name}
|
|
12
|
+
description: >-
|
|
13
|
+
A brief description of what this skill does.
|
|
14
|
+
Use when the user asks about [topic].
|
|
15
|
+
knowledge: true
|
|
16
|
+
metadata:
|
|
17
|
+
author: your-name
|
|
18
|
+
version: "0.1.0"
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
# ${name}
|
|
22
|
+
|
|
23
|
+
Instructions for the agent to follow when this skill is activated.
|
|
24
|
+
|
|
25
|
+
## When to use
|
|
26
|
+
|
|
27
|
+
Describe when this skill should be used.
|
|
28
|
+
|
|
29
|
+
## Searching knowledge
|
|
30
|
+
|
|
31
|
+
Run the search script to query the embedded knowledge base:
|
|
32
|
+
|
|
33
|
+
\`\`\`bash
|
|
34
|
+
bash scripts/search.sh "your query here"
|
|
35
|
+
\`\`\`
|
|
36
|
+
|
|
37
|
+
Always search before answering domain-specific questions.
|
|
38
|
+
Do NOT rely on training data for specific rules or thresholds.
|
|
39
|
+
Cite the source_file and section from search results.
|
|
40
|
+
|
|
41
|
+
## Instructions
|
|
42
|
+
|
|
43
|
+
1. First, search the knowledge base for relevant information
|
|
44
|
+
2. Use the search results to ground your response
|
|
45
|
+
3. Always cite the source file and section
|
|
46
|
+
4. If no results found, say so explicitly
|
|
47
|
+
`;
|
|
48
|
+
const SEARCH_SCRIPT = `#!/bin/bash
|
|
49
|
+
set -e
|
|
50
|
+
|
|
51
|
+
SCRIPT_DIR="$(cd "$(dirname "\${BASH_SOURCE[0]}")" && pwd)"
|
|
52
|
+
SKILL_DIR="$(dirname "$SCRIPT_DIR")"
|
|
53
|
+
INDEX_PATH="$SKILL_DIR/knowledge/index.json"
|
|
54
|
+
|
|
55
|
+
if [ ! -f "$INDEX_PATH" ]; then
|
|
56
|
+
echo '{"error": "No index found. Run: npx deepskill build"}'
|
|
57
|
+
exit 1
|
|
58
|
+
fi
|
|
59
|
+
|
|
60
|
+
if [ -z "$1" ]; then
|
|
61
|
+
echo '{"error": "Usage: search.sh <query>"}'
|
|
62
|
+
exit 1
|
|
63
|
+
fi
|
|
64
|
+
|
|
65
|
+
node --input-type=module -e "
|
|
66
|
+
import { create, load, search } from '@orama/orama';
|
|
67
|
+
import { readFileSync } from 'fs';
|
|
68
|
+
|
|
69
|
+
const idx = JSON.parse(readFileSync('$INDEX_PATH', 'utf-8'));
|
|
70
|
+
const db = create({
|
|
71
|
+
schema: {
|
|
72
|
+
title: 'string',
|
|
73
|
+
content: 'string',
|
|
74
|
+
source_file: 'string',
|
|
75
|
+
section: 'string',
|
|
76
|
+
tags: 'string',
|
|
77
|
+
},
|
|
78
|
+
});
|
|
79
|
+
load(db, idx);
|
|
80
|
+
const r = search(db, { term: \\\`$1\\\`, limit: 5 });
|
|
81
|
+
console.log(JSON.stringify(r.hits.map(h => ({
|
|
82
|
+
title: h.document.title,
|
|
83
|
+
content: h.document.content,
|
|
84
|
+
source: h.document.source_file,
|
|
85
|
+
section: h.document.section,
|
|
86
|
+
score: h.score,
|
|
87
|
+
})), null, 2));
|
|
88
|
+
"
|
|
89
|
+
`;
|
|
90
|
+
const DEEPSKILL_JSON_TEMPLATE = (name) => JSON.stringify({
|
|
91
|
+
name,
|
|
92
|
+
version: "0.1.0",
|
|
93
|
+
author: "",
|
|
94
|
+
description: "",
|
|
95
|
+
corpus: {
|
|
96
|
+
files: 0,
|
|
97
|
+
chunks: 0,
|
|
98
|
+
tokens: 0
|
|
99
|
+
},
|
|
100
|
+
index_hash: "",
|
|
101
|
+
built_at: ""
|
|
102
|
+
}, null, 2);
|
|
103
|
+
function runInit(args) {
|
|
104
|
+
const cwd = process.cwd();
|
|
105
|
+
const name = args[0] || basename(cwd);
|
|
106
|
+
const hasName = args[0] !== void 0;
|
|
107
|
+
const skillDir = hasName ? join(cwd, name) : cwd;
|
|
108
|
+
if (hasName && existsSync(skillDir)) {
|
|
109
|
+
console.log(pc.red(` Directory already exists: ${name}/`));
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
const skillFile = join(skillDir, "SKILL.md");
|
|
113
|
+
if (existsSync(skillFile)) {
|
|
114
|
+
console.log(pc.yellow(` SKILL.md already exists in ${hasName ? name + "/" : "./"}`));
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
mkdirSync(join(skillDir, "knowledge", "corpus"), { recursive: true });
|
|
118
|
+
mkdirSync(join(skillDir, "scripts"), { recursive: true });
|
|
119
|
+
writeFileSync(skillFile, SKILL_TEMPLATE(name));
|
|
120
|
+
writeFileSync(join(skillDir, "knowledge", "corpus", ".gitkeep"), "");
|
|
121
|
+
writeFileSync(join(skillDir, "scripts", "search.sh"), SEARCH_SCRIPT, { mode: 493 });
|
|
122
|
+
writeFileSync(join(skillDir, "deepskill.json"), DEEPSKILL_JSON_TEMPLATE(name));
|
|
123
|
+
const prefix = hasName ? `${name}/` : "";
|
|
124
|
+
console.log(pc.white(` Created ${hasName ? name + "/" : "deep skill"}`));
|
|
125
|
+
console.log();
|
|
126
|
+
console.log(pc.dim(" Files:"));
|
|
127
|
+
console.log(` ${prefix}SKILL.md`);
|
|
128
|
+
console.log(` ${prefix}knowledge/corpus/ ${pc.dim("(drop your .md, .csv, .txt, .json files here)")}`);
|
|
129
|
+
console.log(` ${prefix}scripts/search.sh ${pc.dim("(pre-wired search bridge for agents)")}`);
|
|
130
|
+
console.log(` ${prefix}deepskill.json`);
|
|
131
|
+
console.log();
|
|
132
|
+
console.log(pc.dim(" Next steps:"));
|
|
133
|
+
console.log(` 1. Edit ${pc.white("SKILL.md")} with your skill instructions`);
|
|
134
|
+
console.log(` 2. Add documents to ${pc.white("knowledge/corpus/")}`);
|
|
135
|
+
console.log(` 3. Run: ${pc.white("npx deepskill build")}`);
|
|
136
|
+
console.log(` 4. Test: ${pc.white("npx deepskill search \"your query\"")}`);
|
|
137
|
+
console.log();
|
|
138
|
+
}
|
|
139
|
+
//#endregion
|
|
140
|
+
//#region src/chunkers/markdown.ts
|
|
141
|
+
const HEADING_RE = /^(#{1,6})\s+(.+)$/;
|
|
142
|
+
function extractKeywords$1(text) {
|
|
143
|
+
const words = text.toLowerCase().replace(/[^a-z0-9\s-]/g, "").split(/\s+/).filter((w) => w.length > 3);
|
|
144
|
+
const freq = /* @__PURE__ */ new Map();
|
|
145
|
+
for (const w of words) freq.set(w, (freq.get(w) ?? 0) + 1);
|
|
146
|
+
return [...freq.entries()].sort((a, b) => b[1] - a[1]).slice(0, 8).map(([w]) => w).join(", ");
|
|
147
|
+
}
|
|
148
|
+
function chunkMarkdown(content, sourceFile) {
|
|
149
|
+
const lines = content.split("\n");
|
|
150
|
+
const chunks = [];
|
|
151
|
+
const headingStack = [];
|
|
152
|
+
let currentTitle = "";
|
|
153
|
+
let currentLines = [];
|
|
154
|
+
function flush() {
|
|
155
|
+
const body = currentLines.join("\n").trim();
|
|
156
|
+
if (!body) return;
|
|
157
|
+
const section = headingStack.join(" > ");
|
|
158
|
+
chunks.push({
|
|
159
|
+
title: currentTitle || sourceFile,
|
|
160
|
+
content: body,
|
|
161
|
+
source_file: sourceFile,
|
|
162
|
+
section,
|
|
163
|
+
tags: extractKeywords$1(body)
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
for (const line of lines) {
|
|
167
|
+
const match = HEADING_RE.exec(line);
|
|
168
|
+
if (match) {
|
|
169
|
+
flush();
|
|
170
|
+
currentLines = [];
|
|
171
|
+
const level = match[1].length;
|
|
172
|
+
const text = match[2].trim();
|
|
173
|
+
while (headingStack.length >= level) headingStack.pop();
|
|
174
|
+
headingStack.push(text);
|
|
175
|
+
currentTitle = text;
|
|
176
|
+
} else currentLines.push(line);
|
|
177
|
+
}
|
|
178
|
+
flush();
|
|
179
|
+
return chunks;
|
|
180
|
+
}
|
|
181
|
+
//#endregion
|
|
182
|
+
//#region src/chunkers/csv.ts
|
|
183
|
+
function parseCsvLine(line) {
|
|
184
|
+
const cells = [];
|
|
185
|
+
let current = "";
|
|
186
|
+
let inQuotes = false;
|
|
187
|
+
for (let i = 0; i < line.length; i++) {
|
|
188
|
+
const char = line[i];
|
|
189
|
+
if (char === "\"") if (inQuotes && line[i + 1] === "\"") {
|
|
190
|
+
current += "\"";
|
|
191
|
+
i++;
|
|
192
|
+
} else inQuotes = !inQuotes;
|
|
193
|
+
else if (char === "," && !inQuotes) {
|
|
194
|
+
cells.push(current.trim());
|
|
195
|
+
current = "";
|
|
196
|
+
} else current += char;
|
|
197
|
+
}
|
|
198
|
+
cells.push(current.trim());
|
|
199
|
+
return cells;
|
|
200
|
+
}
|
|
201
|
+
function chunkCsv(content, sourceFile) {
|
|
202
|
+
const lines = content.split("\n").filter((l) => l.trim());
|
|
203
|
+
if (lines.length < 2) return [];
|
|
204
|
+
const headers = parseCsvLine(lines[0]);
|
|
205
|
+
const chunks = [];
|
|
206
|
+
for (let i = 1; i < lines.length; i++) {
|
|
207
|
+
const cells = parseCsvLine(lines[i]);
|
|
208
|
+
const title = cells[0] ?? `Row ${i}`;
|
|
209
|
+
const pairs = headers.map((h, j) => `${h}: ${cells[j] ?? ""}`);
|
|
210
|
+
chunks.push({
|
|
211
|
+
title,
|
|
212
|
+
content: pairs.join("\n"),
|
|
213
|
+
source_file: sourceFile,
|
|
214
|
+
section: `Row ${i}`,
|
|
215
|
+
tags: headers.slice(0, 5).join(", ")
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
return chunks;
|
|
219
|
+
}
|
|
220
|
+
//#endregion
|
|
221
|
+
//#region src/chunkers/text.ts
|
|
222
|
+
const WINDOW_CHARS = 2048;
|
|
223
|
+
const OVERLAP_CHARS = 256;
|
|
224
|
+
function extractKeywords(text) {
|
|
225
|
+
const words = text.toLowerCase().replace(/[^a-z0-9\s-]/g, "").split(/\s+/).filter((w) => w.length > 3);
|
|
226
|
+
const freq = /* @__PURE__ */ new Map();
|
|
227
|
+
for (const w of words) freq.set(w, (freq.get(w) ?? 0) + 1);
|
|
228
|
+
return [...freq.entries()].sort((a, b) => b[1] - a[1]).slice(0, 8).map(([w]) => w).join(", ");
|
|
229
|
+
}
|
|
230
|
+
function chunkText(content, sourceFile) {
|
|
231
|
+
const chunks = [];
|
|
232
|
+
let offset = 0;
|
|
233
|
+
let chunkIndex = 0;
|
|
234
|
+
while (offset < content.length) {
|
|
235
|
+
const window = content.slice(offset, offset + WINDOW_CHARS);
|
|
236
|
+
const firstLine = window.split("\n")[0]?.trim() || `Chunk ${chunkIndex + 1}`;
|
|
237
|
+
chunks.push({
|
|
238
|
+
title: firstLine.slice(0, 120),
|
|
239
|
+
content: window.trim(),
|
|
240
|
+
source_file: sourceFile,
|
|
241
|
+
section: `Chunk ${chunkIndex + 1}`,
|
|
242
|
+
tags: extractKeywords(window)
|
|
243
|
+
});
|
|
244
|
+
offset += WINDOW_CHARS - OVERLAP_CHARS;
|
|
245
|
+
chunkIndex++;
|
|
246
|
+
}
|
|
247
|
+
return chunks;
|
|
248
|
+
}
|
|
249
|
+
//#endregion
|
|
250
|
+
//#region src/chunkers/json.ts
|
|
251
|
+
function chunkJson(content, sourceFile) {
|
|
252
|
+
let parsed;
|
|
253
|
+
try {
|
|
254
|
+
parsed = JSON.parse(content);
|
|
255
|
+
} catch {
|
|
256
|
+
return chunkText(content, sourceFile);
|
|
257
|
+
}
|
|
258
|
+
if (Array.isArray(parsed)) return parsed.flatMap((item, i) => {
|
|
259
|
+
const str = JSON.stringify(item, null, 2);
|
|
260
|
+
return [{
|
|
261
|
+
title: (typeof item === "object" && item !== null && "name" in item ? String(item.name) : null) ?? (typeof item === "object" && item !== null && "title" in item ? String(item.title) : null) ?? `Item ${i + 1}`,
|
|
262
|
+
content: str,
|
|
263
|
+
source_file: sourceFile,
|
|
264
|
+
section: `Item ${i + 1}`,
|
|
265
|
+
tags: ""
|
|
266
|
+
}];
|
|
267
|
+
});
|
|
268
|
+
return chunkText(JSON.stringify(parsed, null, 2), sourceFile);
|
|
269
|
+
}
|
|
270
|
+
//#endregion
|
|
271
|
+
//#region src/chunkers/index.ts
|
|
272
|
+
const SUPPORTED_EXTENSIONS = new Set([
|
|
273
|
+
".md",
|
|
274
|
+
".csv",
|
|
275
|
+
".txt",
|
|
276
|
+
".json"
|
|
277
|
+
]);
|
|
278
|
+
function isSupportedFile(filePath) {
|
|
279
|
+
return SUPPORTED_EXTENSIONS.has(extname(filePath).toLowerCase());
|
|
280
|
+
}
|
|
281
|
+
function chunkFile(content, sourceFile) {
|
|
282
|
+
switch (extname(sourceFile).toLowerCase()) {
|
|
283
|
+
case ".md": return chunkMarkdown(content, sourceFile);
|
|
284
|
+
case ".csv": return chunkCsv(content, sourceFile);
|
|
285
|
+
case ".json": return chunkJson(content, sourceFile);
|
|
286
|
+
default: return chunkText(content, sourceFile);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
//#endregion
|
|
290
|
+
//#region src/indexer/schema.ts
|
|
291
|
+
const SKILL_SCHEMA = {
|
|
292
|
+
title: "string",
|
|
293
|
+
content: "string",
|
|
294
|
+
source_file: "string",
|
|
295
|
+
section: "string",
|
|
296
|
+
tags: "string"
|
|
297
|
+
};
|
|
298
|
+
//#endregion
|
|
299
|
+
//#region src/indexer/build.ts
|
|
300
|
+
function buildIndex(chunks) {
|
|
301
|
+
const db = create({ schema: SKILL_SCHEMA });
|
|
302
|
+
for (const chunk of chunks) insert(db, {
|
|
303
|
+
title: chunk.title,
|
|
304
|
+
content: chunk.content,
|
|
305
|
+
source_file: chunk.source_file,
|
|
306
|
+
section: chunk.section,
|
|
307
|
+
tags: chunk.tags
|
|
308
|
+
});
|
|
309
|
+
return save(db);
|
|
310
|
+
}
|
|
311
|
+
//#endregion
|
|
312
|
+
//#region src/utils/hash.ts
|
|
313
|
+
function sha256(data) {
|
|
314
|
+
return createHash("sha256").update(data).digest("hex");
|
|
315
|
+
}
|
|
316
|
+
//#endregion
|
|
317
|
+
//#region src/utils/tokens.ts
|
|
318
|
+
/**
|
|
319
|
+
* Naive token count: ~4 characters per token.
|
|
320
|
+
* Good enough for corpus stats without pulling in a tokenizer.
|
|
321
|
+
*/
|
|
322
|
+
function countTokens(text) {
|
|
323
|
+
return Math.ceil(text.length / 4);
|
|
324
|
+
}
|
|
325
|
+
//#endregion
|
|
326
|
+
//#region src/commands/build.ts
|
|
327
|
+
function walkDir(dir) {
|
|
328
|
+
const results = [];
|
|
329
|
+
if (!existsSync(dir)) return results;
|
|
330
|
+
const entries = readdirSync(dir);
|
|
331
|
+
for (const entry of entries) {
|
|
332
|
+
const full = join(dir, entry);
|
|
333
|
+
if (statSync(full).isDirectory()) results.push(...walkDir(full));
|
|
334
|
+
else if (isSupportedFile(full)) results.push(full);
|
|
335
|
+
}
|
|
336
|
+
return results;
|
|
337
|
+
}
|
|
338
|
+
async function runBuild() {
|
|
339
|
+
const cwd = process.cwd();
|
|
340
|
+
const corpusDir = join(cwd, "knowledge", "corpus");
|
|
341
|
+
const indexPath = join(cwd, "knowledge", "index.json");
|
|
342
|
+
const metaPath = join(cwd, "deepskill.json");
|
|
343
|
+
if (!existsSync(corpusDir)) {
|
|
344
|
+
console.log(pc.red(" No knowledge/corpus/ directory found."));
|
|
345
|
+
console.log(pc.dim(" Run \"npx deepskill init\" first, or create knowledge/corpus/ manually."));
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
const files = walkDir(corpusDir);
|
|
349
|
+
if (files.length === 0) {
|
|
350
|
+
console.log(pc.yellow(" No supported files in knowledge/corpus/"));
|
|
351
|
+
console.log(pc.dim(" Add .md, .csv, .txt, or .json files and try again."));
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
console.log(pc.white(" Building search index..."));
|
|
355
|
+
console.log();
|
|
356
|
+
const allChunks = [];
|
|
357
|
+
let totalTokens = 0;
|
|
358
|
+
for (const file of files) {
|
|
359
|
+
const relPath = relative(cwd, file);
|
|
360
|
+
const content = readFileSync(file, "utf-8");
|
|
361
|
+
const chunks = chunkFile(content, relative(join(cwd, "knowledge"), file));
|
|
362
|
+
const tokens = countTokens(content);
|
|
363
|
+
totalTokens += tokens;
|
|
364
|
+
allChunks.push(...chunks);
|
|
365
|
+
console.log(` ${pc.dim("+")} ${relPath} ${pc.dim(`(${chunks.length} chunks, ~${tokens} tokens)`)}`);
|
|
366
|
+
}
|
|
367
|
+
console.log();
|
|
368
|
+
if (allChunks.length === 0) {
|
|
369
|
+
console.log(pc.yellow(" No chunks produced. Check your corpus files."));
|
|
370
|
+
return;
|
|
371
|
+
}
|
|
372
|
+
const indexData = buildIndex(allChunks);
|
|
373
|
+
const indexJson = JSON.stringify(indexData);
|
|
374
|
+
writeFileSync(indexPath, indexJson);
|
|
375
|
+
const indexHash = sha256(indexJson);
|
|
376
|
+
const indexSizeMB = (Buffer.byteLength(indexJson) / (1024 * 1024)).toFixed(1);
|
|
377
|
+
let meta = {};
|
|
378
|
+
if (existsSync(metaPath)) try {
|
|
379
|
+
meta = JSON.parse(readFileSync(metaPath, "utf-8"));
|
|
380
|
+
} catch {}
|
|
381
|
+
meta.corpus = {
|
|
382
|
+
files: files.length,
|
|
383
|
+
chunks: allChunks.length,
|
|
384
|
+
tokens: totalTokens
|
|
385
|
+
};
|
|
386
|
+
meta.index_hash = indexHash;
|
|
387
|
+
meta.built_at = (/* @__PURE__ */ new Date()).toISOString();
|
|
388
|
+
writeFileSync(metaPath, JSON.stringify(meta, null, 2));
|
|
389
|
+
console.log(pc.white(" Build complete"));
|
|
390
|
+
console.log();
|
|
391
|
+
console.log(` Files: ${pc.white(String(files.length))} Chunks: ${pc.white(String(allChunks.length))} Tokens: ${pc.white(`~${totalTokens}`)} Index: ${pc.white(`${indexSizeMB}MB`)}`);
|
|
392
|
+
console.log();
|
|
393
|
+
console.log(` ${pc.dim("Test it:")} npx deepskill search "your query"`);
|
|
394
|
+
console.log();
|
|
395
|
+
}
|
|
396
|
+
//#endregion
|
|
397
|
+
//#region src/indexer/search.ts
|
|
398
|
+
function searchIndex(indexData, query, limit = 5) {
|
|
399
|
+
const db = create({ schema: SKILL_SCHEMA });
|
|
400
|
+
load(db, indexData);
|
|
401
|
+
const start = performance.now();
|
|
402
|
+
const result = search(db, {
|
|
403
|
+
term: query,
|
|
404
|
+
limit
|
|
405
|
+
});
|
|
406
|
+
const elapsed = performance.now() - start;
|
|
407
|
+
return {
|
|
408
|
+
count: result.count,
|
|
409
|
+
elapsed_ms: Math.round(elapsed * 100) / 100,
|
|
410
|
+
results: result.hits.map((hit) => {
|
|
411
|
+
const doc = hit.document;
|
|
412
|
+
const fileName = doc.source_file.replace(/^corpus\//, "");
|
|
413
|
+
const sectionSlug = doc.section.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/(^-|-$)/g, "");
|
|
414
|
+
return {
|
|
415
|
+
title: doc.title,
|
|
416
|
+
content: doc.content,
|
|
417
|
+
source_file: doc.source_file,
|
|
418
|
+
section: doc.section,
|
|
419
|
+
score: hit.score,
|
|
420
|
+
citation: sectionSlug ? `${fileName}#${sectionSlug}` : fileName
|
|
421
|
+
};
|
|
422
|
+
})
|
|
423
|
+
};
|
|
424
|
+
}
|
|
425
|
+
//#endregion
|
|
426
|
+
//#region src/commands/search.ts
|
|
427
|
+
async function runSearch(args) {
|
|
428
|
+
const query = args.join(" ").trim();
|
|
429
|
+
if (!query) {
|
|
430
|
+
console.log(pc.yellow("Usage: deepskill search <query>"));
|
|
431
|
+
console.log(pc.dim(" Example: deepskill search \"transfer limits\""));
|
|
432
|
+
return;
|
|
433
|
+
}
|
|
434
|
+
const indexPath = join(process.cwd(), "knowledge", "index.json");
|
|
435
|
+
if (!existsSync(indexPath)) {
|
|
436
|
+
console.log(pc.red("No index found at knowledge/index.json"));
|
|
437
|
+
console.log(pc.dim("Run \"npx deepskill build\" first."));
|
|
438
|
+
return;
|
|
439
|
+
}
|
|
440
|
+
let indexData;
|
|
441
|
+
try {
|
|
442
|
+
indexData = JSON.parse(readFileSync(indexPath, "utf-8"));
|
|
443
|
+
} catch {
|
|
444
|
+
console.log(pc.red("Failed to parse knowledge/index.json"));
|
|
445
|
+
return;
|
|
446
|
+
}
|
|
447
|
+
const limitArg = args.find((a) => a.startsWith("--limit="));
|
|
448
|
+
const limit = limitArg ? parseInt(limitArg.split("=")[1], 10) : 5;
|
|
449
|
+
const queryText = args.filter((a) => !a.startsWith("--")).join(" ").trim();
|
|
450
|
+
const response = searchIndex(indexData, queryText || query, limit);
|
|
451
|
+
if (response.results.length === 0) {
|
|
452
|
+
console.log();
|
|
453
|
+
console.log(pc.yellow(` No results for "${queryText || query}"`));
|
|
454
|
+
console.log();
|
|
455
|
+
return;
|
|
456
|
+
}
|
|
457
|
+
console.log();
|
|
458
|
+
console.log(pc.dim(` ${response.count} result(s) in ${response.elapsed_ms}ms`));
|
|
459
|
+
console.log();
|
|
460
|
+
for (let i = 0; i < response.results.length; i++) {
|
|
461
|
+
const r = response.results[i];
|
|
462
|
+
const num = pc.dim(` ${i + 1}.`);
|
|
463
|
+
console.log(`${num} ${pc.white(pc.bold(r.title))}`);
|
|
464
|
+
console.log(` ${pc.dim("source:")} ${r.source_file} ${pc.dim("section:")} ${r.section} ${pc.dim("score:")} ${r.score.toFixed(2)}`);
|
|
465
|
+
const snippet = r.content.replace(/\n/g, " ").slice(0, 200);
|
|
466
|
+
console.log(` ${pc.dim(snippet)}${r.content.length > 200 ? pc.dim("...") : ""}`);
|
|
467
|
+
console.log(` ${pc.dim("cite:")} ${r.citation}`);
|
|
468
|
+
console.log();
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
//#endregion
|
|
472
|
+
//#region src/commands/validate.ts
|
|
473
|
+
function countFiles(dir) {
|
|
474
|
+
if (!existsSync(dir)) return 0;
|
|
475
|
+
let count = 0;
|
|
476
|
+
const entries = readdirSync(dir);
|
|
477
|
+
for (const entry of entries) {
|
|
478
|
+
const full = join(dir, entry);
|
|
479
|
+
if (statSync(full).isDirectory()) count += countFiles(full);
|
|
480
|
+
else if (isSupportedFile(full)) count++;
|
|
481
|
+
}
|
|
482
|
+
return count;
|
|
483
|
+
}
|
|
484
|
+
async function runValidate() {
|
|
485
|
+
const cwd = process.cwd();
|
|
486
|
+
const skillPath = join(cwd, "SKILL.md");
|
|
487
|
+
const indexPath = join(cwd, "knowledge", "index.json");
|
|
488
|
+
const corpusDir = join(cwd, "knowledge", "corpus");
|
|
489
|
+
const metaPath = join(cwd, "deepskill.json");
|
|
490
|
+
let errors = 0;
|
|
491
|
+
let warnings = 0;
|
|
492
|
+
function pass(msg) {
|
|
493
|
+
console.log(` ${pc.green("✓")} ${msg}`);
|
|
494
|
+
}
|
|
495
|
+
function fail(msg) {
|
|
496
|
+
console.log(` ${pc.red("✗")} ${msg}`);
|
|
497
|
+
errors++;
|
|
498
|
+
}
|
|
499
|
+
function warn(msg) {
|
|
500
|
+
console.log(` ${pc.yellow("!")} ${msg}`);
|
|
501
|
+
warnings++;
|
|
502
|
+
}
|
|
503
|
+
if (!existsSync(skillPath)) {
|
|
504
|
+
fail("SKILL.md not found");
|
|
505
|
+
console.log();
|
|
506
|
+
console.log(pc.dim(" Run \"npx deepskill init\" to create one."));
|
|
507
|
+
console.log();
|
|
508
|
+
return;
|
|
509
|
+
}
|
|
510
|
+
pass("SKILL.md exists");
|
|
511
|
+
const skillContent = readFileSync(skillPath, "utf-8");
|
|
512
|
+
let frontmatter;
|
|
513
|
+
try {
|
|
514
|
+
frontmatter = matter(skillContent).data;
|
|
515
|
+
pass("Frontmatter is valid YAML");
|
|
516
|
+
} catch (e) {
|
|
517
|
+
fail(`Frontmatter parse error: ${e instanceof Error ? e.message : "unknown"}`);
|
|
518
|
+
console.log();
|
|
519
|
+
return;
|
|
520
|
+
}
|
|
521
|
+
if (frontmatter.name && typeof frontmatter.name === "string") pass(`Name: ${frontmatter.name}`);
|
|
522
|
+
else fail("Missing \"name\" in frontmatter");
|
|
523
|
+
if (frontmatter.description && typeof frontmatter.description === "string") {
|
|
524
|
+
const desc = frontmatter.description;
|
|
525
|
+
if (desc.length < 10) warn(`Description is very short (${desc.length} chars)`);
|
|
526
|
+
else pass(`Description: ${desc.slice(0, 80)}${desc.length > 80 ? "..." : ""}`);
|
|
527
|
+
} else fail("Missing \"description\" in frontmatter");
|
|
528
|
+
if (frontmatter.knowledge === true) pass("knowledge: true");
|
|
529
|
+
else warn("knowledge: true not set in frontmatter (this is a thin skill)");
|
|
530
|
+
if (existsSync(corpusDir)) {
|
|
531
|
+
const fileCount = countFiles(corpusDir);
|
|
532
|
+
if (fileCount > 0) pass(`Corpus: ${fileCount} file(s) in knowledge/corpus/`);
|
|
533
|
+
else warn("knowledge/corpus/ exists but has no supported files");
|
|
534
|
+
} else if (frontmatter.knowledge === true) fail("knowledge/corpus/ not found (required when knowledge: true)");
|
|
535
|
+
else pass(pc.dim("No knowledge/corpus/ (thin skill)"));
|
|
536
|
+
if (existsSync(indexPath)) try {
|
|
537
|
+
const indexData = JSON.parse(readFileSync(indexPath, "utf-8"));
|
|
538
|
+
load(create({ schema: SKILL_SCHEMA }), indexData);
|
|
539
|
+
pass("knowledge/index.json loads successfully");
|
|
540
|
+
} catch (e) {
|
|
541
|
+
fail(`knowledge/index.json failed to load: ${e instanceof Error ? e.message : "unknown"}`);
|
|
542
|
+
}
|
|
543
|
+
else if (frontmatter.knowledge === true) warn("knowledge/index.json not found — run \"npx deepskill build\"");
|
|
544
|
+
if (existsSync(metaPath)) try {
|
|
545
|
+
const corpus = JSON.parse(readFileSync(metaPath, "utf-8")).corpus;
|
|
546
|
+
if (corpus) pass(`deepskill.json: ${corpus.files ?? 0} files, ${corpus.chunks ?? 0} chunks, ~${corpus.tokens ?? 0} tokens`);
|
|
547
|
+
else pass("deepskill.json exists");
|
|
548
|
+
} catch {
|
|
549
|
+
warn("deepskill.json exists but is not valid JSON");
|
|
550
|
+
}
|
|
551
|
+
else warn("deepskill.json not found");
|
|
552
|
+
console.log();
|
|
553
|
+
if (errors === 0 && warnings === 0) console.log(pc.green(" All checks passed"));
|
|
554
|
+
else if (errors === 0) console.log(pc.yellow(` Passed with ${warnings} warning(s)`));
|
|
555
|
+
else console.log(pc.red(` ${errors} error(s), ${warnings} warning(s)`));
|
|
556
|
+
console.log();
|
|
557
|
+
}
|
|
558
|
+
//#endregion
|
|
559
|
+
//#region src/cli.ts
|
|
560
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
561
|
+
function getVersion() {
|
|
562
|
+
try {
|
|
563
|
+
const pkgPath = join(__dirname, "..", "package.json");
|
|
564
|
+
return JSON.parse(readFileSync(pkgPath, "utf-8")).version;
|
|
565
|
+
} catch {
|
|
566
|
+
return "0.1.0";
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
const VERSION = getVersion();
|
|
570
|
+
const LOGO_LINES = [
|
|
571
|
+
"██████╗ ███████╗███████╗██████╗ ███████╗██╗ ██╗██╗██╗ ██╗ ",
|
|
572
|
+
"██╔══██╗██╔════╝██╔════╝██╔══██╗██╔════╝██║ ██╔╝██║██║ ██║ ",
|
|
573
|
+
"██║ ██║█████╗ █████╗ ██████╔╝███████╗█████╔╝ ██║██║ ██║ ",
|
|
574
|
+
"██║ ██║██╔══╝ ██╔══╝ ██╔═══╝ ╚════██║██╔═██╗ ██║██║ ██║ ",
|
|
575
|
+
"██████╔╝███████╗███████╗██║ ███████║██║ ██╗██║███████╗███████╗",
|
|
576
|
+
"╚═════╝ ╚══════╝╚══════╝╚═╝ ╚══════╝╚═╝ ╚═╝╚═╝╚══════╝╚══════╝"
|
|
577
|
+
];
|
|
578
|
+
const GRAYS = [
|
|
579
|
+
"\x1B[38;5;250m",
|
|
580
|
+
"\x1B[38;5;248m",
|
|
581
|
+
"\x1B[38;5;245m",
|
|
582
|
+
"\x1B[38;5;243m",
|
|
583
|
+
"\x1B[38;5;240m",
|
|
584
|
+
"\x1B[38;5;238m"
|
|
585
|
+
];
|
|
586
|
+
const RESET = "\x1B[0m";
|
|
587
|
+
function showLogo() {
|
|
588
|
+
console.log();
|
|
589
|
+
LOGO_LINES.forEach((line, i) => {
|
|
590
|
+
console.log(`${GRAYS[i]}${line}${RESET}`);
|
|
591
|
+
});
|
|
592
|
+
}
|
|
593
|
+
function showBanner() {
|
|
594
|
+
showLogo();
|
|
595
|
+
console.log();
|
|
596
|
+
console.log(pc.dim(" Agent skills with searchable knowledge"));
|
|
597
|
+
console.log();
|
|
598
|
+
console.log(` ${pc.dim("$")} ${pc.white("npx deepskill init")} ${pc.dim("<name>")} ${pc.dim("Create a new deep skill")}`);
|
|
599
|
+
console.log(` ${pc.dim("$")} ${pc.white("npx deepskill build")} ${pc.dim("Build search index from corpus")}`);
|
|
600
|
+
console.log(` ${pc.dim("$")} ${pc.white("npx deepskill search")} ${pc.dim("<query>")} ${pc.dim("Search the knowledge base")}`);
|
|
601
|
+
console.log(` ${pc.dim("$")} ${pc.white("npx deepskill validate")} ${pc.dim("Validate skill structure")}`);
|
|
602
|
+
console.log();
|
|
603
|
+
console.log(` ${pc.dim("try:")} npx deepskill init my-skill`);
|
|
604
|
+
console.log();
|
|
605
|
+
console.log(` ${pc.dim(`v${VERSION}`)} ${pc.white("https://deepskill.sh")}`);
|
|
606
|
+
console.log();
|
|
607
|
+
}
|
|
608
|
+
function showHelp() {
|
|
609
|
+
console.log(`
|
|
610
|
+
${pc.bold("Usage:")} deepskill <command> [options]
|
|
611
|
+
|
|
612
|
+
${pc.bold("Commands:")}
|
|
613
|
+
init <name> Create a new deep skill with knowledge directory
|
|
614
|
+
build Chunk corpus and build Orama search index
|
|
615
|
+
search <query> Query the local knowledge index
|
|
616
|
+
validate Check skill structure, frontmatter, and index
|
|
617
|
+
|
|
618
|
+
${pc.bold("Options:")}
|
|
619
|
+
--help, -h Show this help message
|
|
620
|
+
--version, -v Show version number
|
|
621
|
+
|
|
622
|
+
${pc.bold("Examples:")}
|
|
623
|
+
${pc.dim("$")} deepskill init compliance-checker
|
|
624
|
+
${pc.dim("$")} deepskill build
|
|
625
|
+
${pc.dim("$")} deepskill search "transfer limits"
|
|
626
|
+
${pc.dim("$")} deepskill validate
|
|
627
|
+
|
|
628
|
+
${pc.dim(`v${VERSION}`)} ${pc.white("https://deepskill.sh")}
|
|
629
|
+
`);
|
|
630
|
+
}
|
|
631
|
+
async function main() {
|
|
632
|
+
const args = process.argv.slice(2);
|
|
633
|
+
if (args.length === 0) {
|
|
634
|
+
showBanner();
|
|
635
|
+
return;
|
|
636
|
+
}
|
|
637
|
+
const command = args[0];
|
|
638
|
+
const restArgs = args.slice(1);
|
|
639
|
+
switch (command) {
|
|
640
|
+
case "init":
|
|
641
|
+
showLogo();
|
|
642
|
+
console.log();
|
|
643
|
+
runInit(restArgs);
|
|
644
|
+
break;
|
|
645
|
+
case "build":
|
|
646
|
+
showLogo();
|
|
647
|
+
console.log();
|
|
648
|
+
await runBuild();
|
|
649
|
+
break;
|
|
650
|
+
case "search":
|
|
651
|
+
case "s":
|
|
652
|
+
await runSearch(restArgs);
|
|
653
|
+
break;
|
|
654
|
+
case "validate":
|
|
655
|
+
case "check":
|
|
656
|
+
showLogo();
|
|
657
|
+
console.log();
|
|
658
|
+
await runValidate();
|
|
659
|
+
break;
|
|
660
|
+
case "--help":
|
|
661
|
+
case "-h":
|
|
662
|
+
showHelp();
|
|
663
|
+
break;
|
|
664
|
+
case "--version":
|
|
665
|
+
case "-v":
|
|
666
|
+
console.log(VERSION);
|
|
667
|
+
break;
|
|
668
|
+
default:
|
|
669
|
+
console.log(`Unknown command: ${command}`);
|
|
670
|
+
console.log(`Run ${pc.bold("deepskill --help")} for usage.`);
|
|
671
|
+
process.exit(1);
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
main();
|
|
675
|
+
//#endregion
|
|
676
|
+
export {};
|
package/package.json
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "deepskill",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Agent skills that carry their own searchable knowledge",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"bin": {
|
|
7
|
+
"deepskill": "./bin/cli.mjs"
|
|
8
|
+
},
|
|
9
|
+
"files": [
|
|
10
|
+
"dist",
|
|
11
|
+
"bin",
|
|
12
|
+
"README.md"
|
|
13
|
+
],
|
|
14
|
+
"scripts": {
|
|
15
|
+
"build": "obuild",
|
|
16
|
+
"dev": "node --import tsx src/cli.ts",
|
|
17
|
+
"prepublishOnly": "npm run build"
|
|
18
|
+
},
|
|
19
|
+
"keywords": [
|
|
20
|
+
"cli",
|
|
21
|
+
"agent-skills",
|
|
22
|
+
"skills",
|
|
23
|
+
"ai-agents",
|
|
24
|
+
"knowledge-base",
|
|
25
|
+
"search",
|
|
26
|
+
"orama",
|
|
27
|
+
"deepskill"
|
|
28
|
+
],
|
|
29
|
+
"repository": {
|
|
30
|
+
"type": "git",
|
|
31
|
+
"url": "git+https://github.com/deepskill/dskill.git"
|
|
32
|
+
},
|
|
33
|
+
"homepage": "https://deepskill.sh",
|
|
34
|
+
"license": "MIT",
|
|
35
|
+
"devDependencies": {
|
|
36
|
+
"@types/node": "^22.10.0",
|
|
37
|
+
"obuild": "^0.4.22",
|
|
38
|
+
"tsx": "^4.19.0",
|
|
39
|
+
"typescript": "^5.9.3"
|
|
40
|
+
},
|
|
41
|
+
"dependencies": {
|
|
42
|
+
"@orama/orama": "^3.1.18",
|
|
43
|
+
"gray-matter": "^4.0.3",
|
|
44
|
+
"picocolors": "^1.1.1"
|
|
45
|
+
},
|
|
46
|
+
"engines": {
|
|
47
|
+
"node": ">=18"
|
|
48
|
+
},
|
|
49
|
+
"packageManager": "pnpm@10.17.1"
|
|
50
|
+
}
|