librebot-tools 0.0.0-stage → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +277 -2
- package/bin/librebot.js +262 -0
- package/package.json +38 -4
- package/schema/v1/bot-yaml-schema.json +193 -0
- package/schema/v1/skills-sources-schema.json +31 -0
- package/src/contract.d.ts +25 -0
- package/src/contract.js +23 -0
- package/src/errors.js +28 -0
- package/src/icon.js +130 -0
- package/src/index.d.ts +271 -0
- package/src/index.js +57 -0
- package/src/parse.js +702 -0
- package/src/scaffold.js +279 -0
- package/src/schema.js +14 -0
- package/src/skills-import.js +50 -0
- package/src/skills-lock.js +697 -0
- package/src/util.js +27 -0
- package/src/validate.js +835 -0
- package/test/assets.test.mjs +82 -0
- package/test/export.test.mjs +155 -0
- package/test/gitlab-preview.test.mjs +23 -0
- package/test/gitlab-triggers.test.mjs +53 -0
- package/test/icon.test.mjs +137 -0
- package/test/lock.test.mjs +92 -0
- package/test/malformed.test.mjs +79 -0
- package/test/readme.test.mjs +109 -0
- package/test/semantics.test.mjs +186 -0
- package/test/standard-skills.test.mjs +41 -0
- package/test/vendor.test.mjs +135 -0
- package/test/whatsapp-actions.test.mjs +31 -0
package/src/parse.js
ADDED
|
@@ -0,0 +1,702 @@
|
|
|
1
|
+
// Stateless bot-source parser. Operates on an in-memory file map
|
|
2
|
+
// (Record<posix-rel-path, Uint8Array|string>) so Forgejo trees, zips, and
|
|
3
|
+
// local directories all parse identically with zero filesystem access.
|
|
4
|
+
// Fully offline: external skills must be vendored under skills/.vendor/.
|
|
5
|
+
|
|
6
|
+
import { readdir, readFile } from "node:fs/promises";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
import { parse as parseYaml } from "yaml";
|
|
9
|
+
import { BotSourceError, isRecord } from "./errors.js";
|
|
10
|
+
import { checkSourcesLock, parseSourcesYaml } from "./skills-lock.js";
|
|
11
|
+
import { sha256Hex, toBytes } from "./util.js";
|
|
12
|
+
import { vendorContentHash } from "./skills-import.js";
|
|
13
|
+
import { iconPath, validateIcon } from "./icon.js";
|
|
14
|
+
import { FORMAT_VERSION } from "./contract.js";
|
|
15
|
+
import {
|
|
16
|
+
entryId,
|
|
17
|
+
requiredString,
|
|
18
|
+
validateConnectors,
|
|
19
|
+
validateActions,
|
|
20
|
+
validateKnowledgeSources,
|
|
21
|
+
validateModelRoles,
|
|
22
|
+
validateTriggerDefinitions,
|
|
23
|
+
validateWorkspaceMounts,
|
|
24
|
+
} from "./validate.js";
|
|
25
|
+
|
|
26
|
+
const FILE_BOT_YAML = "bot.yaml";
|
|
27
|
+
const FILE_INSTRUCTIONS = "instructions.md";
|
|
28
|
+
const FILE_README = "README.md";
|
|
29
|
+
const DIR_KNOWLEDGE = "knowledge/";
|
|
30
|
+
const DIR_SKILLS = "skills/";
|
|
31
|
+
const DIR_VENDOR = "skills/.vendor/";
|
|
32
|
+
const FILE_SOURCES = "skills/sources.yaml";
|
|
33
|
+
const FILE_LOCK = "skills/sources.lock";
|
|
34
|
+
const KNOWLEDGE_INDEX = "knowledge/README.md";
|
|
35
|
+
|
|
36
|
+
const TOP_LEVEL_KEYS = [
|
|
37
|
+
"formatVersion",
|
|
38
|
+
"name",
|
|
39
|
+
"description",
|
|
40
|
+
"icon",
|
|
41
|
+
"modelRoles",
|
|
42
|
+
"workspace",
|
|
43
|
+
"triggers",
|
|
44
|
+
"actions",
|
|
45
|
+
"connectors",
|
|
46
|
+
"knowledgeSources",
|
|
47
|
+
];
|
|
48
|
+
|
|
49
|
+
export const MAX_TEXT_KNOWLEDGE_BYTES = 100_000;
|
|
50
|
+
export const MAX_SKILL_BYTES = 50_000;
|
|
51
|
+
// Binary byte limits match platform validation.
|
|
52
|
+
export const MAX_ASSET_BYTES = 10 * 1024 * 1024;
|
|
53
|
+
export const MAX_DRAFT_BYTES = 750_000;
|
|
54
|
+
export const MAX_FILES = 5000;
|
|
55
|
+
export const MAX_SKILL_FILES = 100;
|
|
56
|
+
export const MAX_SKILL_FILE_BYTES = 1024 * 1024;
|
|
57
|
+
|
|
58
|
+
const MIME_BY_EXT = {
|
|
59
|
+
".png": "image/png",
|
|
60
|
+
".jpg": "image/jpeg",
|
|
61
|
+
".jpeg": "image/jpeg",
|
|
62
|
+
".gif": "image/gif",
|
|
63
|
+
".webp": "image/webp",
|
|
64
|
+
".svg": "image/svg+xml",
|
|
65
|
+
".pdf": "application/pdf",
|
|
66
|
+
".docx":
|
|
67
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
68
|
+
".xlsx":
|
|
69
|
+
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
70
|
+
".md": "text/markdown",
|
|
71
|
+
".txt": "text/plain",
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Normalize + validate a caller-supplied file map. Keys must be posix-style
|
|
77
|
+
* relative paths (no leading slash, no backslash, no `..` segments).
|
|
78
|
+
* @param {Record<string, Uint8Array|string>} files
|
|
79
|
+
* @returns {Map<string, Uint8Array>} sorted by path (deterministic)
|
|
80
|
+
*/
|
|
81
|
+
export function normalizeFileMap(files) {
|
|
82
|
+
if (!isRecord(files)) throw new Error("files must be an object");
|
|
83
|
+
const out = new Map();
|
|
84
|
+
for (const [rawPath, value] of Object.entries(files)) {
|
|
85
|
+
if (typeof rawPath !== "string" || !rawPath || rawPath.startsWith("/")) {
|
|
86
|
+
throw new Error(`invalid file path: ${rawPath}`);
|
|
87
|
+
}
|
|
88
|
+
const path = rawPath.replace(/\\/g, "/");
|
|
89
|
+
const segments = path.split("/");
|
|
90
|
+
if (segments.some((s) => s === "" || s === "." || s === "..")) {
|
|
91
|
+
throw new Error(`invalid file path: ${rawPath}`);
|
|
92
|
+
}
|
|
93
|
+
if (out.has(path)) throw new Error(`duplicate file path: ${rawPath}`);
|
|
94
|
+
out.set(path, toBytes(value));
|
|
95
|
+
}
|
|
96
|
+
if (out.size > MAX_FILES) throw new Error(`too many files (max ${MAX_FILES})`);
|
|
97
|
+
return new Map([...out.entries()].sort(([a], [b]) => (a < b ? -1 : 1)));
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Paths the parser ignores (dotfiles), except inside the vendor tree. */
|
|
101
|
+
function isSkipped(path) {
|
|
102
|
+
if (path.startsWith(DIR_VENDOR)) return false;
|
|
103
|
+
return path.split("/").some((segment) => segment.startsWith("."));
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Strict UTF-8 text, or null when the bytes are binary. Length caps apply at the call site in chars, matching the platform. */
|
|
107
|
+
function asInlineText(bytes) {
|
|
108
|
+
try {
|
|
109
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
110
|
+
if (text.includes("\0")) return null;
|
|
111
|
+
return text;
|
|
112
|
+
} catch {
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function mimeFor(path) {
|
|
118
|
+
const dot = path.lastIndexOf(".");
|
|
119
|
+
const ext = dot >= 0 ? path.slice(dot).toLowerCase() : "";
|
|
120
|
+
return MIME_BY_EXT[ext] ?? "application/octet-stream";
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function baseName(path) {
|
|
124
|
+
return path.slice(path.lastIndexOf("/") + 1);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Deterministic knowledge id derived from the path under knowledge/. */
|
|
128
|
+
export function knowledgeIdFor(rel) {
|
|
129
|
+
let id = rel.replace(/\//g, "__").replace(/[^A-Za-z0-9._-]/g, "_");
|
|
130
|
+
if (!/^[A-Za-z0-9]/.test(id)) id = "k_" + id;
|
|
131
|
+
return id.slice(0, 64);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Parse SKILL.md frontmatter (`name:`/`description:` between `---` fences).
|
|
136
|
+
* @returns {{ name?: string; description?: string; body: string }}
|
|
137
|
+
*/
|
|
138
|
+
export function parseSkillFrontmatter(text) {
|
|
139
|
+
if (!text.startsWith("---\n") && !text.startsWith("---\r\n")) {
|
|
140
|
+
return { body: text };
|
|
141
|
+
}
|
|
142
|
+
const end = text.indexOf("\n---", 3);
|
|
143
|
+
if (end < 0) return { body: text };
|
|
144
|
+
const head = text.slice(3, end);
|
|
145
|
+
const body = text.slice(end + 4).replace(/^\r?\n/, "");
|
|
146
|
+
/** @type {{ name?: string; description?: string; body: string }} */
|
|
147
|
+
const out = { body };
|
|
148
|
+
for (const line of head.split("\n")) {
|
|
149
|
+
const match = /^([A-Za-z0-9_-]+):\s*(.*)$/.exec(line.trim());
|
|
150
|
+
if (!match) continue;
|
|
151
|
+
const [, key, raw] = match;
|
|
152
|
+
const value = raw.trim().replace(/^["']|["']$/g, "");
|
|
153
|
+
if ((key === "name" || key === "description") && value) out[key] = value;
|
|
154
|
+
}
|
|
155
|
+
return out;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** First `# heading` in markdown, for a skill display name fallback. */
|
|
159
|
+
function firstHeading(text) {
|
|
160
|
+
for (const line of text.split("\n")) {
|
|
161
|
+
const match = /^#{1,3}\s+(.+)$/.exec(line.trim());
|
|
162
|
+
if (match) return match[1].trim().slice(0, 200);
|
|
163
|
+
}
|
|
164
|
+
return undefined;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* @param {Map<string, Uint8Array>} files
|
|
169
|
+
* @param {{ errors: Array<{file:string;field:string;message:string}>; warnings: Array<{file:string;field:string;message:string}> }} sink
|
|
170
|
+
*/
|
|
171
|
+
function readYamlFile(files, path, sink) {
|
|
172
|
+
const bytes = files.get(path);
|
|
173
|
+
if (bytes === undefined) return undefined;
|
|
174
|
+
let doc;
|
|
175
|
+
try {
|
|
176
|
+
doc = parseYaml(new TextDecoder().decode(bytes));
|
|
177
|
+
} catch (error) {
|
|
178
|
+
sink.errors.push({
|
|
179
|
+
file: path,
|
|
180
|
+
field: "",
|
|
181
|
+
message: `is not valid YAML: ${error.message}`,
|
|
182
|
+
});
|
|
183
|
+
return undefined;
|
|
184
|
+
}
|
|
185
|
+
return doc;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Parse + semantically validate a bot source tree.
|
|
190
|
+
* @param {Record<string, Uint8Array|string>} input
|
|
191
|
+
* @returns {Promise<{ formatVersion: number; draft: Record<string, unknown>; assets: Array<{ knowledgeId: string; path: string; bytes: Uint8Array; sha256: string; size: number; mime: string }>; skillFiles: Array<{ skillId: string; path: string; bytes: Uint8Array; sha256: string; size: number }>; warnings: Array<{file:string;field:string;message:string}>; source: Record<string, unknown>; lock: Record<string, unknown> | null }>}
|
|
192
|
+
*/
|
|
193
|
+
export async function parseBotFiles(input) {
|
|
194
|
+
const result = await validateFiles(input);
|
|
195
|
+
if (!result.ok) {
|
|
196
|
+
throw new BotSourceError(result.errors, result.warnings);
|
|
197
|
+
}
|
|
198
|
+
return {
|
|
199
|
+
formatVersion: result.formatVersion,
|
|
200
|
+
draft: result.draft,
|
|
201
|
+
assets: result.assets,
|
|
202
|
+
icon: result.icon,
|
|
203
|
+
skillFiles: result.skillFiles,
|
|
204
|
+
warnings: result.warnings,
|
|
205
|
+
source: result.source,
|
|
206
|
+
lock: result.lock,
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** Non-throwing variant of parseBotFiles for CLI/server ingestion. */
|
|
211
|
+
export async function validateFiles(input) {
|
|
212
|
+
/** @type {Array<{file:string;field:string;message:string}>} */
|
|
213
|
+
const errors = [];
|
|
214
|
+
/** @type {Array<{file:string;field:string;message:string}>} */
|
|
215
|
+
const warnings = [];
|
|
216
|
+
const sink = { errors, warnings };
|
|
217
|
+
/** @type {Map<string, Uint8Array>} */
|
|
218
|
+
let files;
|
|
219
|
+
try {
|
|
220
|
+
files = normalizeFileMap(input);
|
|
221
|
+
} catch (error) {
|
|
222
|
+
return { ok: false, errors: [{ file: "", field: "", message: error.message }], warnings };
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const botDoc = readYamlFile(files, FILE_BOT_YAML, sink);
|
|
226
|
+
if (!files.has(FILE_BOT_YAML)) {
|
|
227
|
+
errors.push({ file: FILE_BOT_YAML, field: "", message: "is required" });
|
|
228
|
+
}
|
|
229
|
+
const instructionsBytes = files.get(FILE_INSTRUCTIONS);
|
|
230
|
+
if (instructionsBytes === undefined) {
|
|
231
|
+
errors.push({ file: FILE_INSTRUCTIONS, field: "", message: "is required" });
|
|
232
|
+
}
|
|
233
|
+
const readmeBytes = files.get(FILE_README);
|
|
234
|
+
if (readmeBytes === undefined) {
|
|
235
|
+
errors.push({ file: FILE_README, field: "", message: "is required" });
|
|
236
|
+
}
|
|
237
|
+
if (botDoc === undefined || instructionsBytes === undefined) {
|
|
238
|
+
return { ok: false, errors, warnings };
|
|
239
|
+
}
|
|
240
|
+
if (!isRecord(botDoc)) {
|
|
241
|
+
errors.push({ file: FILE_BOT_YAML, field: "", message: "must be a YAML mapping" });
|
|
242
|
+
return { ok: false, errors, warnings };
|
|
243
|
+
}
|
|
244
|
+
for (const key of Object.keys(botDoc)) {
|
|
245
|
+
if (!TOP_LEVEL_KEYS.includes(key)) {
|
|
246
|
+
errors.push({
|
|
247
|
+
file: FILE_BOT_YAML,
|
|
248
|
+
field: key,
|
|
249
|
+
message:
|
|
250
|
+
key === "tools"
|
|
251
|
+
? "was removed: builtin tools are always on and must not be declared"
|
|
252
|
+
: `contains unsupported field ${key}`,
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
if (botDoc.formatVersion !== FORMAT_VERSION) {
|
|
257
|
+
errors.push({
|
|
258
|
+
file: FILE_BOT_YAML,
|
|
259
|
+
field: "formatVersion",
|
|
260
|
+
message: `must be ${FORMAT_VERSION}`,
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
const name = requiredString(botDoc.name, "name", 100, sink, FILE_BOT_YAML);
|
|
265
|
+
const description = requiredString(
|
|
266
|
+
botDoc.description,
|
|
267
|
+
"description",
|
|
268
|
+
1000,
|
|
269
|
+
sink,
|
|
270
|
+
FILE_BOT_YAML,
|
|
271
|
+
);
|
|
272
|
+
let icon;
|
|
273
|
+
if (botDoc.icon !== undefined) {
|
|
274
|
+
try {
|
|
275
|
+
const path = iconPath(botDoc.icon);
|
|
276
|
+
icon = validateIcon(path, files.get(path));
|
|
277
|
+
} catch (error) {
|
|
278
|
+
errors.push({ file: FILE_BOT_YAML, field: "icon", message: error.message });
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
let botPrompt;
|
|
282
|
+
try {
|
|
283
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(instructionsBytes);
|
|
284
|
+
if (!text.trim()) {
|
|
285
|
+
errors.push({ file: FILE_INSTRUCTIONS, field: "", message: "must not be empty" });
|
|
286
|
+
} else if (text.trim().length > 50000) {
|
|
287
|
+
errors.push({ file: FILE_INSTRUCTIONS, field: "", message: "is too long (max 50000 chars)" });
|
|
288
|
+
} else {
|
|
289
|
+
botPrompt = text.trim();
|
|
290
|
+
}
|
|
291
|
+
} catch {
|
|
292
|
+
errors.push({ file: FILE_INSTRUCTIONS, field: "", message: "must be UTF-8 text" });
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
let readme;
|
|
296
|
+
if (readmeBytes !== undefined) {
|
|
297
|
+
try {
|
|
298
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(readmeBytes).trim();
|
|
299
|
+
if (!text) {
|
|
300
|
+
errors.push({ file: FILE_README, field: "", message: "must not be empty" });
|
|
301
|
+
} else if (text.length > 100000) {
|
|
302
|
+
errors.push({ file: FILE_README, field: "", message: "is too long (max 100000 chars)" });
|
|
303
|
+
} else {
|
|
304
|
+
readme = text;
|
|
305
|
+
}
|
|
306
|
+
} catch {
|
|
307
|
+
errors.push({ file: FILE_README, field: "", message: "must be UTF-8 text" });
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
const modelRoles = validateModelRoles(botDoc.modelRoles, sink, FILE_BOT_YAML);
|
|
312
|
+
// Builtins are always on, so the default role always requires tool support.
|
|
313
|
+
for (const role of modelRoles) {
|
|
314
|
+
if (role.role === "default") role.requirements.tools = true;
|
|
315
|
+
}
|
|
316
|
+
const workspace = validateWorkspaceMounts(botDoc.workspace, sink, FILE_BOT_YAML);
|
|
317
|
+
const triggers = validateTriggerDefinitions(botDoc.triggers, sink, FILE_BOT_YAML);
|
|
318
|
+
const actions = validateActions(botDoc.actions, sink, FILE_BOT_YAML);
|
|
319
|
+
const connectors = validateConnectors(botDoc.connectors, sink, FILE_BOT_YAML);
|
|
320
|
+
|
|
321
|
+
for (const trigger of triggers) {
|
|
322
|
+
if (
|
|
323
|
+
trigger.deciderRole !== undefined &&
|
|
324
|
+
!modelRoles.some((role) => role.role === trigger.deciderRole)
|
|
325
|
+
) {
|
|
326
|
+
errors.push({
|
|
327
|
+
file: FILE_BOT_YAML,
|
|
328
|
+
field: `triggers.${trigger.key}.deciderRole`,
|
|
329
|
+
message: "must be one of the bot's declared model roles",
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
// --- knowledge ---
|
|
335
|
+
const knowledge = [];
|
|
336
|
+
const assets = [];
|
|
337
|
+
const knowledgeFiles = [];
|
|
338
|
+
const knowledgeIds = {};
|
|
339
|
+
for (const [path, bytes] of files) {
|
|
340
|
+
if (path === icon?.path) continue;
|
|
341
|
+
if (!path.startsWith(DIR_KNOWLEDGE) || path === KNOWLEDGE_INDEX) continue;
|
|
342
|
+
if (isSkipped(path)) continue;
|
|
343
|
+
const rel = path.slice(DIR_KNOWLEDGE.length);
|
|
344
|
+
knowledgeFiles.push(rel);
|
|
345
|
+
const id = knowledgeIdFor(rel);
|
|
346
|
+
if (knowledgeIds[id]) {
|
|
347
|
+
errors.push({ file: path, field: "", message: `collides with another knowledge id ${id}` });
|
|
348
|
+
continue;
|
|
349
|
+
}
|
|
350
|
+
knowledgeIds[id] = true;
|
|
351
|
+
if (rel.length > 200) {
|
|
352
|
+
errors.push({ file: path, field: "", message: "path is too long (max 200 chars)" });
|
|
353
|
+
continue;
|
|
354
|
+
}
|
|
355
|
+
const text = asInlineText(bytes);
|
|
356
|
+
if (text !== null && text.length <= MAX_TEXT_KNOWLEDGE_BYTES) {
|
|
357
|
+
knowledge.push({ id, name: baseName(rel), kind: "file", content: text });
|
|
358
|
+
} else if (bytes.length > MAX_ASSET_BYTES) {
|
|
359
|
+
errors.push({
|
|
360
|
+
file: path,
|
|
361
|
+
field: "",
|
|
362
|
+
message: `exceeds the ${MAX_ASSET_BYTES} byte per-file limit`,
|
|
363
|
+
});
|
|
364
|
+
} else {
|
|
365
|
+
knowledge.push({ id, name: baseName(rel), kind: "file" });
|
|
366
|
+
assets.push({
|
|
367
|
+
knowledgeId: id,
|
|
368
|
+
path: rel,
|
|
369
|
+
bytes,
|
|
370
|
+
sha256: sha256Hex(bytes),
|
|
371
|
+
size: bytes.length,
|
|
372
|
+
mime: mimeFor(rel),
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
// Structured external references (resource/repo kinds). README links stay
|
|
377
|
+
// documentation-only; only these entries become draft knowledge.
|
|
378
|
+
const knowledgeSources = validateKnowledgeSources(botDoc.knowledgeSources, sink, FILE_BOT_YAML);
|
|
379
|
+
for (const item of knowledgeSources) {
|
|
380
|
+
if (knowledgeIds[item.id]) {
|
|
381
|
+
errors.push({ file: FILE_BOT_YAML, field: `knowledgeSources.${item.id}`, message: `contains duplicate knowledge id ${item.id}` });
|
|
382
|
+
continue;
|
|
383
|
+
}
|
|
384
|
+
knowledgeIds[item.id] = true;
|
|
385
|
+
knowledge.push(item);
|
|
386
|
+
}
|
|
387
|
+
if (knowledge.length > 100) {
|
|
388
|
+
errors.push({ file: DIR_KNOWLEDGE, field: "", message: "has too many entries (max 100)" });
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// --- skills (local + vendored externals) ---
|
|
392
|
+
const skills = [];
|
|
393
|
+
const skillFiles = [];
|
|
394
|
+
const skillIds = {};
|
|
395
|
+
const skillDirs = [];
|
|
396
|
+
/**
|
|
397
|
+
* @param {string} skillId
|
|
398
|
+
* @param {string} skillRoot prefix within files, e.g. skills/demo/
|
|
399
|
+
* @param {string} blamePath file blamed for validation errors
|
|
400
|
+
*/
|
|
401
|
+
const ingestSkillDir = (skillId, skillRoot, blamePath) => {
|
|
402
|
+
if (skillIds[skillId]) {
|
|
403
|
+
errors.push({ file: blamePath, field: "", message: `contains duplicate skill id ${skillId}` });
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
406
|
+
skillIds[skillId] = true;
|
|
407
|
+
const skillDocPath = skillRoot + "SKILL.md";
|
|
408
|
+
const skillBytes = files.get(skillDocPath);
|
|
409
|
+
if (skillBytes === undefined) {
|
|
410
|
+
errors.push({ file: blamePath, field: "", message: "has no SKILL.md" });
|
|
411
|
+
return;
|
|
412
|
+
}
|
|
413
|
+
let skillText;
|
|
414
|
+
try {
|
|
415
|
+
skillText = new TextDecoder("utf-8", { fatal: true }).decode(skillBytes);
|
|
416
|
+
} catch {
|
|
417
|
+
errors.push({ file: skillDocPath, field: "", message: "must be UTF-8 text" });
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
// Platform requires non-empty skill content: reject here so an empty
|
|
421
|
+
// SKILL.md can never pass local validation yet fail ingest.
|
|
422
|
+
if (!skillText.trim()) {
|
|
423
|
+
errors.push({ file: skillDocPath, field: "", message: "must not be empty" });
|
|
424
|
+
return;
|
|
425
|
+
}
|
|
426
|
+
if (new TextEncoder().encode(skillText).length > MAX_SKILL_BYTES) {
|
|
427
|
+
errors.push({
|
|
428
|
+
file: skillDocPath,
|
|
429
|
+
field: "",
|
|
430
|
+
message: `exceeds the ${MAX_SKILL_BYTES} byte skill limit`,
|
|
431
|
+
});
|
|
432
|
+
return;
|
|
433
|
+
}
|
|
434
|
+
const frontmatter = parseSkillFrontmatter(skillText);
|
|
435
|
+
const skillName =
|
|
436
|
+
(typeof frontmatter.name === "string" && frontmatter.name.trim()) ||
|
|
437
|
+
firstHeading(skillText) ||
|
|
438
|
+
skillId;
|
|
439
|
+
const description =
|
|
440
|
+
(typeof frontmatter.description === "string" && frontmatter.description.trim()) || "";
|
|
441
|
+
if (skillName.length > 200) {
|
|
442
|
+
errors.push({ file: skillDocPath, field: "name", message: "is too long (max 200 chars)" });
|
|
443
|
+
return;
|
|
444
|
+
}
|
|
445
|
+
if (description.length > 1000) {
|
|
446
|
+
errors.push({ file: skillDocPath, field: "description", message: "is too long (max 1000 chars)" });
|
|
447
|
+
return;
|
|
448
|
+
}
|
|
449
|
+
skills.push({ id: skillId, name: skillName, description, content: skillText });
|
|
450
|
+
// Supporting files travel with the skill so the runtime materializes
|
|
451
|
+
// the full skill directory, not just SKILL.md content.
|
|
452
|
+
let count = 0;
|
|
453
|
+
for (const [path, bytes] of files) {
|
|
454
|
+
if (!path.startsWith(skillRoot) || path === skillDocPath) continue;
|
|
455
|
+
if (isSkipped(path)) continue;
|
|
456
|
+
count++;
|
|
457
|
+
if (count > MAX_SKILL_FILES) {
|
|
458
|
+
errors.push({ file: skillRoot, field: "", message: `has too many supporting files (max ${MAX_SKILL_FILES})` });
|
|
459
|
+
return;
|
|
460
|
+
}
|
|
461
|
+
if (bytes.length > MAX_SKILL_FILE_BYTES) {
|
|
462
|
+
errors.push({ file: path, field: "", message: `exceeds the ${MAX_SKILL_FILE_BYTES} byte skill-file limit` });
|
|
463
|
+
continue;
|
|
464
|
+
}
|
|
465
|
+
skillFiles.push({
|
|
466
|
+
skillId,
|
|
467
|
+
path: path.slice(skillRoot.length),
|
|
468
|
+
bytes,
|
|
469
|
+
sha256: sha256Hex(bytes),
|
|
470
|
+
size: bytes.length,
|
|
471
|
+
});
|
|
472
|
+
}
|
|
473
|
+
};
|
|
474
|
+
|
|
475
|
+
for (const [path] of files) {
|
|
476
|
+
if (!path.startsWith(DIR_SKILLS) || path.startsWith(DIR_VENDOR)) continue;
|
|
477
|
+
if (isSkipped(path)) continue;
|
|
478
|
+
const rest = path.slice(DIR_SKILLS.length);
|
|
479
|
+
if (rest === "sources.yaml" || rest === "sources.lock") continue;
|
|
480
|
+
const slash = rest.indexOf("/");
|
|
481
|
+
if (slash < 0) {
|
|
482
|
+
if (rest === "README.md") continue;
|
|
483
|
+
errors.push({
|
|
484
|
+
file: path,
|
|
485
|
+
field: "",
|
|
486
|
+
message: "skills/ holds skill directories (with SKILL.md), sources.yaml, or README.md",
|
|
487
|
+
});
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
const dirName = rest.slice(0, slash);
|
|
491
|
+
if (dirName === ".vendor") continue;
|
|
492
|
+
const id = entryId(dirName, `skills/${dirName}`, sink, FILE_BOT_YAML);
|
|
493
|
+
if (id === undefined) continue;
|
|
494
|
+
if (skillDirs.includes(dirName)) continue;
|
|
495
|
+
skillDirs.push(dirName);
|
|
496
|
+
if (rest === `${dirName}/SKILL.md`) {
|
|
497
|
+
ingestSkillDir(id, `${DIR_SKILLS}${dirName}/`, path);
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
// Directories seen but without SKILL.md at depth 1 still error loudly.
|
|
501
|
+
for (const dirName of skillDirs) {
|
|
502
|
+
if (!skills.some((s) => s.id === dirName) && !skillIds[dirName]) {
|
|
503
|
+
ingestSkillDir(dirName, `${DIR_SKILLS}${dirName}/`, `${DIR_SKILLS}${dirName}/`);
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
// --- external skill sources (lock + vendored content) ---
|
|
508
|
+
let lock = null;
|
|
509
|
+
const sourcesDoc = readYamlFile(files, FILE_SOURCES, sink);
|
|
510
|
+
const lockDoc = readYamlFile(files, FILE_LOCK, sink);
|
|
511
|
+
if (sourcesDoc !== undefined || lockDoc !== undefined) {
|
|
512
|
+
const parsed = parseSourcesYaml(sourcesDoc, sink);
|
|
513
|
+
if (lockDoc === null || lockDoc === undefined) {
|
|
514
|
+
if (parsed) {
|
|
515
|
+
errors.push({
|
|
516
|
+
file: FILE_LOCK,
|
|
517
|
+
field: "",
|
|
518
|
+
message: "is required when skills/sources.yaml exists: run `librebot resolve-skills --vendor`",
|
|
519
|
+
});
|
|
520
|
+
}
|
|
521
|
+
} else if (parsed) {
|
|
522
|
+
lock = checkSourcesLock(parsed, lockDoc, sink);
|
|
523
|
+
if (lock) {
|
|
524
|
+
const vendorDirs = [];
|
|
525
|
+
const ingestVendorRoot = (entry, root, skill) => {
|
|
526
|
+
const skillId = ("ext-" + entry.name + "-" + skill).replace(/[^A-Za-z0-9._-]/g, "_").slice(0, 64);
|
|
527
|
+
ingestSkillDir(skillId, root, root + "SKILL.md");
|
|
528
|
+
vendorDirs.push(root);
|
|
529
|
+
};
|
|
530
|
+
for (const entry of lock.resolved) {
|
|
531
|
+
if (entry.contentSha256 && vendorContentHash(Object.fromEntries(files), `${DIR_VENDOR}${entry.name}/`) !== entry.contentSha256) {
|
|
532
|
+
errors.push({ file: FILE_LOCK, field: entry.name, message: "vendored content does not match its locked digest; restore it or run resolve-skills --update --network --vendor" });
|
|
533
|
+
continue;
|
|
534
|
+
}
|
|
535
|
+
if (entry.skills.length > 0) {
|
|
536
|
+
// Explicit selection: exact vendored subtrees only.
|
|
537
|
+
for (const skill of entry.skills) {
|
|
538
|
+
const roots = [
|
|
539
|
+
`${DIR_VENDOR}${entry.name}/${skill}/`,
|
|
540
|
+
`${DIR_VENDOR}${entry.name}/`,
|
|
541
|
+
];
|
|
542
|
+
const root = roots.find((r) => files.get(r + "SKILL.md") !== undefined);
|
|
543
|
+
if (root === undefined) {
|
|
544
|
+
errors.push({
|
|
545
|
+
file: FILE_LOCK,
|
|
546
|
+
field: entry.name,
|
|
547
|
+
message: `locked skill ${skill} is not vendored: run \`librebot resolve-skills --vendor\``,
|
|
548
|
+
});
|
|
549
|
+
continue;
|
|
550
|
+
}
|
|
551
|
+
ingestVendorRoot(entry, root, skill);
|
|
552
|
+
}
|
|
553
|
+
continue;
|
|
554
|
+
}
|
|
555
|
+
// Default selection: discover the same roots the vendor step
|
|
556
|
+
// writes — immediate subdirectories holding SKILL.md, or the
|
|
557
|
+
// source root itself for dedicated single-skill repos.
|
|
558
|
+
const prefix = `${DIR_VENDOR}${entry.name}/`;
|
|
559
|
+
if (files.get(prefix + "SKILL.md") !== undefined) {
|
|
560
|
+
ingestVendorRoot(entry, prefix, entry.name);
|
|
561
|
+
continue;
|
|
562
|
+
}
|
|
563
|
+
const discovered = new Set();
|
|
564
|
+
for (const [path] of files) {
|
|
565
|
+
if (!path.startsWith(prefix) || !path.endsWith("/SKILL.md")) continue;
|
|
566
|
+
const rest = path.slice(prefix.length, -"SKILL.md".length);
|
|
567
|
+
if (rest.endsWith("/") && !rest.slice(0, -1).includes("/")) {
|
|
568
|
+
discovered.add(rest);
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
if (discovered.size === 0) {
|
|
572
|
+
errors.push({
|
|
573
|
+
file: FILE_LOCK,
|
|
574
|
+
field: entry.name,
|
|
575
|
+
message: `locked source has no vendored skills: run \`librebot resolve-skills --vendor\``,
|
|
576
|
+
});
|
|
577
|
+
continue;
|
|
578
|
+
}
|
|
579
|
+
for (const root of [...discovered].sort()) {
|
|
580
|
+
ingestVendorRoot(entry, prefix + root, root.slice(0, -1));
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
lock.vendorDirs = vendorDirs;
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
if (skills.length > 100) {
|
|
588
|
+
errors.push({ file: DIR_SKILLS, field: "", message: "has too many entries (max 100)" });
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
if (errors.length > 0 || name === undefined || description === undefined || botPrompt === undefined) {
|
|
592
|
+
return { ok: false, errors, warnings };
|
|
593
|
+
}
|
|
594
|
+
const draft = {
|
|
595
|
+
name,
|
|
596
|
+
description,
|
|
597
|
+
...(icon ? { icon: `./${icon.path}` } : {}),
|
|
598
|
+
botPrompt,
|
|
599
|
+
readme,
|
|
600
|
+
modelRoles,
|
|
601
|
+
knowledge,
|
|
602
|
+
skills,
|
|
603
|
+
connectors,
|
|
604
|
+
triggers,
|
|
605
|
+
workspace,
|
|
606
|
+
actions,
|
|
607
|
+
};
|
|
608
|
+
// Byte length (UTF-8), exactly like the platform: char counts would let
|
|
609
|
+
// multibyte-heavy drafts pass locally then fail the 750 KB ingest cap.
|
|
610
|
+
const iconPayloadBytes = icon ? 13 + `data:${icon.mime};base64,`.length + 4 * Math.ceil(icon.size / 3) : 0;
|
|
611
|
+
if (new TextEncoder().encode(JSON.stringify(draft)).length + iconPayloadBytes > MAX_DRAFT_BYTES) {
|
|
612
|
+
return {
|
|
613
|
+
ok: false,
|
|
614
|
+
errors: [
|
|
615
|
+
...errors,
|
|
616
|
+
{ file: FILE_BOT_YAML, field: "", message: "bot definition exceeds the 750 KB limit" },
|
|
617
|
+
],
|
|
618
|
+
warnings,
|
|
619
|
+
};
|
|
620
|
+
}
|
|
621
|
+
return {
|
|
622
|
+
ok: true,
|
|
623
|
+
errors,
|
|
624
|
+
warnings,
|
|
625
|
+
formatVersion: FORMAT_VERSION,
|
|
626
|
+
draft,
|
|
627
|
+
assets,
|
|
628
|
+
icon,
|
|
629
|
+
skillFiles,
|
|
630
|
+
source: {
|
|
631
|
+
botYaml: FILE_BOT_YAML,
|
|
632
|
+
instructions: FILE_INSTRUCTIONS,
|
|
633
|
+
knowledgeFiles,
|
|
634
|
+
skillDirs,
|
|
635
|
+
lockPresent: lockDoc !== undefined,
|
|
636
|
+
sourcesPresent: sourcesDoc !== undefined,
|
|
637
|
+
},
|
|
638
|
+
lock,
|
|
639
|
+
};
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
// --- filesystem wrappers (local dev + server-side extracted trees) ---
|
|
643
|
+
|
|
644
|
+
/**
|
|
645
|
+
* Read a directory recursively into a posix-relative file map.
|
|
646
|
+
* Skips dotfile segments (except under skills/.vendor/) and node_modules.
|
|
647
|
+
* Symlinks and special files throw: they must fail loudly, never parse as
|
|
648
|
+
* a silently different tree than server-side ingestion sees.
|
|
649
|
+
*/
|
|
650
|
+
export async function readDirFiles(dir) {
|
|
651
|
+
/** @type {Record<string, Uint8Array>} */
|
|
652
|
+
const files = {};
|
|
653
|
+
async function walk(relative) {
|
|
654
|
+
const entries = await readdir(join(dir, relative), { withFileTypes: true });
|
|
655
|
+
entries.sort((a, b) => (a.name < b.name ? -1 : 1));
|
|
656
|
+
for (const entry of entries) {
|
|
657
|
+
const rel = relative ? `${relative}/${entry.name}` : entry.name;
|
|
658
|
+
if (rel === "node_modules" || rel.startsWith("node_modules/")) continue;
|
|
659
|
+
if (rel !== "skills/.vendor" && !rel.startsWith("skills/.vendor/") && rel.split("/").some((s) => s.startsWith("."))) {
|
|
660
|
+
continue;
|
|
661
|
+
}
|
|
662
|
+
// Symlinks and special files are rejected loudly, never silently
|
|
663
|
+
// omitted: a tree that parses locally must ingest identically
|
|
664
|
+
// server-side, where Git tree modes reject mode120000 blobs.
|
|
665
|
+
if (entry.isSymbolicLink()) {
|
|
666
|
+
throw new Error(`symlinks are not supported: ${rel}`);
|
|
667
|
+
}
|
|
668
|
+
if (entry.isDirectory()) {
|
|
669
|
+
await walk(rel);
|
|
670
|
+
} else if (entry.isFile()) {
|
|
671
|
+
files[rel] = await readFile(join(dir, rel));
|
|
672
|
+
} else {
|
|
673
|
+
throw new Error(`unsupported file type: ${rel}`);
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
await walk("");
|
|
678
|
+
return files;
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* Validate a bot source directory. NEVER throws on invalid source: returns
|
|
683
|
+
* `{ ok, errors, warnings, ... }`. Only filesystem failures throw.
|
|
684
|
+
* - ok true: `{ ok: true, errors: [], warnings, formatVersion, draft,
|
|
685
|
+
* assets, skillFiles, source, lock }`
|
|
686
|
+
* - ok false: `{ ok: false, errors: [{ file, field, message }], warnings }`
|
|
687
|
+
* `bytes` fields are Uint8Arrays; use preview stripping for JSON output.
|
|
688
|
+
*/
|
|
689
|
+
export async function validateSource(dir) {
|
|
690
|
+
return validateFiles(await readDirFiles(dir));
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
/**
|
|
694
|
+
* Parse a bot source directory. Throws BotSourceError (carrying
|
|
695
|
+
* `.errors[]`/`warnings[]`) on invalid source; filesystem failures throw
|
|
696
|
+
* natively. Returns `{ formatVersion, draft, assets, skillFiles, warnings,
|
|
697
|
+
* source, lock }`.
|
|
698
|
+
*/
|
|
699
|
+
export async function parseBotSource(dir) {
|
|
700
|
+
return parseBotFiles(await readDirFiles(dir));
|
|
701
|
+
}
|
|
702
|
+
|