librebot-tools 0.0.0-stage → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,697 @@
1
+ // Skill dependencies resolve locally and are committed as immutable vendor files.
2
+ // Explicit GitHub commit pins use bounded HTTP downloads; other standard skills
3
+ // locators are installed by the pinned skills CLI only with explicit network consent.
4
+
5
+ import { createGunzip } from "node:zlib";
6
+ import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
7
+ import { isRecord } from "./errors.js";
8
+ import { canonicalJson, sha256Hex } from "./util.js";
9
+ import { vendorContentHash } from "./skills-import.js";
10
+ import { entryId } from "./validate.js";
11
+
12
+ export const LOCK_VERSION = 1;
13
+ export const SOURCES_VERSION = 1;
14
+
15
+ const SHA_RE = /^[0-9a-f]{40}$/;
16
+ const SHORTHAND_RE = /^[A-Za-z0-9_.-]{1,100}\/[A-Za-z0-9_.-]{1,100}$/;
17
+
18
+ function err(sink, file, field, message) {
19
+ sink.errors.push({ file, field, message });
20
+ }
21
+
22
+ /**
23
+ * Normalize GitHub repository pins or preserve a standard skills CLI locator.
24
+ */
25
+ export function parseRepo(source) {
26
+ const raw = String(source ?? "").trim().replace(/\/+$/, "");
27
+ const generic = () => raw && raw.length <= 2000 && !raw.startsWith("-") && !/[\0\r\n]/.test(raw)
28
+ ? { owner: "skills-cli", repo: raw, url: raw, source: raw } : null;
29
+ if (SHORTHAND_RE.test(raw)) {
30
+ const [owner, repo] = raw.split("/");
31
+ return { owner, repo: repo.replace(/\.git$/, ""), url: `https://github.com/${owner}/${repo.replace(/\.git$/, "")}` };
32
+ }
33
+ let parsed;
34
+ try {
35
+ parsed = new URL(raw);
36
+ } catch {
37
+ return generic();
38
+ }
39
+ if (parsed.password || (parsed.protocol === "https:" && parsed.username)) return null;
40
+ const parts = parsed.pathname.split("/").filter(Boolean);
41
+ if (parsed.protocol !== "https:" || parsed.hostname !== "github.com" || parts.length !== 2) return generic();
42
+ return {
43
+ owner: parts[0],
44
+ repo: parts[1].replace(/\.git$/, ""),
45
+ url: `https://github.com/${parts[0]}/${parts[1].replace(/\.git$/, "")}`,
46
+ };
47
+ }
48
+
49
+ /**
50
+ * Validate a parsed sources.yaml document.
51
+ * @returns {{ version: 1; sources: Array<{ name: string; repo: { owner: string; repo: string; url: string }; ref: string|undefined; skills: string[] }> }|null}
52
+ */
53
+ export function parseSourcesYaml(doc, sink) {
54
+ const file = "skills/sources.yaml";
55
+ if (!isRecord(doc)) {
56
+ err(sink, file, "", "must be a YAML mapping");
57
+ return null;
58
+ }
59
+ if (doc.version !== SOURCES_VERSION) {
60
+ err(sink, file, "version", `must be ${SOURCES_VERSION}`);
61
+ return null;
62
+ }
63
+ if (!Array.isArray(doc.sources)) {
64
+ err(sink, file, "sources", "must be an array");
65
+ return null;
66
+ }
67
+ if (doc.sources.length > 50) {
68
+ err(sink, file, "sources", "has too many entries (max 50)");
69
+ return null;
70
+ }
71
+ const seen = {};
72
+ const sources = [];
73
+ let ok = true;
74
+ doc.sources.forEach((rawEntry, i) => {
75
+ const entry = typeof rawEntry === "string" ? { name: "source-" + sha256Hex(new TextEncoder().encode(rawEntry)).slice(0, 12), source: rawEntry } : rawEntry;
76
+ const field = `sources[${i}]`;
77
+ if (!isRecord(entry)) {
78
+ err(sink, file, field, "must be an object");
79
+ ok = false;
80
+ return;
81
+ }
82
+ for (const key of Object.keys(entry)) {
83
+ if (!["name", "source", "ref", "skills"].includes(key)) {
84
+ err(sink, file, field, `contains unsupported field ${key}`);
85
+ ok = false;
86
+ }
87
+ }
88
+ const name = entryId(entry.name, field, sink, file);
89
+ if (name === undefined) {
90
+ ok = false;
91
+ return;
92
+ }
93
+ if (seen[name]) {
94
+ err(sink, file, field, `contains duplicate source ${name}`);
95
+ ok = false;
96
+ return;
97
+ }
98
+ seen[name] = true;
99
+ let repo = parseRepo(entry.source);
100
+ if (!repo) {
101
+ err(
102
+ sink,
103
+ file,
104
+ `${field}.source`,
105
+ "must be a non-empty skills CLI source locator without embedded credentials",
106
+ );
107
+ ok = false;
108
+ return;
109
+ }
110
+ // Without an explicit GitHub ref, delegate discovery and default branches
111
+ // to the same CLI developers use with `npx skills`.
112
+ if (entry.ref === undefined && !repo.source) repo = { owner: "skills-cli", repo: String(entry.source).trim(), url: String(entry.source).trim(), source: String(entry.source).trim() };
113
+ if (repo.source && entry.ref !== undefined) {
114
+ err(sink, file, `${field}.ref`, "encode the ref in the skills source locator for this source format");
115
+ ok = false;
116
+ return;
117
+ }
118
+ let ref;
119
+ if (entry.ref !== undefined) {
120
+ if (typeof entry.ref !== "string" || !entry.ref.trim() || entry.ref.length > 256) {
121
+ err(sink, file, `${field}.ref`, "must be a branch, tag, or commit SHA");
122
+ ok = false;
123
+ return;
124
+ }
125
+ ref = entry.ref.trim();
126
+ }
127
+ let skills = [];
128
+ if (entry.skills !== undefined) {
129
+ if (!Array.isArray(entry.skills) || entry.skills.length > 50) {
130
+ err(sink, file, `${field}.skills`, "must be an array of at most 50 skill names");
131
+ ok = false;
132
+ return;
133
+ }
134
+ for (let j = 0; j < entry.skills.length; j++) {
135
+ const skill = entry.skills[j];
136
+ if (typeof skill !== "string" || !skill.trim() || skill.trim().startsWith("-") || /[\0\r\n]/.test(skill) || skill.length > 100) {
137
+ err(sink, file, `${field}.skills[${j}]`, "must be a non-empty string up to 100 chars");
138
+ ok = false;
139
+ return;
140
+ }
141
+ skills.push(skill.trim());
142
+ }
143
+ }
144
+ sources.push({ name, repo, ref, skills });
145
+ });
146
+ return ok ? { version: SOURCES_VERSION, sources } : null;
147
+ }
148
+
149
+ /** Stable hash of the normalized source declarations. */
150
+ export function sourcesHash(sources) {
151
+ return sha256Hex(
152
+ new TextEncoder().encode(
153
+ canonicalJson(
154
+ sources.map((s) => ({
155
+ name: s.name,
156
+ repo: `${s.repo.owner}/${s.repo.repo}`,
157
+ ref: s.ref ?? null,
158
+ skills: [...s.skills].sort(),
159
+ })),
160
+ ),
161
+ ),
162
+ );
163
+ }
164
+
165
+ function validResolved(entry) {
166
+ return (
167
+ isRecord(entry) &&
168
+ typeof entry.name === "string" &&
169
+ typeof entry.repo === "string" &&
170
+ ((typeof entry.resolvedSha === "string" && SHA_RE.test(entry.resolvedSha)) ||
171
+ (typeof entry.contentSha256 === "string" && /^[0-9a-f]{64}$/.test(entry.contentSha256))) &&
172
+ (entry.tarballSha256 === undefined || /^[0-9a-f]{64}$/.test(entry.tarballSha256)) &&
173
+ Array.isArray(entry.skills)
174
+ );
175
+ }
176
+
177
+ /**
178
+ * Validate sources.lock against parsed sources.yaml. Fails stale locks so a
179
+ * pinned bot can never silently follow a changed declaration.
180
+ */
181
+ export function checkSourcesLock(parsedSources, lockDoc, sink) {
182
+ const file = "skills/sources.lock";
183
+ if (!isRecord(lockDoc)) {
184
+ err(sink, file, "", "must be a YAML mapping");
185
+ return null;
186
+ }
187
+ if (lockDoc.version !== LOCK_VERSION) {
188
+ err(sink, file, "version", `must be ${LOCK_VERSION}`);
189
+ return null;
190
+ }
191
+ if (!Array.isArray(lockDoc.resolved)) {
192
+ err(sink, file, "resolved", "must be an array");
193
+ return null;
194
+ }
195
+ const expected = sourcesHash(parsedSources.sources);
196
+ if (lockDoc.sourcesHash !== expected) {
197
+ err(sink, file, "sourcesHash", "is stale: re-run `librebot resolve-skills`");
198
+ return null;
199
+ }
200
+ const byName = new Map(parsedSources.sources.map((s) => [s.name, s]));
201
+ const resolved = [];
202
+ for (let i = 0; i < lockDoc.resolved.length; i++) {
203
+ const entry = lockDoc.resolved[i];
204
+ if (!validResolved(entry)) {
205
+ err(sink, file, `resolved[${i}]`, "is malformed");
206
+ return null;
207
+ }
208
+ const declared = byName.get(entry.name);
209
+ if (!declared) {
210
+ err(sink, file, `resolved[${i}]`, `resolves undeclared source ${entry.name}`);
211
+ return null;
212
+ }
213
+ if (entry.repo !== `${declared.repo.owner}/${declared.repo.repo}`) {
214
+ err(sink, file, `resolved[${i}].repo`, "does not match skills/sources.yaml");
215
+ return null;
216
+ }
217
+ const declaredSkills = [...declared.skills].sort();
218
+ const lockedSkills = [...entry.skills].sort();
219
+ if (JSON.stringify(declaredSkills) !== JSON.stringify(lockedSkills)) {
220
+ err(sink, file, `resolved[${i}].skills`, "does not match skills/sources.yaml");
221
+ return null;
222
+ }
223
+ if (
224
+ declared.ref !== undefined &&
225
+ SHA_RE.test(declared.ref) &&
226
+ declared.ref.toLowerCase() !== entry.resolvedSha
227
+ ) {
228
+ err(sink, file, `resolved[${i}].resolvedSha`, "does not match the pinned ref");
229
+ return null;
230
+ }
231
+ byName.delete(entry.name);
232
+ resolved.push({
233
+ name: entry.name,
234
+ repo: entry.repo,
235
+ resolvedSha: entry.resolvedSha,
236
+ tarballSha256: entry.tarballSha256,
237
+ ...(entry.contentSha256 ? { contentSha256: entry.contentSha256 } : {}),
238
+ skills: [...entry.skills],
239
+ });
240
+ }
241
+ if (byName.size > 0) {
242
+ err(
243
+ sink,
244
+ file,
245
+ "resolved",
246
+ `missing resolutions for: ${[...byName.keys()].join(", ")}`,
247
+ );
248
+ return null;
249
+ }
250
+ return { version: LOCK_VERSION, sourcesHash: expected, resolved, vendorDirs: [] };
251
+ }
252
+
253
+ export function serializeLock(lock) {
254
+ return stringifyYaml({
255
+ version: LOCK_VERSION,
256
+ sourcesHash: lock.sourcesHash,
257
+ resolved: lock.resolved.map((r) => ({
258
+ name: r.name,
259
+ repo: r.repo,
260
+ ...(r.resolvedSha ? { resolvedSha: r.resolvedSha } : {}),
261
+ ...(r.contentSha256 ? { contentSha256: r.contentSha256 } : {}),
262
+ ...(r.tarballSha256 ? { tarballSha256: r.tarballSha256 } : {}),
263
+ skills: r.skills,
264
+ })),
265
+ });
266
+ }
267
+
268
+ async function resolveRefToSha(repo, ref, fetchImpl) {
269
+ const response = await fetchImpl(
270
+ `https://api.github.com/repos/${repo.owner}/${repo.repo}/commits/${encodeURIComponent(ref)}`,
271
+ { headers: { Accept: "application/vnd.github+json" } },
272
+ );
273
+ if (!response.ok) {
274
+ throw new Error(`could not resolve ${repo.owner}/${repo.repo}@${ref} (HTTP ${response.status})`);
275
+ }
276
+ const body = await response.json();
277
+ if (typeof body?.sha !== "string" || !SHA_RE.test(body.sha)) {
278
+ throw new Error(`could not resolve ${repo.owner}/${repo.repo}@${ref}: bad response`);
279
+ }
280
+ return body.sha;
281
+ }
282
+
283
+ export const MAX_TARBALL_BYTES = 20 * 1024 * 1024;
284
+ export const MAX_DECOMPRESSED_BYTES = 100 * 1024 * 1024;
285
+
286
+ /**
287
+ * Read a byte stream enforcing a hard cap while reading: the cap trips
288
+ * before the full payload is allocated, so a small compressed response
289
+ * hiding a huge expansion cannot exhaust memory first.
290
+ */
291
+ async function readCappedStream(stream, maxBytes, what) {
292
+ const reader = stream.getReader();
293
+ const chunks = [];
294
+ let total = 0;
295
+ try {
296
+ for (;;) {
297
+ const { done, value } = await reader.read();
298
+ if (done) break;
299
+ total += value.byteLength;
300
+ if (total > maxBytes) {
301
+ throw new Error(`${what} exceeds the ${maxBytes} byte limit`);
302
+ }
303
+ chunks.push(value);
304
+ }
305
+ } finally {
306
+ await reader.cancel().catch(() => {});
307
+ }
308
+ const out = new Uint8Array(total);
309
+ let offset = 0;
310
+ for (const chunk of chunks) {
311
+ out.set(chunk, offset);
312
+ offset += chunk.byteLength;
313
+ }
314
+ return out;
315
+ }
316
+
317
+ async function fetchTarball(repo, sha, fetchImpl) {
318
+ const response = await fetchImpl(
319
+ `https://codeload.github.com/${repo.owner}/${repo.repo}/tar.gz/${sha}`,
320
+ );
321
+ if (!response.ok) {
322
+ throw new Error(`could not download ${repo.owner}/${repo.repo}@${sha} (HTTP ${response.status})`);
323
+ }
324
+ if (response.body !== undefined && response.body !== null && typeof response.body.getReader === "function") {
325
+ return readCappedStream(response.body, MAX_TARBALL_BYTES, "tarball");
326
+ }
327
+ const buffer = new Uint8Array(await response.arrayBuffer());
328
+ if (buffer.length > MAX_TARBALL_BYTES) {
329
+ throw new Error(`tarball exceeds the ${MAX_TARBALL_BYTES} byte limit`);
330
+ }
331
+ return buffer;
332
+ }
333
+
334
+ /**
335
+ * Resolve sources.yaml to a lock. SHA-pinned refs resolve offline;
336
+ * branch/tag refs need `network: true`. With `vendor: true`, pinned
337
+ * tarballs are downloaded (hash recorded) and vendor file maps returned.
338
+ * Standard CLI locators require explicit network consent and vendoring.
339
+ */
340
+ export async function resolveSkillsLock({
341
+ sources,
342
+ existingLock,
343
+ update = false,
344
+ network = false,
345
+ vendor = false,
346
+ fetchImpl = globalThis.fetch,
347
+ sourceBaseDir,
348
+ currentFiles = {},
349
+ }) {
350
+ const errors = [];
351
+ const warnings = [];
352
+ const resolved = [];
353
+ /** @type {Record<string, Uint8Array>} */
354
+ const vendorFiles = {};
355
+ for (const source of sources.sources) {
356
+ const label = `sources.${source.name}`;
357
+ const prior = existingLock?.resolved?.find((r) => r.name === source.name);
358
+ if (source.repo.source) {
359
+ try {
360
+ const same = prior?.repo === `${source.repo.owner}/${source.repo.repo}` &&
361
+ JSON.stringify([...(prior.skills ?? [])].sort()) === JSON.stringify([...source.skills].sort());
362
+ const prefix = `skills/.vendor/${source.name}/`;
363
+ const retained = Object.fromEntries(Object.entries(currentFiles).filter(([path]) => path.startsWith(prefix)));
364
+ if (same && !update && prior.contentSha256 && Object.keys(retained).length &&
365
+ vendorContentHash(retained, prefix) === prior.contentSha256) {
366
+ resolved.push(prior);
367
+ if (vendor) Object.assign(vendorFiles, retained);
368
+ continue;
369
+ }
370
+ if (same && !update && !vendor) { resolved.push(prior); continue; }
371
+ if (!network || !vendor) throw new Error("run resolve-skills --network --vendor to install this skills CLI source");
372
+ const { importSkillSource } = await import("./skills-import.js");
373
+ const installed = await importSkillSource(source, { sourceBaseDir });
374
+ if (same && !update && prior.contentSha256 && installed.contentSha256 !== prior.contentSha256) throw new Error("installed content differs from the lock; use --update to adopt it");
375
+ resolved.push({ name: source.name, repo: `${source.repo.owner}/${source.repo.repo}`, contentSha256: installed.contentSha256, skills: [...source.skills] });
376
+ Object.assign(vendorFiles, installed.files);
377
+ } catch (error) { errors.push({ file: "skills/sources.yaml", field: label, message: error.message }); }
378
+ continue;
379
+ }
380
+ const sameDecl =
381
+ prior !== undefined &&
382
+ prior.repo === `${source.repo.owner}/${source.repo.repo}` &&
383
+ JSON.stringify([...prior.skills].sort()) ===
384
+ JSON.stringify([...source.skills].sort()) &&
385
+ (source.ref === undefined ||
386
+ (SHA_RE.test(source.ref) &&
387
+ source.ref.toLowerCase() === prior.resolvedSha));
388
+ if (prior !== undefined && sameDecl && !update) {
389
+ resolved.push({ ...prior, skills: [...prior.skills] });
390
+ if (vendor) {
391
+ try {
392
+ const downloaded = await fetchTarball(source.repo, prior.resolvedSha, fetchImpl);
393
+ if (prior.tarballSha256 !== undefined && sha256Hex(downloaded) !== prior.tarballSha256) {
394
+ throw new Error(`${label}: tarball hash mismatch for pinned ${prior.resolvedSha}`);
395
+ }
396
+ Object.assign(vendorFiles, await extractVendorFiles(source, downloaded));
397
+ const found = resolved[resolved.length - 1];
398
+ if (found.tarballSha256 === undefined) {
399
+ found.tarballSha256 = sha256Hex(downloaded);
400
+ }
401
+ } catch (error) {
402
+ errors.push({ file: "skills/sources.yaml", field: label, message: error.message });
403
+ }
404
+ }
405
+ continue;
406
+ }
407
+ if (source.ref === undefined) {
408
+ errors.push({
409
+ file: "skills/sources.yaml",
410
+ field: label,
411
+ message: "needs an explicit ref (branch, tag, or commit SHA)",
412
+ });
413
+ continue;
414
+ }
415
+ let sha;
416
+ if (SHA_RE.test(source.ref)) {
417
+ sha = source.ref.toLowerCase();
418
+ } else if (network) {
419
+ try {
420
+ sha = await resolveRefToSha(source.repo, source.ref, fetchImpl);
421
+ } catch (error) {
422
+ errors.push({ file: "skills/sources.yaml", field: label, message: error.message });
423
+ continue;
424
+ }
425
+ } else {
426
+ errors.push({ file: "skills/sources.yaml", field: label, message: `ref ${source.ref} is not a commit SHA: pin a SHA or re-run with network access` });
427
+ continue;
428
+ }
429
+ // A recorded tarball hash authenticates exactly one pinned archive:
430
+ // carry it only when repo and SHA are identical, otherwise a real
431
+ // update would always fail against the previous archive's hash.
432
+ const samePin =
433
+ prior?.repo === `${source.repo.owner}/${source.repo.repo}` &&
434
+ prior?.resolvedSha === sha;
435
+ const entry = {
436
+ name: source.name,
437
+ repo: `${source.repo.owner}/${source.repo.repo}`,
438
+ resolvedSha: sha,
439
+ tarballSha256: samePin ? prior?.tarballSha256 : undefined,
440
+ skills: [...source.skills],
441
+ };
442
+ if (vendor) {
443
+ try {
444
+ const downloaded = await fetchTarball(source.repo, sha, fetchImpl);
445
+ if (entry.tarballSha256 !== undefined && sha256Hex(downloaded) !== entry.tarballSha256) {
446
+ throw new Error(`${label}: tarball hash mismatch for pinned ${sha}`);
447
+ }
448
+ entry.tarballSha256 = sha256Hex(downloaded);
449
+ Object.assign(vendorFiles, await extractVendorFiles(source, downloaded));
450
+ } catch (error) {
451
+ errors.push({ file: "skills/sources.yaml", field: label, message: error.message });
452
+ continue;
453
+ }
454
+ } else if (entry.tarballSha256 === undefined) {
455
+ warnings.push({
456
+ file: "skills/sources.yaml",
457
+ field: label,
458
+ message: "locked without a tarball hash: run with --vendor to record one",
459
+ });
460
+ }
461
+ resolved.push(entry);
462
+ }
463
+ const lock = {
464
+ version: LOCK_VERSION,
465
+ sourcesHash: sourcesHash(sources.sources),
466
+ resolved,
467
+ vendorDirs: [],
468
+ };
469
+ return { lock, vendorFiles, errors, warnings };
470
+ }
471
+
472
+ /**
473
+ * Download every locked tarball (verifying hashes) and extract vendor files.
474
+ * Returns `{ vendorFiles, errors }`; vendorFiles keys are repo-relative
475
+ * `skills/.vendor/<name>/...` paths for direct commit into the source tree.
476
+ */
477
+ export async function fetchLockedSkills(lock, { fetchImpl = globalThis.fetch } = {}) {
478
+ const errors = [];
479
+ /** @type {Record<string, Uint8Array>} */
480
+ const vendorFiles = {};
481
+ for (const entry of lock.resolved ?? []) {
482
+ const [owner, repo] = String(entry.repo).split("/");
483
+ try {
484
+ if (entry.contentSha256) throw new Error("content-locked CLI sources restore from committed vendor files; use resolve-skills --network --vendor to fetch and verify them locally");
485
+ const downloaded = await fetchTarball({ owner, repo }, entry.resolvedSha, fetchImpl);
486
+ if (entry.tarballSha256 !== undefined && sha256Hex(downloaded) !== entry.tarballSha256) {
487
+ throw new Error(`tarball hash mismatch for pinned ${entry.resolvedSha}`);
488
+ }
489
+ Object.assign(
490
+ vendorFiles,
491
+ await extractVendorFiles(
492
+ { name: entry.name, skills: entry.skills ?? [] },
493
+ downloaded,
494
+ ),
495
+ );
496
+ } catch (error) {
497
+ errors.push({ file: "skills/sources.lock", field: entry.name, message: error.message });
498
+ }
499
+ }
500
+ return { vendorFiles, errors };
501
+ }
502
+
503
+ export function parseSourcesText(text) {
504
+ return parseYaml(text);
505
+ }
506
+
507
+ // --- tarball extraction (no shell, no command execution) ---
508
+
509
+ const MAX_VENDOR_FILES = 200;
510
+ // Supporting files use the same binary MiB bound as platform validation.
511
+ const MAX_VENDOR_FILE_BYTES = 1024 * 1024;
512
+ // Tar-level bounds enforced while reading, before bytes are copied: they
513
+ const MAX_TAR_ENTRIES = 5000;
514
+ // Copy cap matches the 10,000,000-byte knowledge-asset ceiling so no valid
515
+ // file is ever skipped at the transport layer; policy caps (1 MB skill
516
+ // files, selected-output limits) still decide what gets vendored.
517
+ const MAX_TAR_FILE_BYTES = 10 * 1024 * 1024;
518
+ const MAX_TAR_HEADER_BYTES = 64 * 1024;
519
+
520
+ async function gunzip(bytes) {
521
+ return new Promise((resolve, reject) => {
522
+ const gunzipStream = createGunzip();
523
+ const chunks = [];
524
+ let total = 0;
525
+ let settled = false;
526
+ const fail = (error) => {
527
+ if (settled) return;
528
+ settled = true;
529
+ gunzipStream.destroy();
530
+ reject(error);
531
+ };
532
+ gunzipStream.on("data", (chunk) => {
533
+ total += chunk.length;
534
+ if (total > MAX_DECOMPRESSED_BYTES) {
535
+ fail(new Error(`decompressed tarball exceeds the ${MAX_DECOMPRESSED_BYTES} byte limit`));
536
+ return;
537
+ }
538
+ chunks.push(chunk);
539
+ });
540
+ gunzipStream.on("end", () => {
541
+ if (settled) return;
542
+ settled = true;
543
+ resolve(Buffer.concat(chunks));
544
+ });
545
+ gunzipStream.on("error", fail);
546
+ gunzipStream.end(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength));
547
+ });
548
+ }
549
+
550
+ /**
551
+ * Minimal tar reader: regular files only, deterministic order. Handles the
552
+ * USTAR prefix field plus GNU longname (`L`) and PAX extended-header (`x`)
553
+ * path overrides, so skill trees with long supporting paths extract
554
+ * completely. Symlinks, hardlinks, devices, and other special entries are
555
+ * skipped; absolute or `..`-containing paths throw, so no yielded entry
556
+ * can escape its destination. Entry-count, per-file, and header caps are
557
+ * enforced while reading, before bytes are copied. Exported for
558
+ * server-side archive ingestion that needs the same guarantees.
559
+ */
560
+ export function readTarEntries(buffer) {
561
+ // Entries record `skipped: true` instead of copying bytes once the
562
+ // per-file copy cap trips: huge unrelated files never get allocated,
563
+ // while the extractor can still fail loudly if a *selected* file was
564
+ // skipped rather than silently vendoring a partial tree.
565
+ /** @type {Array<{ name: string; bytes: Uint8Array; skipped: boolean }>} */
566
+ const out = [];
567
+ let offset = 0;
568
+ let pendingLongName = null;
569
+ let pendingPaxPath = null;
570
+ let entryCount = 0;
571
+ const text = (start, length) => {
572
+ let end = start;
573
+ while (end < start + length && buffer[end] !== 0) end++;
574
+ return new TextDecoder().decode(buffer.slice(start, end));
575
+ };
576
+ const decodeDataText = (start, size) => {
577
+ if (size > MAX_TAR_HEADER_BYTES) {
578
+ throw new Error("tarball has an oversized header entry");
579
+ }
580
+ const bytes = buffer.slice(start, start + size);
581
+ let end = bytes.length;
582
+ const nul = bytes.indexOf(0);
583
+ if (nul >= 0) end = nul;
584
+ return new TextDecoder().decode(bytes.slice(0, end));
585
+ };
586
+ while (offset + 512 <= buffer.length) {
587
+ const rawName = text(offset, 100);
588
+ const prefix = text(offset + 345, 155);
589
+ if (!rawName && !prefix && pendingLongName === null && pendingPaxPath === null) break;
590
+ entryCount++;
591
+ if (entryCount > MAX_TAR_ENTRIES) {
592
+ throw new Error(`tarball has too many entries (max ${MAX_TAR_ENTRIES})`);
593
+ }
594
+ const size = parseInt(text(offset + 124, 12).trim() || "0", 8);
595
+ const typeFlag = String.fromCharCode(buffer[offset + 156] ?? 0);
596
+ const dataStart = offset + 512;
597
+ const dataEnd = dataStart + size;
598
+ if (!Number.isFinite(size) || size < 0 || dataEnd > buffer.length) break;
599
+ offset = dataStart + Math.ceil(size / 512) * 512;
600
+ if (typeFlag === "L") {
601
+ pendingLongName = decodeDataText(dataStart, size);
602
+ continue;
603
+ }
604
+ if (typeFlag === "x" || typeFlag === "g") {
605
+ const paxPath = parsePaxPath(decodeDataText(dataStart, size));
606
+ if (paxPath !== null) pendingPaxPath = paxPath;
607
+ continue;
608
+ }
609
+ const headerName = prefix ? `${prefix}/${rawName}` : rawName;
610
+ const name = pendingLongName ?? pendingPaxPath ?? headerName;
611
+ pendingLongName = null;
612
+ pendingPaxPath = null;
613
+ if (!name) continue;
614
+ if (name.startsWith("/") || name.split("/").some((segment) => segment === "..")) {
615
+ throw new Error("tarball has an unsafe entry path");
616
+ }
617
+ if (typeFlag === "0" || typeFlag === "\0") {
618
+ if (size > MAX_TAR_FILE_BYTES) {
619
+ out.push({ name, bytes: new Uint8Array(0), skipped: true });
620
+ } else {
621
+ out.push({ name, bytes: buffer.slice(dataStart, dataEnd), skipped: false });
622
+ }
623
+ }
624
+ }
625
+ return out;
626
+ }
627
+
628
+ /** Extract the `path=` override from PAX extended-header records. */
629
+ function parsePaxPath(records) {
630
+ for (const line of records.split("\n")) {
631
+ const match = /^\d+ path=(.*)$/.exec(line.replace(/\0+$/, ""));
632
+ if (match) return match[1];
633
+ }
634
+ return null;
635
+ }
636
+
637
+ /**
638
+ * Extract a pinned tarball into `skills/.vendor/<name>/...`, selecting only
639
+ * the locked skill subtrees (plus their supporting files). Skips dotfiles,
640
+ * symlinks are never materialized (USTAR reader takes files only).
641
+ */
642
+ export async function extractVendorFiles(source, tarball) {
643
+ const tar = readTarEntries(await gunzip(tarball));
644
+ /** @type {Record<string, Uint8Array>} */
645
+ const vendorFiles = {};
646
+ const wanted = [...source.skills];
647
+ // Strip the `<repo>-<sha>/` prefix codeload prepends.
648
+ const entries = [];
649
+ for (const entry of tar) {
650
+ const slash = entry.name.indexOf("/");
651
+ if (slash < 0) continue;
652
+ entries.push({ name: entry.name.slice(slash + 1), bytes: entry.bytes, skipped: entry.skipped });
653
+ }
654
+ entries.sort((a, b) => (a.name < b.name ? -1 : 1));
655
+ let roots;
656
+ if (wanted.length > 0) {
657
+ roots = wanted.map((s) => s.replace(/\/+$/, "") + "/");
658
+ } else {
659
+ // No skill selection: vendor top-level dirs containing SKILL.md — plus
660
+ // the source root itself when the repo is one dedicated skill
661
+ // (root-level SKILL.md), matching the parser's .vendor/<name>/ fallback.
662
+ const skillRoots = new Set();
663
+ for (const entry of entries) {
664
+ if (entry.name === "SKILL.md") {
665
+ skillRoots.add("");
666
+ } else if (entry.name.endsWith("/SKILL.md")) {
667
+ skillRoots.add(entry.name.slice(0, entry.name.length - "SKILL.md".length));
668
+ }
669
+ }
670
+ roots = [...skillRoots].filter(
671
+ (root) => root.split("/").filter(Boolean).length <= 2,
672
+ );
673
+ }
674
+ let count = 0;
675
+ for (const entry of entries) {
676
+ if (entry.name.split("/").some((s) => s.startsWith("."))) continue;
677
+ if (!roots.some((root) => entry.name === root + "SKILL.md" || entry.name.startsWith(root))) {
678
+ continue;
679
+ }
680
+ if (entry.skipped) {
681
+ throw new Error(`vendored file ${entry.name} exceeds the size limit`);
682
+ }
683
+ if (entry.bytes.length > MAX_VENDOR_FILE_BYTES) {
684
+ throw new Error(`vendored file ${entry.name} exceeds the size limit`);
685
+ }
686
+ count++;
687
+ if (count > MAX_VENDOR_FILES) {
688
+ throw new Error("vendored skill tree has too many files");
689
+ }
690
+ vendorFiles[`skills/.vendor/${source.name}/${entry.name}`] = entry.bytes;
691
+ }
692
+ const hasSkillDoc = Object.keys(vendorFiles).some((k) => k.endsWith("/SKILL.md") || k.endsWith("SKILL.md"));
693
+ if (!hasSkillDoc) {
694
+ throw new Error(`no SKILL.md found for the locked skill selection`);
695
+ }
696
+ return vendorFiles;
697
+ }