@blamejs/exceptd-skills 0.9.1

package/lib/validate-indexes.js

@@ -0,0 +1,83 @@
+"use strict";
+/**
+ * lib/validate-indexes.js
+ *
+ * Predeploy gate. Confirms that `data/_indexes/*.json` is current
+ * against the canonical sources (manifest.json + every data/*.json
+ * minus _indexes/* + every skill body).
+ *
+ * Strategy:
+ *   1. Load `data/_indexes/_meta.json` for the source SHA-256 table.
+ *   2. Re-hash every listed source file.
+ *   3. Fail if any hash diverges (the indexes are stale).
+ *   4. Fail if a new source file exists that's not in the index (the
+ *      index doesn't reflect current state).
+ *
+ * Exit 0 on success, 1 on staleness.
+ *
+ * Run as: node lib/validate-indexes.js
+ * Or as predeploy gate via scripts/predeploy.js.
+ *
+ * Re-build with: npm run build-indexes
+ */
+
+const fs = require("fs");
+const path = require("path");
+const crypto = require("crypto");
+
+const ROOT = path.join(__dirname, "..");
+const ABS = (p) => path.join(ROOT, p);
+const IDX_DIR = ABS("data/_indexes");
+const META = path.join(IDX_DIR, "_meta.json");
+
+function sha256(buf) {
+  return crypto.createHash("sha256").update(buf).digest("hex");
+}
+
+if (!fs.existsSync(META)) {
+  console.error("[validate-indexes] data/_indexes/_meta.json missing — run `npm run build-indexes`.");
+  process.exit(1);
+}
+
+const meta = JSON.parse(fs.readFileSync(META, "utf8"));
+const recorded = meta.source_hashes || {};
+
+// Discover the current canonical source set.
+const manifest = JSON.parse(fs.readFileSync(ABS("manifest.json"), "utf8"));
+const liveSources = new Set();
+liveSources.add("manifest.json");
+for (const f of fs.readdirSync(ABS("data"))) {
+  if (f.endsWith(".json")) liveSources.add("data/" + f);
+}
+for (const s of manifest.skills) liveSources.add(s.path);
+
+const drift = [];
+const missing = [];
+const recordedKeys = new Set(Object.keys(recorded));
+
+for (const p of liveSources) {
+  if (!recordedKeys.has(p)) {
+    missing.push(`new source not in index: ${p}`);
+    continue;
+  }
+  const live = sha256(fs.readFileSync(ABS(p)));
+  if (live !== recorded[p]) {
+    drift.push(`hash drift: ${p} (recorded ${recorded[p].slice(0, 12)}…, live ${live.slice(0, 12)}…)`);
+  }
+}
+for (const p of recordedKeys) {
+  if (!liveSources.has(p)) {
+    missing.push(`stale source in index (file removed): ${p}`);
+  }
+}
+
+const issues = [...drift, ...missing];
+if (issues.length === 0) {
+  console.log(`[validate-indexes] indexes current — ${recordedKeys.size} sources hashed at ${meta.generated_at}.`);
+  process.exit(0);
+}
+
+console.error("[validate-indexes] indexes STALE:");
+for (const i of issues) console.error("  • " + i);
+console.error("[validate-indexes] regenerate with: npm run build-indexes");
+process.exit(1);

package/lib/validate-package.js

@@ -0,0 +1,162 @@
+"use strict";
+/**
+ * lib/validate-package.js
+ *
+ * Predeploy + prepublishOnly gate. Runs `npm pack --dry-run --json` and
+ * asserts the publish tarball is what we expect:
+ *
+ *   - includes every required file from package.json `files`
+ *   - excludes every forbidden file (secrets, tests, caches, dev artifacts)
+ *   - is under the size budget (currently 5 MB)
+ *   - `bin/exceptd.js` has the expected shebang
+ *   - the bin target listed in package.json exists on disk
+ *
+ * Exit 0 on success, 1 on any violation.
+ *
+ * Zero npm deps. Node 24 stdlib.
+ */
+
+const fs = require("fs");
+const path = require("path");
+const { spawnSync } = require("child_process");
+
+const ROOT = path.join(__dirname, "..");
+const ABS = (p) => path.join(ROOT, p);
+const SIZE_BUDGET_BYTES = 5 * 1024 * 1024;          // 5 MB published-tarball cap
+
+const REQUIRED_PATHS = [
+  "package.json",
+  "README.md",
+  "LICENSE",
+  "NOTICE",
+  "AGENTS.md",
+  "manifest.json",
+  "manifest-snapshot.json",
+  "sbom.cdx.json",
+  "bin/exceptd.js",
+  "lib/refresh-external.js",
+  "lib/job-queue.js",
+  "lib/prefetch.js",
+  "lib/worker-pool.js",
+  "lib/verify.js",
+  "vendor/blamejs/retry.js",
+  "vendor/blamejs/worker-pool.js",
+  "vendor/blamejs/_PROVENANCE.json",
+  "vendor/blamejs/LICENSE",
+  "data/_indexes/_meta.json",
+  "keys/public.pem",
+];
+
+// File / directory prefixes that MUST NOT appear in the publish tarball.
+const FORBIDDEN_PATTERNS = [
+  /(^|\/)\.keys(\/|$)/,                              // private signing key
+  /(^|\/)\.cache(\/|$)/,                             // local upstream cache
+  /(^|\/)tests(\/|$)/,                               // test sources + fixtures
+  /(^|\/)refresh-report\.json$/,                     // runtime artifact
+  /(^|\/)\.env(\b|\.)/,                              // any env file
+  /(^|\/)\.git(\/|$|hub\/)/,                         // git internals — but allow .github/workflows in repo,
+                                                     // it's already excluded by .npmignore semantics for files[]
+  /(^|\/)\.DS_Store$/,
+  /(^|\/)node_modules(\/|$)/,
+  /\.pem$/,                                          // catches .keys/private.pem if it sneaks in;
+                                                     // keys/public.pem is whitelisted below
+];
+
+const PEM_ALLOWLIST = new Set(["keys/public.pem"]);
+
+function runNpmPack() {
+  // `npm pack --dry-run --json` writes a JSON array to stdout describing
+  // what would be in the tarball without actually creating it.
+  const res = spawnSync("npm", ["pack", "--dry-run", "--json"], { cwd: ROOT, encoding: "utf8", shell: process.platform === "win32" });
+  if (res.status !== 0) {
+    process.stderr.write(`[validate-package] npm pack failed (exit ${res.status}): ${res.stderr || res.stdout}\n`);
+    process.exit(1);
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(res.stdout);
+  } catch (err) {
+    process.stderr.write(`[validate-package] could not parse npm pack output: ${err.message}\n`);
+    process.exit(1);
+  }
+  const first = Array.isArray(parsed) ? parsed[0] : parsed;
+  if (!first || !Array.isArray(first.files)) {
+    process.stderr.write(`[validate-package] unexpected npm pack output shape\n`);
+    process.exit(1);
+  }
+  return first;
+}
+
+function main() {
+  const issues = [];
+
+  const pkg = JSON.parse(fs.readFileSync(ABS("package.json"), "utf8"));
+
+  // package.json sanity
+  if (pkg.private === true) issues.push(`package.json "private" is true — npm publish will fail`);
+  if (!pkg.bin || !pkg.bin.exceptd) issues.push(`package.json missing bin.exceptd`);
+  if (!Array.isArray(pkg.files) || pkg.files.length === 0) issues.push(`package.json missing files[] whitelist`);
+  if (!pkg.publishConfig || pkg.publishConfig.access !== "public") {
+    issues.push(`package.json missing publishConfig.access: public (scoped package needs explicit access)`);
+  }
+
+  // bin target exists + has a shebang
+  if (pkg.bin && pkg.bin.exceptd) {
+    const binPath = ABS(pkg.bin.exceptd);
+    if (!fs.existsSync(binPath)) {
+      issues.push(`bin target ${pkg.bin.exceptd} does not exist`);
+    } else {
+      const head = fs.readFileSync(binPath, "utf8").slice(0, 64);
+      if (!head.startsWith("#!/usr/bin/env node") && !head.startsWith("#!/usr/bin/node")) {
+        issues.push(`bin/${path.basename(binPath)} missing #!/usr/bin/env node shebang`);
+      }
+    }
+  }
+
+  // npm pack dry-run
+  const packInfo = runNpmPack();
+  const filePaths = packInfo.files.map((f) => f.path.replace(/\\/g, "/"));
+  const fileSet = new Set(filePaths);
+
+  // Required files present
+  for (const r of REQUIRED_PATHS) {
+    if (!fileSet.has(r)) {
+      issues.push(`required file missing from publish tarball: ${r}`);
+    }
+  }
+
+  // Forbidden files absent
+  for (const p of filePaths) {
+    if (PEM_ALLOWLIST.has(p)) continue;
+    for (const re of FORBIDDEN_PATTERNS) {
+      if (re.test(p)) {
+        issues.push(`forbidden file in publish tarball: ${p} (matched ${re})`);
+        break;
+      }
+    }
+  }
+
+  // Size budget
+  if (typeof packInfo.size === "number" && packInfo.size > SIZE_BUDGET_BYTES) {
+    issues.push(`tarball size ${(packInfo.size / 1024 / 1024).toFixed(2)} MB exceeds budget ${(SIZE_BUDGET_BYTES / 1024 / 1024).toFixed(0)} MB`);
+  }
+
+  if (issues.length === 0) {
+    const sizeMB = (packInfo.size / 1024 / 1024).toFixed(2);
+    const unpackedMB = (packInfo.unpackedSize / 1024 / 1024).toFixed(2);
+    process.stdout.write(
+      `[validate-package] OK — ${pkg.name}@${pkg.version}, ` +
+      `${packInfo.files.length} files, ` +
+      `${sizeMB} MB packed / ${unpackedMB} MB unpacked.\n`
+    );
+    process.exit(0);
+  }
+
+  process.stderr.write(`[validate-package] FAILED — ${issues.length} issue(s):\n`);
+  for (const i of issues) process.stderr.write(`  • ${i}\n`);
+  process.exit(1);
+}
+
+if (require.main === module) main();
+
+module.exports = { main, REQUIRED_PATHS, FORBIDDEN_PATTERNS, SIZE_BUDGET_BYTES };

package/lib/validate-vendor.js

@@ -0,0 +1,85 @@
+"use strict";
+/**
+ * lib/validate-vendor.js
+ *
+ * Predeploy gate. Confirms every file recorded in `vendor/blamejs/_PROVENANCE.json`
+ * has the same SHA-256 on disk as the manifest claims. Silent hand-edits
+ * to a vendored copy fail the build.
+ *
+ * Also confirms vendor/blamejs/LICENSE matches the recorded license hash so
+ * a license-text change is detected as a separate event from a code change.
+ *
+ * Exit codes:
+ *   0 — vendor tree in sync with provenance
+ *   1 — at least one file drift / missing
+ *
+ * Re-vendor with: copy upstream, apply strip rules, refresh hashes in
+ * _PROVENANCE.json, re-run this gate.
+ */
+
+const fs = require("fs");
+const path = require("path");
+const crypto = require("crypto");
+
+const ROOT = path.join(__dirname, "..");
+const PROV = path.join(ROOT, "vendor", "blamejs", "_PROVENANCE.json");
+
+function sha256(buf) {
+  return crypto.createHash("sha256").update(buf).digest("hex");
+}
+
+function main() {
+  if (!fs.existsSync(PROV)) {
+    console.error("[validate-vendor] vendor/blamejs/_PROVENANCE.json missing.");
+    process.exit(1);
+  }
+  const prov = JSON.parse(fs.readFileSync(PROV, "utf8"));
+  const issues = [];
+
+  // License file.
+  if (prov.license_file && prov.license_sha256) {
+    const p = path.join(ROOT, "vendor", "blamejs", prov.license_file);
+    if (!fs.existsSync(p)) {
+      issues.push(`missing license file: ${prov.license_file}`);
+    } else {
+      const live = sha256(fs.readFileSync(p));
+      if (live !== prov.license_sha256) {
+        issues.push(`LICENSE drift: recorded ${prov.license_sha256.slice(0, 12)}…, live ${live.slice(0, 12)}…`);
+      }
+    }
+  }
+
+  // Each vendored file.
+  for (const [name, info] of Object.entries(prov.files || {})) {
+    const p = path.join(ROOT, info.vendored_path);
+    if (!fs.existsSync(p)) {
+      issues.push(`missing vendored file: ${info.vendored_path}`);
+      continue;
+    }
+    const live = sha256(fs.readFileSync(p));
+    if (live !== info.vendored_sha256) {
+      issues.push(`drift in ${info.vendored_path}: recorded ${info.vendored_sha256.slice(0, 12)}…, live ${live.slice(0, 12)}…`);
+    }
+    // Smoke-check the vendored module loads.
+    try {
+      require(p);
+    } catch (err) {
+      issues.push(`load error in ${info.vendored_path}: ${err.message}`);
+    }
+  }
+
+  if (issues.length === 0) {
+    const fileCount = Object.keys(prov.files || {}).length;
+    console.log(`[validate-vendor] vendor tree current — ${fileCount} file(s) validated against pin ${prov.pinned_commit?.slice(0, 12) || "?"}.`);
+    process.exit(0);
+  }
+
+  console.error("[validate-vendor] vendor tree DRIFT:");
+  for (const i of issues) console.error("  • " + i);
+  console.error("[validate-vendor] re-vendor instructions: vendor/blamejs/README.md");
+  process.exit(1);
+}
+
+if (require.main === module) main();
+
+module.exports = { main };

package/lib/verify.js

@@ -0,0 +1,216 @@
+'use strict';
+
+/**
+ * Skill integrity verifier — Ed25519 cryptographic signatures.
+ *
+ * SHA-256 hashes alone protect against accidental corruption; anyone with repo
+ * write access can update the hash after tampering. Ed25519 signatures prove a
+ * specific keypair signed each skill. Even if the manifest is updated, a valid
+ * signature requires the private key, which never enters this repository.
+ *
+ * Signing ceremony: see lib/sign.js
+ * Public key: keys/public.pem (tracked in repo)
+ * Private key: .keys/private.pem (gitignored, kept off-repo)
+ *
+ * Usage:
+ *   node lib/verify.js           — verify all skills
+ *   node lib/verify.js <name>    — verify one skill
+ *   node lib/verify.js update    — re-sign all skills (requires private key)
+ *   node lib/verify.js check-key — verify the public key is present and valid
+ */
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+
+const ROOT = path.join(__dirname, '..');
+const MANIFEST_PATH = path.join(ROOT, 'manifest.json');
+const SKILLS_DIR = path.join(ROOT, 'skills');
+const PUBLIC_KEY_PATH = path.join(ROOT, 'keys', 'public.pem');
+const PRIVATE_KEY_PATH = path.join(ROOT, '.keys', 'private.pem');
+
+// --- public API ---
+
+/**
+ * Verify all skills in manifest.json against the Ed25519 public key.
+ * @returns {{ valid: string[], invalid: string[], missing_sig: string[], missing_file: string[], no_key: boolean }}
+ */
+function verifyAll() {
+  const publicKey = loadPublicKey();
+  if (!publicKey) {
+    console.error('[verify] No public key at keys/public.pem — run: node lib/sign.js generate-keypair');
+    return { valid: [], invalid: [], missing_sig: [], missing_file: [], no_key: true };
+  }
+
+  const manifest = loadManifest();
+  const result = { valid: [], invalid: [], missing_sig: [], missing_file: [], no_key: false };
+
+  for (const skill of manifest.skills) {
+    const outcome = verifySkill(skill, publicKey);
+    result[outcome.status].push(skill.name);
+    if (outcome.status !== 'valid') {
+      console.error(`[verify] FAIL ${skill.name}: ${outcome.reason}`);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Verify one skill by name.
+ * @param {string} skillName
+ * @returns {{ status: string, reason?: string }}
+ */
+function verifyOne(skillName) {
+  const publicKey = loadPublicKey();
+  if (!publicKey) throw new Error('No public key at keys/public.pem');
+
+  const manifest = loadManifest();
+  const skill = manifest.skills.find(s => s.name === skillName);
+  if (!skill) throw new Error(`Skill not in manifest: ${skillName}`);
+
+  return verifySkill(skill, publicKey);
+}
+
+/**
+ * Re-sign all skills using the private key and write signatures to manifest.json.
+ * Requires .keys/private.pem — never checked in.
+ * @returns {{ signed: string[], errors: string[] }}
+ */
+function signAll() {
+  const privateKey = loadPrivateKey();
+  if (!privateKey) throw new Error('No private key at .keys/private.pem — run: node lib/sign.js generate-keypair');
+
+  const manifest = loadManifest();
+  const result = { signed: [], errors: [] };
+
+  for (const skill of manifest.skills) {
+    const skillPath = path.join(ROOT, skill.path);
+    if (!fs.existsSync(skillPath)) {
+      result.errors.push(`${skill.name}: file not found at ${skill.path}`);
+      continue;
+    }
+    const content = fs.readFileSync(skillPath, 'utf8');
+    skill.signature = sign(content, privateKey);
+    skill.signed_at = new Date().toISOString();
+    delete skill.sha256;
+    result.signed.push(skill.name);
+  }
+
+  fs.writeFileSync(MANIFEST_PATH, JSON.stringify(manifest, null, 2) + '\n', 'utf8');
+  console.log(`[verify] Signed ${result.signed.length} skills with Ed25519 private key.`);
+  return result;
+}
+
+// --- private helpers ---
+
+function verifySkill(skill, publicKey) {
+  if (!skill.signature) {
+    return { status: 'missing_sig', reason: 'No Ed25519 signature in manifest — run: node lib/sign.js sign-all' };
+  }
+
+  const skillPath = path.join(ROOT, skill.path);
+  if (!fs.existsSync(skillPath)) {
+    return { status: 'missing_file', reason: `File not found: ${skill.path}` };
+  }
+
+  const content = fs.readFileSync(skillPath, 'utf8');
+  const valid = verify(content, skill.signature, publicKey);
+
+  if (!valid) {
+    return {
+      status: 'invalid',
+      reason: `Ed25519 signature verification failed — skill content has been modified since last signing`
+    };
+  }
+
+  return { status: 'valid' };
+}
+
+function sign(content, privateKey) {
+  const signature = crypto.sign(null, Buffer.from(content, 'utf8'), {
+    key: privateKey,
+    dsaEncoding: 'ieee-p1363'
+  });
+  return signature.toString('base64');
+}
+
+function verify(content, signatureBase64, publicKey) {
+  try {
+    const signature = Buffer.from(signatureBase64, 'base64');
+    return crypto.verify(null, Buffer.from(content, 'utf8'), {
+      key: publicKey,
+      dsaEncoding: 'ieee-p1363'
+    }, signature);
+  } catch (_) {
+    return false;
+  }
+}
+
+function loadPublicKey() {
+  if (!fs.existsSync(PUBLIC_KEY_PATH)) return null;
+  return fs.readFileSync(PUBLIC_KEY_PATH, 'utf8');
+}
+
+function loadPrivateKey() {
+  if (!fs.existsSync(PRIVATE_KEY_PATH)) return null;
+  return fs.readFileSync(PRIVATE_KEY_PATH, 'utf8');
+}
+
+function loadManifest() {
+  return JSON.parse(fs.readFileSync(MANIFEST_PATH, 'utf8'));
+}
+
+// --- CLI ---
+
+if (require.main === module) {
+  const arg = process.argv[2];
+
+  if (arg === 'update') {
+    const result = signAll();
+    if (result.errors.length > 0) {
+      console.error('[verify] Errors during signing:', result.errors);
+      process.exit(1);
+    }
+    console.log('[verify] All skills signed. Run node lib/verify.js to confirm.');
+    process.exit(0);
+  }
+
+  if (arg === 'check-key') {
+    const pub = loadPublicKey();
+    if (!pub) {
+      console.error('[verify] No public key — run: node lib/sign.js generate-keypair');
+      process.exit(1);
+    }
+    console.log('[verify] Public key present at keys/public.pem');
+    try {
+      crypto.createPublicKey(pub);
+      console.log('[verify] Public key is valid Ed25519.');
+      process.exit(0);
+    } catch (e) {
+      console.error('[verify] Public key is malformed:', e.message);
+      process.exit(1);
+    }
+  }
+
+  if (arg && arg !== 'verify') {
+    const outcome = verifyOne(arg);
+    console.log(`${arg}: ${outcome.status}${outcome.reason ? ' — ' + outcome.reason : ''}`);
+    process.exit(outcome.status === 'valid' ? 0 : 1);
+  }
+
+  const result = verifyAll();
+  if (result.no_key) process.exit(1);
+
+  const total = Object.values(result).filter(Array.isArray).flat().length;
+  console.log(`\n[verify] ${result.valid.length}/${total} skills passed Ed25519 verification.`);
+
+  if (result.invalid.length > 0) { console.error('[verify] TAMPERED:', result.invalid.join(', ')); process.exit(1); }
+  if (result.missing_sig.length > 0) { console.warn('[verify] UNSIGNED:', result.missing_sig.join(', ')); process.exit(1); }
+  if (result.missing_file.length > 0) { console.error('[verify] MISSING:', result.missing_file.join(', ')); process.exit(1); }
+
+  console.log('[verify] All skills verified.');
+  process.exit(0);
+}
+
+module.exports = { verifyAll, verifyOne, signAll };

package/lib/worker-pool.js

@@ -0,0 +1,84 @@
+"use strict";
+/**
+ * lib/worker-pool.js
+ *
+ * Thin convenience wrapper over the vendored blamejs worker-pool primitive.
+ * The vendored module (vendor/blamejs/worker-pool.js) provides:
+ *
+ *   - bounded concurrency, defaults to max(2, cpus)
+ *   - bounded in-memory queue (default 1024 depth)
+ *   - per-task timeout (default 5min)
+ *   - worker recycle on uncaught error / timeout / exit
+ *
+ * What this wrapper adds:
+ *
+ *   - WorkerPool class around the function-style `create()` API for
+ *     callers that prefer an instance
+ *   - runAll(tasks, opts) helper that runs an array of tasks through a
+ *     fresh pool and terminates it when done
+ *   - DEFAULT_SIZE re-export for callers that want to size manually
+ *
+ * Honest framing: at v0.7.0 corpus size (38 skills, 10 catalogs, ~150ms
+ * total build time) worker-thread spawn cost is comparable to the work
+ * itself. The pool is here so the architecture scales as the corpus
+ * grows and so users can experiment with `--parallel`. Sequential builds
+ * remain the default.
+ */
+
+const os = require("os");
+const vendored = require("../vendor/blamejs/worker-pool");
+
+const DEFAULT_SIZE = Math.max(1, Math.min(8, os.cpus()?.length || 4));
+
+class WorkerPool {
+  /**
+   * @param {object} opts
+   *   - runnerPath: absolute path to the worker script (required)
+   *   - size, maxQueueDepth, taskTimeoutMs, onExit — forwarded to vendored.create
+   */
+  constructor(opts = {}) {
+    if (!opts.runnerPath) throw new Error("WorkerPool: runnerPath is required");
+    const { runnerPath, ...rest } = opts;
+    if (rest.size === undefined) rest.size = DEFAULT_SIZE;
+    this._pool = vendored.create(runnerPath, rest);
+  }
+  run(message, transferList) {
+    return this._pool.run(message, transferList);
+  }
+  drain() {
+    return this._pool.drain();
+  }
+  terminate() {
+    return this._pool.terminate();
+  }
+  stats() {
+    return this._pool.stats();
+  }
+}
+
+/**
+ * Run a list of tasks against a fresh pool, await all results, terminate.
+ */
+async function runAll(tasks, opts = {}) {
+  const pool = new WorkerPool(opts);
+  try {
+    return await Promise.all(tasks.map((t) => pool.run(t)));
+  } finally {
+    await pool.terminate();
+  }
+}
+
+module.exports = {
+  WorkerPool,
+  runAll,
+  DEFAULT_SIZE,
+  // Re-export vendored primitives for callers that prefer the function-style API
+  // or need the size constants.
+  create:                  vendored.create,
+  MIN_SIZE:                vendored.MIN_SIZE,
+  MAX_SIZE:                vendored.MAX_SIZE,
+  DEFAULT_MAX_QUEUE_DEPTH: vendored.DEFAULT_MAX_QUEUE_DEPTH,
+  MAX_QUEUE_DEPTH_CAP:     vendored.MAX_QUEUE_DEPTH_CAP,
+  DEFAULT_TASK_TIMEOUT_MS: vendored.DEFAULT_TASK_TIMEOUT_MS,
+  MAX_TASK_TIMEOUT_MS:     vendored.MAX_TASK_TIMEOUT_MS,
+};