@rckflr/llms-skills 0.1.0 → 0.2.0

package/README.md

@@ -47,6 +47,33 @@ bridge out is the host capabilities the runtime injects (`host.fetchOrigin`,
 scoped to your origin). `publish` computes `tool_sha256` and carries `tool` +
 `tool_sha256` in both the inline metadata and `index.json`.
 
+## Knowledge / serverless RAG (`memory`)
+
+Turn a directory of [OKF](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md)
+concepts (`*.md` with `type`/`title` frontmatter; `index.md`/`log.md` reserved)
+into a serverless RAG any consumer can query — no vector DB, no server:
+
+```bash
+npx @rckflr/llms-skills memory ./knowledge   # snapshot + 3 knowledge skills + manifest wiring
+npx @rckflr/llms-skills publish              # as always
+```
+
+What it generates: a **byte-deterministic** BM25 snapshot (canonicalized, so
+`--check`/CI content-addressing works), pinned by `snapshot_sha256` in the
+`skills-memory` line, plus three executable skills — `search_knowledge`
+(a **universal template**: identical bytes for every publisher, so its
+`tool_sha256` is a stable ecosystem-wide constant — audit once, attest once),
+`get_concept`, and `list_concepts` (concept metadata embedded at build time,
+content-addressed). It also pins the snapshot as `-text` in `.gitattributes`
+so git's CRLF conversion can never break the published hash.
+
+Consumers need nothing new: `npx -y @rckflr/mcpwasm <origin>` (>= 0.4.0)
+verifies the snapshot and injects `host.memorySearch` on both runtimes.
+`memory <bundle> --check` is the CI guard. The BM25 engine
+(`@rckflr/minimemory`, ~630 KB wasm) is an optionalDependency — installed by
+default; if omitted, `memory` fails with a clear message and every other
+command works as before.
+
 ## Signing / attestation (L3)
 
 ```bash

package/bin/llms-skills.mjs

@@ -16,6 +16,7 @@ import {
   loadSkills, loadMemory, renderSkillsSection, renderSkillsMemoryLine, renderLlmsTxt,
   renderIndex, loadSigningKey, signSkills, publicKeyRawB64, generateKeyPair, validateLlmsTxt,
 } from "../lib/core.mjs";
+import { buildMemoryTargets } from "../lib/memory.mjs";
 
 const args = process.argv.slice(2);
 const cmd = args[0];
@@ -28,12 +29,17 @@ const USAGE = `llms-skills — publisher CLI for the llms.txt Skills standard
 
 Usage:
   llms-skills init <name> [--tool] [--root <dir>]   scaffold SKILL.md (+ tool.js) + manifest entry
+  llms-skills memory <bundle-dir> [--check] [--root <dir>] [--license <spdx>]
+                                                    OKF bundle -> BM25 snapshot + knowledge skills (RAG)
   llms-skills publish [--check] [--manifest <p>] [--root <dir>]   generate llms.txt ## Skills + index.json
   llms-skills validate <src> [--strict]             validate an llms.txt (path or URL)
   llms-skills keygen [--out <keyfile>]              generate an ed25519 signing keypair
 
 The minimal case (L0) needs no signing key and no tool.js — just init + publish.
-Add --tool for an executable skill; add a "signing" block to the manifest for L3 attestation.`;
+Add --tool for an executable skill; add a "signing" block to the manifest for L3 attestation.
+\`memory\` turns a directory of OKF concepts (*.md with type/title frontmatter) into a
+serverless RAG: a hash-pinned snapshot + search_knowledge/get_concept/list_concepts skills.
+Run \`publish\` afterwards, as always.`;
 
 function findManifest(root) {
   const cands = [flag("--manifest"), "llms-skills.json", "scripts/skills-manifest.json"].filter(Boolean);
@@ -105,6 +111,48 @@ registerTool({
   console.log(`\nNext: fill in the SKILL.md${withTool ? " and tool.js" : ""}, then run  llms-skills publish`);
 }
 
+// ---- memory (RAG-OKF builder) ----
+async function cmdMemory() {
+  const bundleArg = args[1];
+  if (!bundleArg || bundleArg.startsWith("--")) die("memory: needs a bundle directory, e.g. `llms-skills memory ./knowledge`");
+  const root = flag("--root") || process.cwd();
+  const bundleDir = join(root, bundleArg) === bundleArg ? bundleArg : join(root, bundleArg);
+  if (!existsSync(bundleDir)) die(`memory: bundle directory not found: ${bundleDir}`);
+
+  const manifestPath = findManifest(root) || join(root, "llms-skills.json");
+  const manifest = existsSync(manifestPath)
+    ? JSON.parse(readFileSync(manifestPath, "utf8"))
+    : { section_intro: "Remote Agent Skills published by this domain.", published: [] };
+
+  const { targets, warnings, manifest: newManifest, snapshotSha, conceptCount } =
+    await buildMemoryTargets({ root, bundleDir, manifest, license: flag("--license") });
+  for (const w of warnings) console.error(`[warn] ${w}`);
+
+  const rel = (p) => (relative(root, p) || p).replaceAll("\\", "/");
+  const norm = (s) => s.replaceAll("\r\n", "\n");
+  const manifestContent = JSON.stringify(newManifest, null, 2) + "\n";
+  const all = [...targets, [manifestPath, manifestContent]];
+
+  if (has("--check")) {
+    const drift = all.filter(([p, c]) => (existsSync(p) ? norm(readFileSync(p, "utf8")) : null) !== norm(c));
+    if (drift.length) {
+      console.error("[DRIFT] Out-of-date memory artifacts. Run: llms-skills memory " + bundleArg);
+      for (const [p] of drift) console.error(`  - ${rel(p)}`);
+      process.exit(1);
+    }
+    console.log(`[OK] Memory in sync (${conceptCount} concepts, snapshot ${snapshotSha.slice(0, 12)}…).`);
+    return;
+  }
+
+  for (const [p, c] of all) {
+    mkdirSync(dirname(p), { recursive: true });
+    writeFileSync(p, c, "utf8");
+    console.log(`[WRITE] ${rel(p)}`);
+  }
+  console.log(`\n${conceptCount} concept(s) indexed; snapshot sha256 ${snapshotSha.slice(0, 12)}…`);
+  console.log("Next: run  llms-skills publish  to emit the skills-memory line and hash the tools.");
+}
+
 // ---- publish ----
 function cmdPublish() {
   const root = flag("--root") || process.cwd();
@@ -193,6 +241,7 @@ function cmdKeygen() {
   try {
     switch (cmd) {
       case "init": cmdInit(); break;
+      case "memory": await cmdMemory(); break;
       case "publish": cmdPublish(); break;
       case "validate": await cmdValidate(); break;
       case "keygen": cmdKeygen(); break;

package/lib/memory.mjs

@@ -0,0 +1,260 @@
+// memory.mjs — RAG-OKF builder behind `llms-skills memory`.
+//
+// Turns an OKF bundle (https://github.com/GoogleCloudPlatform/knowledge-catalog
+// okf/SPEC.md v0.1: *.md concepts with `type` frontmatter; index.md/log.md
+// reserved) into the origin-memory artifacts the Executable Skills extension
+// v0.4 §2.4 defines and the mcpwasm runtimes already consume:
+//
+//   - a BM25 snapshot (@rckflr/minimemory, format minimemory-okf-v1),
+//     CANONICALIZED for byte-determinism (see canonicalizeSnapshot),
+//   - three generated knowledge skills (search_knowledge / get_concept /
+//     list_concepts) as tool.js + SKILL.md,
+//   - manifest wiring (`memory` block + published entries) so a plain
+//     `llms-skills publish` emits the skills-memory line and all hashes.
+//
+// The engine is an optionalDependency: lazily imported; a clear error (not a
+// crash) if missing. The consumer side needs nothing new — mcpwasm >= 0.4.0
+// verifies snapshot_sha256 and injects host.memorySearch on both runtimes.
+
+import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, statSync } from "node:fs";
+import { join, relative, basename } from "node:path";
+import { createRequire } from "node:module";
+import { parseFrontmatter, sha256NormalizedBuf } from "./core.mjs";
+
+export const SNAPSHOT_FORMAT = "minimemory-okf-v1";
+export const RESERVED_FILES = new Set(["index.md", "log.md"]); // OKF 0.1 reserved names
+const CHUNK_SIZE = 800; // validated values (same as mcpwasm's docs-site publisher)
+const CHUNK_OVERLAP = 50;
+const MAX_SNAPSHOT_BYTES = 4194304; // 4 MB — the cap both mcpwasm runtimes enforce
+const MAX_EMBED_LIST_BYTES = 262144; // embedded concept list must stay well under the 1 MB tool.js cap
+
+// ---- engine (optionalDependency, lazy) --------------------------------------
+export async function loadMemoryEngine() {
+  try {
+    const require = createRequire(import.meta.url);
+    const mod = await import("@rckflr/minimemory");
+    mod.initSync({ module: readFileSync(require.resolve("@rckflr/minimemory/minimemory_bg.wasm")) });
+    return mod;
+  } catch (e) {
+    const err = new Error(
+      "memory: the BM25 engine (@rckflr/minimemory) is not installed. It is an " +
+      "optionalDependency — `npm install @rckflr/minimemory` (or reinstall without " +
+      "--omit=optional) and retry. Underlying error: " + (e && e.message ? e.message : String(e))
+    );
+    err.code = "ENGINE_MISSING";
+    throw err;
+  }
+}
+
+// ---- bundle scan -------------------------------------------------------------
+// Walks the bundle, returns concepts sorted by id (determinism) plus warnings.
+// concept id = bundle-relative posix path (the okf-integration convention).
+export function scanBundle(bundleDir) {
+  const concepts = [];
+  const warnings = [];
+  const walk = (dir) => {
+    for (const name of readdirSync(dir).sort()) {
+      const p = join(dir, name);
+      if (statSync(p).isDirectory()) { walk(p); continue; }
+      if (!name.endsWith(".md") || RESERVED_FILES.has(name)) continue;
+      const id = relative(bundleDir, p).replaceAll("\\", "/");
+      const raw = readFileSync(p, "utf8");
+      const fm = parseFrontmatter(raw);
+      let type = fm.type;
+      if (!type) {
+        warnings.push(`${id}: frontmatter missing 'type' (OKF 0.1 requires it) — defaulting to "Documentation"`);
+        type = "Documentation";
+      }
+      // body = content after the frontmatter block (or the whole file if none)
+      let body = raw;
+      if (raw.startsWith("---")) {
+        const end = raw.indexOf("---", 3);
+        if (end !== -1) body = raw.slice(end + 3);
+      }
+      body = body.replace(/^\s+/, "");
+      const h1 = body.match(/^#\s+(.+)$/m);
+      const title = fm.title || (h1 ? h1[1].trim() : basename(id, ".md"));
+      const description = fm.description || "";
+      if (!body.trim()) warnings.push(`${id}: empty body — concept will not be searchable`);
+      concepts.push({ id, type, title, description, body });
+    }
+  };
+  walk(bundleDir);
+  concepts.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+  return { concepts, warnings };
+}
+
+// ---- snapshot ----------------------------------------------------------------
+// minimemory's export_snapshot() iterates a Rust HashMap: chunk ORDER varies
+// between builds (verified empirically: identical content, shuffled array).
+// The snapshot is a plain JSON array with already-sorted object keys, so
+// sorting by chunk id and re-serializing compact yields a byte-deterministic,
+// import-compatible snapshot. This canonicalization is REQUIRED for the
+// content-addressing story (`--check`, CI, sha-pinning) to work at all.
+export function canonicalizeSnapshot(snapshot) {
+  const arr = JSON.parse(snapshot);
+  if (!Array.isArray(arr)) throw new Error("memory: unexpected snapshot shape (not a JSON array)");
+  arr.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+  return JSON.stringify(arr);
+}
+
+export function buildSnapshot(mem, concepts) {
+  const idx = mem.WasmOkfIndex.with_chunk_size(CHUNK_SIZE, CHUNK_OVERLAP);
+  for (const c of concepts) {
+    const content = `---\ntype: ${c.type}\ntitle: ${c.title}\n---\n${c.body}`;
+    idx.ingest_concept(c.id, content);
+  }
+  const snapshot = canonicalizeSnapshot(idx.export_snapshot());
+  if (Buffer.byteLength(snapshot, "utf8") > MAX_SNAPSHOT_BYTES) {
+    throw new Error(
+      `memory: snapshot exceeds the 4 MB cap the runtimes enforce (${Buffer.byteLength(snapshot, "utf8")} bytes). ` +
+      "Split the bundle or trim concept bodies."
+    );
+  }
+  return snapshot;
+}
+
+// ---- generated skills ---------------------------------------------------------
+// search_knowledge is a UNIVERSAL template: identical bytes for every
+// publisher, so its tool_sha256 is stable ecosystem-wide — audit once,
+// attest once. Do not interpolate anything publisher-specific into it.
+export function searchToolSource() {
+  return `// search_knowledge — universal origin-memory search tool (llms-skills memory).
+// Identical for every publisher: its sha256 is a stable, ecosystem-wide constant.
+registerTool({
+  name: "search_knowledge",
+  description: "BM25 search over this origin's published knowledge (OKF bundle). Returns the most relevant chunks with concept ids.",
+  inputSchema: { type: "object", properties: { q: { type: "string", description: "search query" }, k: { type: "number", description: "max results (1-10, default 5)" } }, required: ["q"] },
+  handler: async function (args) {
+    return await host.memorySearch(args.q, typeof args.k === "number" ? args.k : 5);
+  }
+});
+`;
+}
+
+export function getConceptToolSource(conceptUrlById) {
+  return `// get_concept — fetch a full concept document from this origin's OKF bundle.
+// The id->url map is embedded at build time (content-addressed via tool_sha256).
+var CONCEPTS = ${JSON.stringify(conceptUrlById)};
+registerTool({
+  name: "get_concept",
+  description: "Fetch the full markdown of a knowledge concept by id (as returned by search_knowledge / list_concepts).",
+  inputSchema: { type: "object", properties: { id: { type: "string", description: "concept id, e.g. policies/refunds.md" } }, required: ["id"] },
+  handler: async function (args) {
+    var url = CONCEPTS[args.id];
+    if (!url) throw new Error("unknown concept id: " + args.id);
+    var r = await host.fetchOrigin(url);
+    if (r.status !== 200) throw new Error("fetch failed: HTTP " + r.status);
+    return { id: args.id, markdown: r.body };
+  }
+});
+`;
+}
+
+export function listConceptsToolSource(conceptMeta) {
+  const embedded = JSON.stringify(conceptMeta);
+  if (Buffer.byteLength(embedded, "utf8") > MAX_EMBED_LIST_BYTES) {
+    throw new Error(
+      "memory: the concept list is too large to embed in list_concepts/tool.js (" +
+      Buffer.byteLength(embedded, "utf8") + " bytes > " + MAX_EMBED_LIST_BYTES + "). " +
+      "Reduce concepts or descriptions."
+    );
+  }
+  return `// list_concepts — enumerate this origin's published knowledge concepts.
+// The list is embedded at build time (content-addressed via tool_sha256).
+var CONCEPTS = ${embedded};
+registerTool({
+  name: "list_concepts",
+  description: "List all knowledge concepts published by this origin (id, type, title, description).",
+  inputSchema: { type: "object", properties: {} },
+  handler: function () { return { concepts: CONCEPTS }; }
+});
+`;
+}
+
+export function knowledgeSkillMd(name, description, license) {
+  return `---
+name: ${name}
+description: ${description}
+version: 0.1.0
+license: ${license}
+---
+
+# ${name}
+
+${description} Generated by \`llms-skills memory\` from this origin's OKF
+knowledge bundle; backed by a hash-verified BM25 snapshot (origin memory,
+Executable Skills v0.4 §2.4). Call the tool rather than improvising: the
+runtime executes it verbatim in a sandbox.
+`;
+}
+
+// ---- orchestrator --------------------------------------------------------------
+// Builds everything under `root`. Returns { targets, warnings, snapshotSha, manifest }
+// where targets is [path, content] pairs (the caller writes or checks them).
+export async function buildMemoryTargets({ root, bundleDir, manifest, license }) {
+  const bundleRel = relative(root, bundleDir).replaceAll("\\", "/");
+  if (bundleRel.startsWith("..")) {
+    throw new Error("memory: the bundle must live inside the publisher root (it is served as static content)");
+  }
+  const { concepts, warnings } = scanBundle(bundleDir);
+  if (concepts.length === 0) throw new Error(`memory: no OKF concepts (*.md) found in ${bundleDir}`);
+
+  const mem = await loadMemoryEngine();
+  const snapshot = buildSnapshot(mem, concepts);
+
+  const urlById = {};
+  for (const c of concepts) urlById[c.id] = "/" + (bundleRel === "" ? "" : bundleRel + "/") + c.id;
+  const conceptMeta = concepts.map((c) => ({ id: c.id, type: c.type, title: c.title, description: c.description }));
+
+  const lic = license || manifest.license || "MIT";
+  const SKILLS = [
+    ["search_knowledge", searchToolSource(), "BM25 search over this origin's published knowledge bundle. Use it to find which concept answers a question before fetching it."],
+    ["get_concept", getConceptToolSource(urlById), "fetch the full markdown of a knowledge concept by id from this origin's bundle."],
+    ["list_concepts", listConceptsToolSource(conceptMeta), "list every knowledge concept this origin publishes (id, type, title, description)."],
+  ];
+
+  const targets = [[join(root, "skills-index.snapshot"), snapshot]];
+  for (const [name, toolSrc, summary] of SKILLS) {
+    targets.push([join(root, "skills", name, "tool.js"), toolSrc]);
+    targets.push([join(root, "skills", name, "SKILL.md"), knowledgeSkillMd(name, summary.charAt(0).toUpperCase() + summary.slice(1), lic)]);
+  }
+
+  // .gitattributes: git autocrlf mangling the snapshot on checkout would break
+  // the published sha (real-world footgun) — pin it as -text.
+  const gaPath = join(root, ".gitattributes");
+  const gaLine = "skills-index.snapshot -text";
+  const gaCurrent = existsSync(gaPath) ? readFileSync(gaPath, "utf8") : "";
+  if (!gaCurrent.split(/\r?\n/).some((l) => l.trim() === gaLine)) {
+    targets.push([gaPath, (gaCurrent ? gaCurrent.replace(/\n?$/, "\n") : "") + gaLine + "\n"]);
+  }
+
+  // manifest wiring (idempotent): memory block + one published entry per skill.
+  const newManifest = JSON.parse(JSON.stringify(manifest));
+  newManifest.memory = {
+    snapshot_path: "skills-index.snapshot",
+    snapshot_url: "/skills-index.snapshot",
+    format: SNAPSHOT_FORMAT,
+  };
+  if (!Array.isArray(newManifest.published)) newManifest.published = [];
+  for (const [name, , summary] of SKILLS) {
+    const path = `skills/${name}/SKILL.md`;
+    if (!newManifest.published.some((e) => e && e.path === path)) {
+      newManifest.published.push({
+        path,
+        url: `/skills/${name}/SKILL.md`,
+        summary,
+        tool: `skills/${name}/tool.js`,
+        tool_url: `/skills/${name}/tool.js`,
+      });
+    }
+  }
+
+  return {
+    targets,
+    warnings,
+    manifest: newManifest,
+    snapshotSha: sha256NormalizedBuf(Buffer.from(snapshot, "utf8")),
+    conceptCount: concepts.length,
+  };
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@rckflr/llms-skills",
-  "version": "0.1.0",
-  "description": "One-command publisher CLI for the llms.txt Skills standard: scaffold, hash, sign, and sync the ## Skills section + .well-known/agent-skills/index.json. Byte-identical to the reference Python generator; zero dependencies.",
+  "version": "0.2.0",
+  "description": "One-command publisher CLI for the llms.txt Skills standard: scaffold, hash, sign, and sync the ## Skills section + .well-known/agent-skills/index.json. Byte-identical to the reference Python generator; zero hard dependencies (BM25 knowledge snapshots via an optional wasm engine).",
   "type": "module",
   "license": "MIT",
   "author": "Mauricio Perera <mauricio.perera@gmail.com>",
@@ -14,7 +14,16 @@
   "bugs": {
     "url": "https://github.com/MauricioPerera/llms-txt-skills/issues"
   },
-  "keywords": ["llms.txt", "agent-skills", "skills", "mcp", "publisher", "cli", "ed25519", "sha256"],
+  "keywords": [
+    "llms.txt",
+    "agent-skills",
+    "skills",
+    "mcp",
+    "publisher",
+    "cli",
+    "ed25519",
+    "sha256"
+  ],
   "engines": {
     "node": ">=18"
   },
@@ -22,11 +31,13 @@
     "llms-skills": "bin/llms-skills.mjs"
   },
   "exports": {
-    ".": "./lib/core.mjs"
+    ".": "./lib/core.mjs",
+    "./memory": "./lib/memory.mjs"
   },
   "files": [
     "bin/llms-skills.mjs",
     "lib/core.mjs",
+    "lib/memory.mjs",
     "README.md"
   ],
   "publishConfig": {
@@ -34,5 +45,8 @@
   },
   "scripts": {
     "test": "node test.mjs"
+  },
+  "optionalDependencies": {
+    "@rckflr/minimemory": "^3.2.0"
   }
 }