@bpmnkit/docspack 0.0.1

package/dist/cli.js

@@ -0,0 +1,160 @@
+#!/usr/bin/env node
+/**
+ * `bpmnkit-docs` — read the installed documentation packages and answer from them.
+ *
+ * Every command reads the filesystem only. There is no server, no network call
+ * and nothing resident between questions.
+ */
+import { readFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { buildPack } from "./build.js";
+import { discoverPacks, indexPacks } from "./load.js";
+import { answer, search } from "./search.js";
+const USAGE = `bpmnkit-docs — offline documentation search for AI agents
+
+Usage:
+  bpmnkit-docs ask <question>     Answer from the installed docs packages
+  bpmnkit-docs search <query>     Rank matching chunks, for reading in a terminal
+  bpmnkit-docs list               Show the docs packages found and their state
+  bpmnkit-docs build              Regenerate this package's .llms/ payload
+
+Options:
+  --limit <n>        Chunks to return (default 3)
+  --max-tokens <n>   Token ceiling for an answer (default 3000)
+  --pack <name>      Restrict to one documentation package
+  --cwd <dir>        Directory to resolve packages from (default: current)
+`;
+function parseArgs(argv) {
+    const options = new Map();
+    const words = [];
+    for (let i = 0; i < argv.length; i++) {
+        const arg = argv[i] ?? "";
+        if (arg.startsWith("--"))
+            options.set(arg.slice(2), argv[++i] ?? "");
+        else
+            words.push(arg);
+    }
+    return { command: words[0] ?? "", rest: words.slice(1), options };
+}
+function number(args, name, fallback) {
+    const raw = args.options.get(name);
+    if (raw === undefined)
+        return fallback;
+    const value = Number(raw);
+    if (!Number.isInteger(value) || value < 1) {
+        throw new Error(`--${name} must be a positive integer (got ${JSON.stringify(raw)})`);
+    }
+    return value;
+}
+function main(argv) {
+    const args = parseArgs(argv);
+    const cwd = resolve(args.options.get("cwd") ?? process.cwd());
+    if (args.command === "help" || args.options.has("help")) {
+        process.stdout.write(USAGE);
+        return 0;
+    }
+    if (args.command === "") {
+        process.stderr.write(USAGE);
+        return 1;
+    }
+    if (args.command === "build")
+        return runBuild(cwd);
+    const packs = discoverPacks(cwd);
+    if (packs.length === 0) {
+        process.stderr.write(`No documentation packages found from ${cwd}.\nInstall one, e.g. npm i -D @bpmnkit/docspack\n`);
+        return 1;
+    }
+    if (args.command === "list")
+        return runList(packs);
+    const query = args.rest.join(" ").trim();
+    if (query === "") {
+        process.stderr.write(`${args.command} needs a query — bpmnkit-docs ${args.command} "..."\n`);
+        return 1;
+    }
+    const selected = args.options.get("pack");
+    const scope = selected ? { packs: [selected] } : {};
+    const index = indexPacks(packs);
+    if (args.command === "search") {
+        const hits = search(index, query, { ...scope, limit: number(args, "limit", 10) });
+        process.stdout.write(formatSearch(query, hits));
+        return 0;
+    }
+    if (args.command === "ask") {
+        const result = answer(index, query, {
+            ...scope,
+            limit: number(args, "limit", 3),
+            maxTokens: number(args, "max-tokens", 3000),
+        });
+        process.stdout.write(formatAnswer(query, result.hits, result.tokens, result.maxTokens));
+        return 0;
+    }
+    process.stderr.write(`Unknown command "${args.command}".\n\n${USAGE}`);
+    return 1;
+}
+function runBuild(cwd) {
+    const pkg = JSON.parse(readFileSync(join(cwd, "package.json"), "utf8"));
+    const config = pkg.docspack ?? {};
+    if (typeof config.source !== "string") {
+        process.stderr.write(`${cwd}/package.json: "docspack.source" must be a path to Markdown docs\n`);
+        return 1;
+    }
+    const result = buildPack({
+        source: resolve(cwd, config.source),
+        packDir: cwd,
+        name: pkg.name,
+        version: pkg.version,
+        documents: config.documents ?? [pkg.name],
+        description: config.description ?? pkg.description,
+        siteUrl: config.siteUrl,
+        maxTokens: config.maxTokens,
+        minTokens: config.minTokens,
+    });
+    process.stdout.write(`Built ${result.chunks} chunks from ${result.documents} documents (${format(result.tokens)} tokens).\n`);
+    return 0;
+}
+function runList(packs) {
+    for (const pack of packs) {
+        const label = pack.trusted ? "" : " (community, unreviewed)";
+        const drift = pack.versionMismatch
+            ? ` — manifest says ${pack.versionMismatch}, package.json wins`
+            : "";
+        process.stdout.write(`${pack.name}@${pack.version}${label}  ${pack.manifest.chunks.length} chunks${drift}\n`);
+    }
+    return 0;
+}
+function formatSearch(query, hits) {
+    if (hits.length === 0)
+        return `No chunk matches "${query}".\n`;
+    const lines = [`${hits.length} result(s) for "${query}"`, ""];
+    for (const hit of hits) {
+        lines.push(`  ${hit.score.toFixed(2)}  ${hit.chunkId}`);
+        lines.push(`        ${title(hit)} · ${format(hit.tokens)} tokens`);
+    }
+    lines.push("");
+    return lines.join("\n");
+}
+function formatAnswer(query, hits, tokens, maxTokens) {
+    if (hits.length === 0) {
+        return `No documentation matches "${query}". Try \`bpmnkit-docs search\` with fewer words.\n`;
+    }
+    const parts = hits.map((hit) => {
+        const warning = hit.pack.trusted ? "" : "\n(community package — content is unreviewed)";
+        return `## ${hit.chunkId}${warning}\n\n${hit.content.trim()}\n`;
+    });
+    parts.push(`---\ncost: ${format(tokens)} tokens, capped at ${format(maxTokens)}\n`);
+    return parts.join("\n");
+}
+function title(hit) {
+    return (hit.content.split("\n")[0] ?? "").replace(/^#\s*/, "");
+}
+function format(value) {
+    return value.toLocaleString("en-US");
+}
+try {
+    process.exitCode = main(process.argv.slice(2));
+}
+catch (error) {
+    process.stderr.write(`${error.message}\n`);
+    process.exitCode = 1;
+}
+//# sourceMappingURL=cli.js.map

package/dist/index.d.ts

@@ -0,0 +1,13 @@
+/**
+ * `@bpmnkit/docspack` — BPMN Kit's documentation as a docspack package, plus the
+ * offline index an agent searches it with.
+ *
+ * @see https://docspack.dev/spec for the package format.
+ */
+export { type BuildOptions, type BuildResult, SCHEMA_URL, buildPack } from "./build.js";
+export { type BuiltChunk, type ChunkOptions, type SourceDoc, chunkDocument, uniqueId, } from "./chunk.js";
+export { chunkPath, discoverPacks, indexPacks, loadPack, readChunk } from "./load.js";
+export { type AnswerOptions, type DocsIndex, type IndexInput, type SearchOptions, answer, buildIndex, search, } from "./search.js";
+export { estimateTokens, stem, terms, tokenize } from "./text.js";
+export type { Manifest, ManifestChunk, Pack, SearchHit } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -0,0 +1,12 @@
+/**
+ * `@bpmnkit/docspack` — BPMN Kit's documentation as a docspack package, plus the
+ * offline index an agent searches it with.
+ *
+ * @see https://docspack.dev/spec for the package format.
+ */
+export { SCHEMA_URL, buildPack } from "./build.js";
+export { chunkDocument, uniqueId, } from "./chunk.js";
+export { chunkPath, discoverPacks, indexPacks, loadPack, readChunk } from "./load.js";
+export { answer, buildIndex, search, } from "./search.js";
+export { estimateTokens, stem, terms, tokenize } from "./text.js";
+//# sourceMappingURL=index.js.map

package/dist/load.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Finding and reading documentation packages on disk.
+ *
+ * A manifest is third-party input that ends up in a model's context, so it is
+ * validated as a security boundary: a chunk path that leaves `.llms/` is refused,
+ * and the installed `package.json` version supersedes whatever the manifest claims.
+ */
+import { type DocsIndex } from "./search.js";
+import type { ManifestChunk, Pack } from "./types.js";
+/** Read one documentation package from its root directory. */
+export declare function loadPack(dir: string): Pack;
+/**
+ * Resolve a chunk's file inside `.llms/`, refusing any path that escapes it.
+ * The manifest is untrusted, so this is the only way a chunk path is turned
+ * into a real one.
+ */
+export declare function chunkPath(packDir: string, chunk: ManifestChunk): string;
+export declare function readChunk(pack: Pack, chunk: ManifestChunk): string;
+/**
+ * Every documentation package reachable from `cwd`: the vendor-scoped
+ * `@<vendor>/docspack` and community `@docspack-community/*` packages installed
+ * in any `node_modules` up the tree, plus `cwd` itself when it is a pack.
+ */
+export declare function discoverPacks(cwd?: string): Pack[];
+/** Load every chunk of every pack into a searchable index. */
+export declare function indexPacks(packs: readonly Pack[]): DocsIndex;
+//# sourceMappingURL=load.d.ts.map

package/dist/load.js

@@ -0,0 +1,144 @@
+/**
+ * Finding and reading documentation packages on disk.
+ *
+ * A manifest is third-party input that ends up in a model's context, so it is
+ * validated as a security boundary: a chunk path that leaves `.llms/` is refused,
+ * and the installed `package.json` version supersedes whatever the manifest claims.
+ */
+import { readFileSync, readdirSync, statSync } from "node:fs";
+import { dirname, join, relative, resolve, sep } from "node:path";
+import { buildIndex } from "./search.js";
+import { estimateTokens } from "./text.js";
+const CHUNK_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+const COMMUNITY_SCOPE = "@docspack-community";
+/** Read one documentation package from its root directory. */
+export function loadPack(dir) {
+    const root = resolve(dir);
+    const pkg = readJson(join(root, "package.json"), "package.json");
+    const manifest = readJson(join(root, ".llms", "manifest.json"), "manifest");
+    if (typeof pkg.name !== "string" || typeof pkg.version !== "string") {
+        throw new Error(`${root}: package.json needs a "name" and a "version"`);
+    }
+    if (!Array.isArray(manifest.chunks)) {
+        throw new Error(`${root}: .llms/manifest.json needs a "chunks" array`);
+    }
+    const seen = new Set();
+    for (const chunk of manifest.chunks) {
+        if (!CHUNK_ID.test(chunk.id ?? "")) {
+            throw new Error(`${pkg.name}: chunk id ${JSON.stringify(chunk.id)} is not a valid id`);
+        }
+        if (seen.has(chunk.id))
+            throw new Error(`${pkg.name}: duplicate chunk id "${chunk.id}"`);
+        seen.add(chunk.id);
+        if (chunk.tokens !== undefined && (!Number.isInteger(chunk.tokens) || chunk.tokens < 1)) {
+            throw new Error(`${pkg.name}/${chunk.id}: "tokens" must be an integer of at least 1`);
+        }
+        chunkPath(root, chunk);
+    }
+    return {
+        name: pkg.name,
+        version: pkg.version,
+        dir: root,
+        trusted: !pkg.name.startsWith(`${COMMUNITY_SCOPE}/`),
+        manifest,
+        ...(manifest.version && manifest.version !== pkg.version
+            ? { versionMismatch: manifest.version }
+            : {}),
+    };
+}
+/**
+ * Resolve a chunk's file inside `.llms/`, refusing any path that escapes it.
+ * The manifest is untrusted, so this is the only way a chunk path is turned
+ * into a real one.
+ */
+export function chunkPath(packDir, chunk) {
+    const base = resolve(packDir, ".llms");
+    const target = resolve(base, chunk.file ?? "");
+    const inside = relative(base, target);
+    if (inside === "" || inside.startsWith("..") || inside.startsWith(`..${sep}`)) {
+        throw new Error(`${chunk.id}: chunk file "${chunk.file}" escapes .llms/`);
+    }
+    return target;
+}
+export function readChunk(pack, chunk) {
+    return readFileSync(chunkPath(pack.dir, chunk), "utf8");
+}
+/**
+ * Every documentation package reachable from `cwd`: the vendor-scoped
+ * `@<vendor>/docspack` and community `@docspack-community/*` packages installed
+ * in any `node_modules` up the tree, plus `cwd` itself when it is a pack.
+ */
+export function discoverPacks(cwd = process.cwd()) {
+    const found = new Map();
+    const offer = (dir) => {
+        try {
+            const pack = loadPack(dir);
+            if (!found.has(pack.name))
+                found.set(pack.name, pack);
+        }
+        catch {
+            // Not a documentation package, or one we must not read. Skip it.
+        }
+    };
+    offer(cwd);
+    for (const modules of nodeModulesChain(resolve(cwd))) {
+        for (const scope of listDirectory(modules)) {
+            if (!scope.startsWith("@"))
+                continue;
+            if (scope === COMMUNITY_SCOPE) {
+                for (const name of listDirectory(join(modules, scope)))
+                    offer(join(modules, scope, name));
+            }
+            else {
+                offer(join(modules, scope, "docspack"));
+            }
+        }
+    }
+    return [...found.values()].sort((a, b) => a.name.localeCompare(b.name));
+}
+/** Load every chunk of every pack into a searchable index. */
+export function indexPacks(packs) {
+    const inputs = [];
+    for (const pack of packs) {
+        for (const chunk of pack.manifest.chunks) {
+            const content = readChunk(pack, chunk);
+            inputs.push({ pack, chunk, content, tokens: chunk.tokens ?? estimateTokens(content) });
+        }
+    }
+    return buildIndex(inputs);
+}
+function nodeModulesChain(from) {
+    const dirs = [];
+    let current = from;
+    for (;;) {
+        dirs.push(join(current, "node_modules"));
+        const parent = dirname(current);
+        if (parent === current)
+            return dirs;
+        current = parent;
+    }
+}
+function listDirectory(dir) {
+    try {
+        return readdirSync(dir).filter((name) => statSync(join(dir, name)).isDirectory());
+    }
+    catch {
+        return [];
+    }
+}
+function readJson(path, label) {
+    let raw;
+    try {
+        raw = readFileSync(path, "utf8");
+    }
+    catch {
+        throw new Error(`${path}: ${label} not found`);
+    }
+    try {
+        return JSON.parse(raw);
+    }
+    catch (error) {
+        throw new Error(`${path}: ${label} is not valid JSON — ${error.message}`);
+    }
+}
+//# sourceMappingURL=load.js.map

package/dist/search.d.ts

@@ -0,0 +1,53 @@
+/**
+ * The retrieval side: a BM25 index over chunk text, with tags and entities
+ * weighted above prose. Built in memory from the manifests — the corpus is a
+ * project's own documentation, not the whole registry, so there is nothing to
+ * persist and no database to keep in step.
+ */
+import type { ManifestChunk, Pack, SearchHit } from "./types.js";
+interface IndexedChunk {
+    chunkId: string;
+    pack: Pack;
+    chunk: ManifestChunk;
+    tokens: number;
+    content: string;
+    frequencies: Map<string, number>;
+    length: number;
+}
+export interface DocsIndex {
+    chunks: IndexedChunk[];
+    documentFrequency: Map<string, number>;
+    averageLength: number;
+}
+/** One chunk's content, paired with the pack it came from. */
+export interface IndexInput {
+    pack: Pack;
+    chunk: ManifestChunk;
+    content: string;
+    tokens: number;
+}
+export declare function buildIndex(inputs: IndexInput[]): DocsIndex;
+export interface SearchOptions {
+    /** Maximum hits to return. */
+    limit?: number;
+    /** Restrict to these package names — a project asks only about what it installed. */
+    packs?: readonly string[];
+}
+/** Rank every chunk that shares a term with the query. Ties break on chunk id. */
+export declare function search(index: DocsIndex, query: string, options?: SearchOptions): SearchHit[];
+export interface AnswerOptions extends SearchOptions {
+    /** Hard ceiling on the tokens an answer may spend, counted before content. */
+    maxTokens?: number;
+}
+/**
+ * The hits an agent gets back: the top matches that fit the token budget.
+ * The budget is spent from the manifest counts, so a chunk that would overrun
+ * it is dropped rather than truncated mid-sentence.
+ */
+export declare function answer(index: DocsIndex, query: string, options?: AnswerOptions): {
+    hits: SearchHit[];
+    tokens: number;
+    maxTokens: number;
+};
+export {};
+//# sourceMappingURL=search.d.ts.map

package/dist/search.js

@@ -0,0 +1,104 @@
+/**
+ * The retrieval side: a BM25 index over chunk text, with tags and entities
+ * weighted above prose. Built in memory from the manifests — the corpus is a
+ * project's own documentation, not the whole registry, so there is nothing to
+ * persist and no database to keep in step.
+ */
+import { terms } from "./text.js";
+/** BM25 saturation and length-normalisation, at their conventional values. */
+const K1 = 1.2;
+const B = 0.75;
+/** A tag is a deliberate index term; a word in a paragraph is incidental. */
+const TAG_WEIGHT = 3;
+const ENTITY_WEIGHT = 3;
+export function buildIndex(inputs) {
+    const chunks = [];
+    const documentFrequency = new Map();
+    for (const input of inputs) {
+        const frequencies = new Map();
+        const add = (values, weight) => {
+            for (const term of values)
+                frequencies.set(term, (frequencies.get(term) ?? 0) + weight);
+        };
+        const contentTerms = terms(input.content);
+        add(contentTerms, 1);
+        add(terms((input.chunk.tags ?? []).join(" ")), TAG_WEIGHT);
+        add(terms((input.chunk.entities ?? []).join(" ")), ENTITY_WEIGHT);
+        for (const term of frequencies.keys()) {
+            documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1);
+        }
+        chunks.push({
+            chunkId: `${input.pack.name}@${input.pack.version}/${input.chunk.id}`,
+            pack: input.pack,
+            chunk: input.chunk,
+            tokens: input.tokens,
+            content: input.content,
+            frequencies,
+            length: contentTerms.length,
+        });
+    }
+    const total = chunks.reduce((sum, c) => sum + c.length, 0);
+    return {
+        chunks,
+        documentFrequency,
+        averageLength: chunks.length === 0 ? 1 : Math.max(1, total / chunks.length),
+    };
+}
+/** Rank every chunk that shares a term with the query. Ties break on chunk id. */
+export function search(index, query, options = {}) {
+    const limit = options.limit ?? 3;
+    const queryTerms = terms(query);
+    if (queryTerms.length === 0)
+        return [];
+    const allowed = options.packs ? new Set(options.packs) : null;
+    const total = index.chunks.length;
+    const hits = [];
+    for (const candidate of index.chunks) {
+        if (allowed && !allowed.has(candidate.pack.name))
+            continue;
+        let score = 0;
+        for (const term of new Set(queryTerms)) {
+            const frequency = candidate.frequencies.get(term);
+            if (!frequency)
+                continue;
+            const documents = index.documentFrequency.get(term) ?? 0;
+            const idf = Math.log(1 + (total - documents + 0.5) / (documents + 0.5));
+            const norm = K1 * (1 - B + (B * candidate.length) / index.averageLength);
+            score += idf * ((frequency * (K1 + 1)) / (frequency + norm));
+        }
+        if (score <= 0)
+            continue;
+        hits.push({
+            chunkId: candidate.chunkId,
+            pack: candidate.pack,
+            chunk: candidate.chunk,
+            tokens: candidate.tokens,
+            content: candidate.content,
+            score,
+        });
+    }
+    hits.sort((a, b) => b.score - a.score || a.chunkId.localeCompare(b.chunkId));
+    return hits.slice(0, limit);
+}
+/**
+ * The hits an agent gets back: the top matches that fit the token budget.
+ * The budget is spent from the manifest counts, so a chunk that would overrun
+ * it is dropped rather than truncated mid-sentence.
+ */
+export function answer(index, query, options = {}) {
+    const maxTokens = options.maxTokens ?? 3000;
+    const limit = options.limit ?? 3;
+    const ranked = search(index, query, { ...options, limit: Math.max(limit * 4, limit) });
+    const selected = [];
+    let spent = 0;
+    for (const hit of ranked) {
+        if (selected.length >= limit)
+            break;
+        if (spent + hit.tokens > maxTokens)
+            continue;
+        selected.push(hit);
+        spent += hit.tokens;
+    }
+    return { hits: selected, tokens: spent, maxTokens };
+}
+//# sourceMappingURL=search.js.map

package/dist/text.d.ts

@@ -0,0 +1,22 @@
+/** Tokenising, stemming and token counting — shared by the builder and the index. */
+/**
+ * The Porter (1980) stemmer, so a query for `authenticate` reaches a chunk that
+ * only ever says `authentication`. Both reduce to `authent`.
+ */
+export declare function stem(input: string): string;
+/**
+ * Words a query and a chunk share no meaning through: indexing them costs space
+ * and matching on them ranks every chunk equally.
+ */
+export declare const STOP_WORDS: ReadonlySet<string>;
+/** Split text into lowercase word tokens, keeping the dots inside `a.b` identifiers. */
+export declare function tokenize(text: string): string[];
+/** Tokenize, drop stop words, stem — the form both the index and a query use. */
+export declare function terms(text: string): string[];
+/**
+ * Approximate the token count of a string. Deterministic and close enough to
+ * budget an answer against — docspack counts from the manifest before it reads
+ * any content, so this must not need the content to be tokenised twice.
+ */
+export declare function estimateTokens(text: string): number;
+//# sourceMappingURL=text.d.ts.map