@msareen/knowledge-hub-builder 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/.agents/skills/catalog/SKILL.md +7 -0
  2. package/.agents/skills/export/SKILL.md +7 -0
  3. package/.agents/skills/ingest/SKILL.md +7 -0
  4. package/.agents/skills/lint/SKILL.md +7 -0
  5. package/.agents/skills/new-bundle/SKILL.md +7 -0
  6. package/.agents/skills/query/SKILL.md +7 -0
  7. package/.agents/skills/visualize/SKILL.md +7 -0
  8. package/.bundle_template/index.md +9 -0
  9. package/.bundle_template/log.md +10 -0
  10. package/.bundle_template/raw/.gitkeep +15 -0
  11. package/.bundle_template/refs.md +6 -0
  12. package/.bundle_template/sources.yaml +13 -0
  13. package/.claude/skills/catalog/SKILL.md +7 -0
  14. package/.claude/skills/export/SKILL.md +7 -0
  15. package/.claude/skills/ingest/SKILL.md +7 -0
  16. package/.claude/skills/lint/SKILL.md +7 -0
  17. package/.claude/skills/new-bundle/SKILL.md +7 -0
  18. package/.claude/skills/query/SKILL.md +7 -0
  19. package/.claude/skills/visualize/SKILL.md +7 -0
  20. package/AGENTS.md +167 -0
  21. package/CLAUDE.md +13 -0
  22. package/README.md +289 -0
  23. package/SPEC.md +354 -0
  24. package/document/faq.md +156 -0
  25. package/package.json +52 -0
  26. package/scripts/cli.ts +66 -0
  27. package/scripts/export.ts +42 -0
  28. package/scripts/ingest/acquire.ts +189 -0
  29. package/scripts/ingest/exts.ts +29 -0
  30. package/scripts/ingest/files.ts +29 -0
  31. package/scripts/ingest/folder.ts +44 -0
  32. package/scripts/ingest/index.ts +125 -0
  33. package/scripts/ingest/protect.ts +42 -0
  34. package/scripts/ingest/web.ts +54 -0
  35. package/scripts/init.ts +93 -0
  36. package/scripts/lib/args.ts +8 -0
  37. package/scripts/lib/extract.ts +384 -0
  38. package/scripts/lib/graph-page.ts +477 -0
  39. package/scripts/lib/graph.ts +117 -0
  40. package/scripts/lib/ledger.ts +136 -0
  41. package/scripts/lib/log.ts +53 -0
  42. package/scripts/lib/paths.ts +55 -0
  43. package/scripts/lib/scaffold.ts +56 -0
  44. package/scripts/lib/util.ts +154 -0
  45. package/scripts/lint.ts +165 -0
  46. package/scripts/new-bundle.ts +17 -0
  47. package/scripts/visualize.ts +104 -0
  48. package/skills/catalog/SKILL.md +164 -0
  49. package/skills/export/SKILL.md +31 -0
  50. package/skills/ingest/SKILL.md +229 -0
  51. package/skills/lint/SKILL.md +69 -0
  52. package/skills/new-bundle/SKILL.md +25 -0
  53. package/skills/query/SKILL.md +114 -0
  54. package/skills/visualize/SKILL.md +33 -0
  55. package/templates/hub/gitattributes +12 -0
  56. package/templates/hub/gitignore +12 -0
  57. package/templates/hub/outer.index.md +13 -0
@@ -0,0 +1,104 @@
1
+ // khb visualize — scan bundles + refs + concept links, serve an interactive graph from a
2
+ // local server. Two zoom levels: bundles (outer graph, refs.md edges) and, click a bundle,
3
+ // its concepts (inner graph, markdown-link edges) with a panel to view a concept's body.
4
+ // Unpinned, it picks a random free port (and retries on another if that one's taken)
5
+ // rather than fighting over one fixed number; it also exits on its own once the browser
6
+ // tab goes away, since a background `khb visualize` nobody's looking at is just a stray
7
+ // process. `--port N` pins a specific port.
8
+ import { buildGraphData, readConceptFile } from "./lib/graph";
9
+ import { renderGraphPage } from "./lib/graph-page";
10
+
11
+ const argv = process.argv.slice(2);
12
+ // port 0 asks the OS for any free port — no fixed default to collide with something else
13
+ // already running on the machine. Both `--port N` and `--port=N` pin it, since the docs
14
+ // have always shown the spaced form.
15
+ const eqArg = argv.find((a) => a.startsWith("--port="));
16
+ const spacedArg = argv[argv.indexOf("--port") + 1];
17
+ const requestedPort = eqArg
18
+ ? Number(eqArg.slice("--port=".length))
19
+ : argv.includes("--port") && spacedArg
20
+ ? Number(spacedArg)
21
+ : 0;
22
+ const noOpen = argv.includes("--no-open");
23
+
24
+ function summarize(data: ReturnType<typeof buildGraphData>) {
25
+ const concepts = Object.values(data.bundleGraphs).reduce((n, g) => n + g.concepts.length, 0);
26
+ const links = Object.values(data.bundleGraphs).reduce((n, g) => n + g.edges.length, 0);
27
+ return `${data.bundles.length} bundles, ${data.bundleEdges.length} refs, ${concepts} concepts, ${links} concept links`;
28
+ }
29
+
30
+ let data = buildGraphData();
31
+
32
+ // The page pings /api/heartbeat every few seconds and beacons /api/close on unload.
33
+ // Nothing marks the server "connected" until the first heartbeat, so a slow first
34
+ // page-load never races the idle timeout below.
35
+ let connected = false;
36
+ let lastHeartbeat = Date.now();
37
+ const IDLE_TIMEOUT_MS = 10_000;
38
+ const HEARTBEAT_INTERVAL_MS = 3_000;
39
+
40
+ const fetchHandler = (req: Request) => {
41
+ const url = new URL(req.url);
42
+ if (url.pathname === "/")
43
+ return new Response(renderGraphPage(data), { headers: { "content-type": "text/html; charset=utf-8" } });
44
+ if (url.pathname === "/api/graph") {
45
+ if (url.searchParams.get("rebuild")) data = buildGraphData();
46
+ return Response.json(data);
47
+ }
48
+ if (url.pathname === "/api/file") {
49
+ const bundle = url.searchParams.get("bundle") ?? "";
50
+ const path = url.searchParams.get("path") ?? "";
51
+ const body = readConceptFile(bundle, path);
52
+ return body === undefined
53
+ ? new Response("not found", { status: 404 })
54
+ : new Response(body, { headers: { "content-type": "text/plain; charset=utf-8" } });
55
+ }
56
+ if (url.pathname === "/api/heartbeat") {
57
+ connected = true;
58
+ lastHeartbeat = Date.now();
59
+ return new Response("ok");
60
+ }
61
+ if (url.pathname === "/api/close" && req.method === "POST") {
62
+ console.log("khb visualize — browser closed, shutting down.");
63
+ setTimeout(() => process.exit(0), 50); // let the response flush first
64
+ return new Response("ok");
65
+ }
66
+ return new Response("not found", { status: 404 });
67
+ };
68
+
69
+ let server;
70
+ try {
71
+ server = Bun.serve({ port: requestedPort, fetch: fetchHandler });
72
+ } catch (e) {
73
+ if (requestedPort && (e as { code?: string }).code === "EADDRINUSE") {
74
+ console.warn(`Port ${requestedPort} is busy — picking a free one instead.`);
75
+ server = Bun.serve({ port: 0, fetch: fetchHandler });
76
+ } else throw e;
77
+ }
78
+
79
+ setInterval(() => {
80
+ if (connected && Date.now() - lastHeartbeat > IDLE_TIMEOUT_MS) {
81
+ console.log("khb visualize — browser tab gone, shutting down.");
82
+ process.exit(0);
83
+ }
84
+ }, HEARTBEAT_INTERVAL_MS).unref();
85
+
86
+ const url = `http://localhost:${server.port}`;
87
+ console.log(`khb visualize → ${url} (${summarize(data)})`);
88
+
89
+ // Open the default browser. If that fails the URL is already printed above, so a headless
90
+ // or locked-down box just falls back to copy-paste rather than erroring out.
91
+ if (!noOpen) {
92
+ const cmd =
93
+ process.platform === "win32"
94
+ ? ["cmd", "/c", "start", "", url]
95
+ : process.platform === "darwin"
96
+ ? ["open", url]
97
+ : ["xdg-open", url];
98
+ try {
99
+ Bun.spawn(cmd, { stdout: "ignore", stderr: "ignore" }).unref();
100
+ } catch {
101
+ console.warn("Could not launch a browser — open the URL above yourself.");
102
+ }
103
+ }
104
+ console.log(`The server exits on its own once you close the tab. Ctrl+C also works.`);
@@ -0,0 +1,164 @@
1
+ ---
2
+ name: catalog
3
+ description: Turn one bundle's raw/ material into OKF concept docs — read each raw file, split it into sub-topics, label each with type/title/description/tags, link them, and register them in index.md. Uses parallel subagents when the runtime supports them. Use after khb ingest, or when the user asks to curate, catalog, organize, or write up a bundle.
4
+ ---
5
+
6
+ # Catalog a bundle
7
+
8
+ The second half of the pipeline: `khb ingest` produced faithful text in `raw/`; catalog
9
+ turns that text into **knowledge**. One bundle at a time, always.
10
+
11
+ The unit of output is the **concept** — one idea, one markdown file, OKF frontmatter,
12
+ registered in an index. A raw file is not a concept. A 40-page contract is a dozen concepts;
13
+ three meeting transcripts about the same decision are one. Splitting and merging is the
14
+ whole job, and it is why this pass needs a model and `khb ingest` does not.
15
+
16
+ `khb` has no `catalog` command. Nothing here is mechanical enough to script.
17
+
18
+ **Catalog classifies concepts and links them. It never reorganizes bundles.** The splitting
19
+ it does is *within* a bundle — raw files into concepts, concepts into subdirectories. A
20
+ bundle is a logical unit its owner defined (a person, a team, a project) and it holds many
21
+ topics by design, so finding three unrelated subjects in one bundle is the expected case,
22
+ not a problem to fix. Never create a bundle, move material to another bundle, or propose a
23
+ split because the contents look heterogeneous. If material clearly belongs to someone else,
24
+ it gets a `refs.md` line and stays where it is.
25
+
26
+ ## 1. Scope the work
27
+
28
+ ```
29
+ khb ingest <bundle> # if the user asked to catalog something not yet ingested
30
+ ```
31
+
32
+ That only moves bytes for sources already declared in the bundle's `sources.yaml`; if the
33
+ material isn't declared yet, you are at the start of the [ingest skill](../ingest/SKILL.md),
34
+ not this one.
35
+
36
+ The worklist is `bundles/<bundle>/log.md`: **every row with a `raw` path and an empty
37
+ `curated` column.** Nothing else records what is outstanding, so work from the ledger rather
38
+ than from a directory listing — a `raw/` file whose row is already filled has been done.
39
+
40
+ Rows with an empty `raw` were never extracted; they are ingest's problem, not yours. If
41
+ there are many, say so and offer to fix ingest first.
42
+
43
+ ## 2. Read the map before writing anything
44
+
45
+ Open the bundle's `index.md` and any nested indexes, and list the existing concepts: path,
46
+ `title`, `type`, `tags`. This is the **vocabulary**, and it does two jobs — it stops you
47
+ coining `billing` when `invoices` already exists, and it is what lets a new concept link to
48
+ an old one. Hold it in this thread; every subagent gets a copy.
49
+
50
+ Also decide the **grouping** now, before any file is written: which subdirectories the new
51
+ concepts go in (`contracts/`, `metrics/`, `notes/`, `playbooks/` — whatever fits this
52
+ domain; structure carries no fixed meaning). Reuse existing directories wherever they fit.
53
+ Inventing a new subdirectory per raw file is the classic failure here.
54
+
55
+ ## 3. Seed wave — one file, checked by hand
56
+
57
+ Pick the most representative raw file and delegate it to **one subagent** with the prompt
58
+ below. Use a fast, economical model when the runtime exposes model selection; otherwise use
59
+ its default subagent. If subagents are unavailable, do the same work in the current thread.
60
+ Then read what it wrote. You are checking the shape, not the facts: right granularity,
61
+ frontmatter complete, titles that read like knowledge rather than filenames. Correct the
62
+ prompt if it is off, and take the concept titles it produced into your vocabulary before
63
+ going wide.
64
+
65
+ ## 4. Bulk wave — the rest in parallel
66
+
67
+ Use one subagent per remaining raw file and start them concurrently when the runtime
68
+ supports parallel delegation. Group several small files into one task when they are
69
+ obviously the same subject; never split one file across two tasks. Without subagents,
70
+ process the files sequentially in the current thread and keep the same single-writer rules.
71
+
72
+ Parallel subagents cannot see each other's output, so two rules are absolute:
73
+
74
+ - **Every prompt carries the vocabulary.** Otherwise you get `q3-budget`, `budget-q3` and
75
+ `quarterly-budget` as three concepts.
76
+ - **Subagents write concept docs only.** `index.md`, `log.md` and `refs.md` have exactly one
77
+ writer — you, in step 5. Concurrent edits to a shared index silently lose rows.
78
+
79
+ Prompt each subagent with, substituting the bracketed parts:
80
+
81
+ > Read `bundles/<bundle>/raw/<file>.md`. It is ingested source material with a provenance
82
+ > header naming the original file.
83
+ >
84
+ > Split its content into distinct **concepts** — one idea per concept. A concept is
85
+ > something someone would ask a question about on its own. Do not summarize the document as
86
+ > a whole, and do not create a concept per section heading.
87
+ >
88
+ > Write each concept as its own markdown file in `bundles/<bundle>/<group>/`, named
89
+ > `<kebab-case-title>.md`. If that filename already exists and is about something else,
90
+ > append `-2`. Each file starts with OKF frontmatter:
91
+ >
92
+ > ```yaml
93
+ > ---
94
+ > type: <Table | Metric | Playbook | Decision Log | Reference | Contract | …>
95
+ > title: <display name>
96
+ > description: <one line, under 20 words>
97
+ > tags: [<lowercase kebab-case>]
98
+ > timestamp: <ISO from the source if it has one, else today>
99
+ > ---
100
+ > ```
101
+ >
102
+ > Then the body: the actual knowledge, in your own words, structured with headings. End with
103
+ > a `# Citations` section listing the raw file path and the `source:` value from its header.
104
+ >
105
+ > Link to related concepts with plain markdown links using bundle-root paths
106
+ > (`/metrics/churn.md`). You may link to any of these existing concepts:
107
+ > `<vocabulary: path — title — description, one per line>`
108
+ > Reuse an existing concept's subject rather than restating it: if this file only adds a
109
+ > detail to one of the above, say so in your reply instead of writing a near-duplicate.
110
+ > Never link outside this bundle.
111
+ >
112
+ > If the provenance header says `quality: low`, the text came from OCR or a transcript and
113
+ > may be garbled. Do not invent through it — quote what is legible, and flag what is not in
114
+ > your reply.
115
+ >
116
+ > If the file contains nothing worth keeping (a receipt, boilerplate, a duplicate), write
117
+ > nothing and say so.
118
+ >
119
+ > Reply with one line per concept you wrote: `<path> | <title> | <description>`, then any
120
+ > flags. Do not edit `index.md`, `log.md` or `refs.md`.
121
+
122
+ ## 5. Merge — you, single writer
123
+
124
+ Everything in this step happens in this thread, sequentially, because every file it touches
125
+ is shared.
126
+
127
+ 1. **Review and dedupe.** Read the concepts the wave produced. Two files covering the same
128
+ idea get merged into the better one and the loser deleted. A concept that only restates
129
+ an existing one gets folded into it instead.
130
+ 2. **Cross-link.** Subagents could link to pre-existing concepts but not to each other's
131
+ output. Add the sibling links now — that is where most of a bundle's value ends up.
132
+ 3. **Register every concept in `index.md`**, in the OKF form
133
+ `* [Title](path.md) - description`. **An unindexed concept is invisible to every query**
134
+ and lint will flag it. Add a nested `index.md` if a subdirectory grew past ~10 concepts.
135
+ 4. **Fill the `curated` column in `log.md`** for every row you worked. Use the concept
136
+ paths, comma-separated. For a raw file you deliberately declined, write `declined` — a
137
+ row that stays empty will be offered as backlog forever.
138
+ 5. **Foreign material → `refs.md`.** A raw file that turns out to belong to a different
139
+ bundle is not curated here: note the target bundle and the reason in `refs.md` and, if
140
+ the user agrees, add the source to that bundle's `sources.yaml`. Never inline-link across
141
+ bundles, and never copy the content over.
142
+
143
+ ## 6. Verify
144
+
145
+ ```
146
+ khb lint # index coverage, frontmatter, ref targets, no cross-bundle links
147
+ khb visualize # optional: refresh visualizer/graph.html
148
+ ```
149
+
150
+ Fix every error before finishing. Then report: concepts created, raw files declined, rows
151
+ still outstanding, and anything you flagged as low-quality and worth re-reading from source.
152
+
153
+ ## Judgement notes
154
+
155
+ - **Curate selectively.** Raw is bulk; concepts are distilled. Most corpora are 80% receipts
156
+ and boilerplate. Declining is a real outcome, not a failure.
157
+ - **Granularity.** If two concepts are always read together, they are one. If one concept
158
+ has two `# Schema` sections, it is two.
159
+ - **Write knowledge, not summaries.** "This document discusses the retention policy" is
160
+ useless; the retention policy is what belongs in the file.
161
+ - **Never put content in an index.** Indexes route. Knowledge goes in a concept doc.
162
+ - **`quality: low` sources** deserve a look at the original before you commit their claims —
163
+ the `source:` path in the raw header is exactly for this. Reading a chart or a scanned
164
+ table with vision recovers what OCR dropped.
@@ -0,0 +1,31 @@
1
+ ---
2
+ name: export
3
+ description: Export a KHB bundle as a standalone shareable folder with the common patterns injected. Use when the user wants to share or ship a single bundle.
4
+ ---
5
+
6
+ # Export a bundle
7
+
8
+ Bundles stay lean inside the hub because the common patterns live at hub root; export
9
+ injects those patterns so the folder works alone with any agent.
10
+
11
+ 1. Run `khb export <bundle> [dest]` (default dest: `export/<bundle>/`). The command
12
+ **refuses to write into an existing destination** — re-exporting means removing the old
13
+ folder first, or passing a new `dest`.
14
+ 2. The result is a miniature hub, with the bundle itself one level down:
15
+
16
+ ```
17
+ <dest>/bundle/ the bundle, copied whole
18
+ <dest>/outer.index.md single-bundle router pointing at bundle/index.md
19
+ <dest>/AGENTS.md the contract, plus CLAUDE.md
20
+ <dest>/skills/ the canonical protocols (query, ingest, lint, …)
21
+ <dest>/.claude/skills/ discovery adapters, and .agents/skills/ likewise
22
+ <dest>/README.md provenance: what this is and when it was exported
23
+ ```
24
+
25
+ 3. Tell the user two things before they send it anywhere:
26
+ - `refs.md` entries pointing at other bundles **will not resolve** — the export is one
27
+ bundle, and its cross-bundle pointers now dangle.
28
+ - The copy is literal, so `raw/` and `log.md` go with it. `log.md` records **absolute
29
+ source paths** from the machine that ingested them, and `raw/` is uncurated source
30
+ material that was never written for an outside reader. Check both before sharing
31
+ outside the team, and prune if they say more than intended.
@@ -0,0 +1,229 @@
1
+ ---
2
+ name: ingest
3
+ description: Acquire external material (folders, files, web pages, Confluence, ADO, git) into a KHB bundle's raw/ folder as markdown with provenance — one flat mechanical phase, no interpretation. Use when the user wants to add, import, dump, pull, or refresh source data in the knowledge base.
4
+ ---
5
+
6
+ # Ingest into KHB
7
+
8
+ **Ingest gets bytes into `bundles/<bundle>/raw/` as markdown with a provenance header.
9
+ That is all it does.** It is one flat phase, it is mechanical, and it ends the moment the
10
+ text exists. Deciding what the text *means* — splitting it into concepts, titling,
11
+ tagging, linking, indexing — is the [catalog skill](../catalog/SKILL.md), a separate step
12
+ you run afterwards.
13
+
14
+ Do not curate here. Do not create, split, or merge bundles here — routing material to a
15
+ bundle that exists is fine, reshaping the hub is not. If you find yourself reading a document
16
+ to understand it, you have left this skill.
17
+
18
+ ## 1. Declare the sources
19
+
20
+ Ingest is bundle-first: material lands in one bundle, and you say where it comes from. Edit
21
+ `bundles/<bundle>/sources.yaml`:
22
+
23
+ ```yaml
24
+ sources:
25
+ - type: folder # walk a directory tree
26
+ path: /abs/path/to/project-x
27
+ - type: files # a scattered, explicitly named set
28
+ paths:
29
+ - /abs/path/to/one.pdf
30
+ - /abs/path/to/two.xlsx
31
+ - type: web
32
+ urls:
33
+ - https://example.com/design-doc
34
+ # Types with no scripted ingester are still declared here, for the record —
35
+ # you pull them yourself in step 3.
36
+ - type: confluence
37
+ space: PROJX
38
+ ```
39
+
40
+ If the user has not explicitly named the source locations, inspect the bundle's current
41
+ `sources.yaml`, then ask which files, folders, URLs, or services to ingest. Include any
42
+ existing declarations in the question so the user can confirm or replace them. Do not
43
+ infer sources from nearby files, edit `sources.yaml`, or run `khb ingest` until the user
44
+ answers.
45
+
46
+ Nothing is copied by declaring a source.
47
+
48
+ **Which bundle — take the first of these that applies, and do not go further:**
49
+
50
+ 1. **The user named a bundle** → use it. A named bundle that does not exist is an error,
51
+ not an invitation to create one.
52
+ 2. **The hub has bundles and exactly one plainly owns the material** → use it, and say
53
+ which you picked. If several could own it, ask which — this is the only bundle question
54
+ ingest ever asks.
55
+ 3. **Anything else** — no bundle named, or the hub has no bundles at all → `default`,
56
+ created on the spot, without asking.
57
+
58
+ Never ask the user to name or create a bundle *for the ingest to land in*. `default` exists
59
+ so that question never has to be asked at this stage: bytes always have somewhere to go, and
60
+ which bundle owns them is a cheaper decision later, once the text exists and the user can see
61
+ what they actually have.
62
+
63
+ **The `default` bundle.** When no bundle is named, ingest targets `default` and creates it
64
+ if the hub has none — a first `khb ingest` never fails for want of a destination. It is not a
65
+ way around step 2: when a bundle in the hub plainly owns the material, that bundle wins. What
66
+ lands there is ordinary bundle content: catalog it like any other. Do **not** graduate it into new
67
+ bundles on your own — a bundle is a logical unit the user defines (a person, a team, a
68
+ project), so material leaves `default` only when the user says which bundle owns it. An
69
+ explicitly named bundle that doesn't exist is still an error — only `default` is conjured.
70
+
71
+ ## 2. Run it
72
+
73
+ ```
74
+ khb ingest # no bundle named → the 'default' bundle
75
+ khb ingest <bundle> # incremental: unchanged content hashes are skipped
76
+ khb ingest <bundle> --force # re-acquire everything
77
+ khb ingest <bundle> --skip-ocr # leave scans and images unread
78
+ khb ingest <bundle> --skip-audio # leave audio/video untranscribed
79
+ ```
80
+
81
+ One command handles every scripted source in `sources.yaml` and extracts everything it can,
82
+ locally. Read the summary it prints — the counts are the state of the world:
83
+
84
+ | line | meaning |
85
+ |---|---|
86
+ | `unchanged, skipped` | already acquired at this exact content hash |
87
+ | `extracted` / `reused from the extraction cache` | converted now / converted by an earlier run or another bundle |
88
+ | `read by OCR` / `transcribed` | lossy routes — see quality, below |
89
+ | `marked quality: low` | verify these against the source when cataloging |
90
+ | `not extracted` | got a ledger row with an empty `raw`; the per-file line says why |
91
+
92
+ Above that summary, every file gets its own trace — announced *before* the work starts, so a
93
+ run that is taking minutes always names the file it is taking them on:
94
+
95
+ ```
96
+ [ 7/94] D:\corpus\board-pack.pdf
97
+ extracting pdf …
98
+ no text layer, 12p — scanned, running OCR (seconds per page)
99
+ page 1/12 — 1843 chars
100
+
101
+ extracted → raw/folder/board-pack.pdf.md [tesseract.js @ 216dpi, quality: low] (48.1s)
102
+ ```
103
+
104
+ There is no quiet mode, and this is deliberate: the trace is the audit trail for a pass that
105
+ rewrites `raw/`. When something looks wrong later, that line — tool, quality, elapsed — is
106
+ what tells you which file to distrust and which extractor to blame.
107
+
108
+ ### What khb extracts
109
+
110
+ All of it runs locally and none of it contacts a model — that is the `AGENTS.md` division of
111
+ labor. khb converts bytes to text as cheaply as possible; your judgement is spent on
112
+ curation, not transcription.
113
+
114
+ | Format | Tool | Quality |
115
+ |---|---|---|
116
+ | `.md .txt .rst .adoc .html .csv .json .yaml` | copied verbatim | high |
117
+ | `.pdf` (born-digital) | `unpdf`, then `pdftotext` if on PATH | high |
118
+ | `.docx` / `.odt` / `.pptx` | `mammoth` / `fflate` / `fflate`, `pandoc` if on PATH | high |
119
+ | `.xlsx` | `fflate` → one markdown table per sheet | high |
120
+ | `.pdf` (scanned, no text layer) | `pdfium` + `tesseract.js`, automatically | **low** |
121
+ | `.png .jpg .webp .tif .gif` | `tesseract.js`, automatically | **low** |
122
+ | `.mp3 .wav .m4a .mp4 .mov .mkv` | local `whisper` / `faster-whisper` | **low** |
123
+
124
+ Extracted text is cached hub-wide by content hash at `inbox/extracted/<sha256>.md`, so the
125
+ same file appearing in two bundles converts once.
126
+
127
+ OCR and transcription need optional dependencies. When they are missing khb says so once and
128
+ records the affected files as pending rather than failing the run:
129
+
130
+ ```
131
+ bun add @hyzyla/pdfium sharp tesseract.js # OCR — ~75 MB WASM, no system binary
132
+ pip install -U openai-whisper # transcription (faster-whisper also works)
133
+ ```
134
+
135
+ Install them where `khb` resolves modules from — for a global install that is the khb
136
+ package directory, not your hub. khb prints the exact `cd … && bun add …` to use.
137
+
138
+ ## 3. Sources khb cannot reach
139
+
140
+ Anything behind an authenticated API has no scripted ingester, because maintaining API
141
+ wrappers is not what this tool is for. Pull those yourself with the site's MCP server or
142
+ official CLI, and write the result into `raw/<type>/` **in exactly the format khb
143
+ produces** — same folder shape, same header — so the catalog pass cannot tell the
144
+ difference:
145
+
146
+ | Source | How |
147
+ |---|---|
148
+ | Confluence | MCP server or `confluence` CLI → `raw/confluence/<page>.md` |
149
+ | Azure DevOps | MCP/CLI → wiki pages, work items → `raw/ado/<item>.md` |
150
+ | GitHub / GitLab issues, PRs, wikis | `gh` / `glab` CLI → `raw/git/<thing>.md` |
151
+ | Source code repository | do **not** copy — record the location in `sources.yaml` and read it in place |
152
+ | A diagram or chart no OCR can read | vision read the image → `raw/images/<file>.md`, `extract_tool: claude-vision` |
153
+
154
+ Then add the row to `log.md` yourself (`source`, `sha256` if you have it, `fetched`, `raw`),
155
+ so the ledger stays the complete record regardless of who did the fetching.
156
+
157
+ ### The provenance header — the contract
158
+
159
+ Every file in `raw/` starts with it. This is the reason ingest is a separate phase from
160
+ catalog: extraction is sometimes lossy, and curation must always be able to walk back to the
161
+ original bytes.
162
+
163
+ ```yaml
164
+ ---
165
+ source: /abs/path/to/original.pdf # or the url, or the tool query
166
+ fetched: 2026-07-23T09:14:02Z
167
+ sha256: db2ee470c95d
168
+ extract_tool: tesseract.js # what produced the text below
169
+ quality: low # high = real text; low = OCR or a transcript
170
+ ---
171
+ ```
172
+
173
+ `quality: low` is a standing invitation to distrust the body. When cataloging one of these
174
+ and the text reads thin, garbled, or contradictory, **open the `source:` file and read it
175
+ directly** — a vision pass over a chart or a scanned table recovers what OCR drops. Rewrite
176
+ the `raw/` file with `extract_tool: claude-vision` and `quality: high` when you do.
177
+
178
+ ## 4. The ledger — `log.md`
179
+
180
+ Every bundle keeps its ingest ledger in `log.md` (OKF-reserved, so it is never mistaken for
181
+ a concept doc, and committed, so it survives `raw/` being deleted and re-derived).
182
+
183
+ | column | owner | meaning |
184
+ |---|---|---|
185
+ | `source` | khb | origin URI: absolute path, url, or tool query |
186
+ | `sha256` | khb | content hash (12-char prefix) — drives skip-unchanged, move detection and dedup |
187
+ | `fetched` | khb | ISO timestamp of last acquisition |
188
+ | `raw` | khb | bundle-relative `raw/` path; **empty = never extracted** |
189
+ | `curated` | agent | concept doc(s) distilled from it; **empty = catalog backlog** |
190
+
191
+ `khb ingest` maintains the first four and never touches `curated`.
192
+
193
+ ### Moved and renamed sources
194
+
195
+ The bytes are a source's identity; the path is only where they live today. When a file
196
+ appears at a path the ledger has not seen, khb looks for an existing row with the same hash
197
+ whose own path has since disappeared. If exactly one matches, that row is **re-pointed** at
198
+ the new path: same `raw` file (concepts cite it by name, so it is never renamed), same
199
+ `curated`, and the raw file's `source:` provenance header is corrected. Nothing is
200
+ re-extracted and nothing re-enters the backlog.
201
+
202
+ Two cases are deliberately *not* treated as moves, because both would silently rewire
203
+ provenance:
204
+
205
+ - **Copy** — the twin row's path still exists, so these are two real sources with the same
206
+ bytes. Both are ingested; the run says so, and folding or declining the second is a
207
+ cataloging judgement.
208
+ - **Ambiguous** — several vanished rows share the hash, so which one moved here is
209
+ unknowable. The new file is ingested as its own source and the run says why.
210
+
211
+ Still on you, not khb: a source **modified in place** keeps its `curated` value, so the
212
+ concept derived from it does not re-enter the backlog even though its material changed.
213
+ Watch for `raw/` files whose content shifted and re-catalog them deliberately.
214
+
215
+ ## Hand off
216
+
217
+ Ingest is done when the summary shows nothing unexpectedly pending. Report to the user what
218
+ landed, what didn't and why, and how many rows are uncurated — then continue with the
219
+ [catalog skill](../catalog/SKILL.md) to turn `raw/` into concept docs.
220
+
221
+ ## Hygiene
222
+
223
+ - `raw/` is gitignored, derived, and **never canonical**. Never cite it in an answer.
224
+ - Never copy a bulk corpus into a bundle wholesale. Extraction shrinks documents to text;
225
+ source-code repos and media libraries stay where they are and get a `sources.yaml` entry.
226
+ - The same file in two bundles: one bundle owns it, the other gets a `refs.md` entry. Never
227
+ two copies. The content hash in `log.md` is how you spot it.
228
+ - `log.md` records absolute source paths. If those paths are themselves sensitive, gitignore
229
+ it before the first commit.
@@ -0,0 +1,69 @@
1
+ ---
2
+ name: lint
3
+ description: Validate KHB structure (routing integrity, bundle shape, OKF conformance). Use after any structural edit, or when the user asks to check/validate/fix the knowledge base.
4
+ ---
5
+
6
+ # Lint KHB
7
+
8
+ 1. Run `khb lint` from anywhere inside the hub — it walks up to `khb.json` to find the root.
9
+ 2. Fix every ERROR (structure, routing, OKF frontmatter); judge warnings case by case
10
+ (broken index links may be intentional not-yet-written knowledge).
11
+ 3. Re-run until 0 errors. If a fix changes root files **and the hub has a `meta` bundle**,
12
+ log it in `bundles/meta/notes/decisions.md`. No meta bundle means no decision log — do
13
+ not create one to have somewhere to write.
14
+
15
+ ## The rules (L1–L9)
16
+
17
+ Enforced by `khb lint`. Combines KHB routing rules with
18
+ OKF v0.1 conformance (see the OKF spec). Reserved filenames: `index.md`, `log.md`
19
+ (OKF) and `refs.md` (KHB). Every other `.md` in a bundle — outside `raw/` —
20
+ is a **concept document**.
21
+
22
+ ### Bundle shape
23
+
24
+ - L1. Every `bundles/<name>/` has: `index.md`, `refs.md`, `sources.yaml`. Concept docs
25
+ live in whatever subdirectory grouping fits the domain. No per-bundle AGENTS.md —
26
+ root `AGENTS.md` is the common contract; `khb export` injects it for standalone
27
+ sharing.
28
+ - L2. Bundle names: lowercase, digits, hyphens (`^[a-z0-9][a-z0-9-]*$`).
29
+
30
+ ### Routing integrity
31
+
32
+ - L3. Every bundle is listed in `outer.index.md`; every bundle linked from
33
+ `outer.index.md` exists on disk.
34
+ - L4. Every concept doc is listed in at least one of the bundle's `index.md` files
35
+ (error). Index links pointing at missing files are a warning only — OKF treats
36
+ broken links as not-yet-written knowledge.
37
+ - L5. Index files contain routing only: headings, bullet/table link lines, one-line
38
+ descriptions. Paragraph-length prose is a violation (warning).
39
+
40
+ ### Independence
41
+
42
+ - L6. No markdown link from a concept doc into another bundle's files. Cross-bundle
43
+ pointers live in `refs.md` only.
44
+ - L7. Every target bundle named in `refs.md` exists.
45
+
46
+ ### Provenance
47
+
48
+ - L8. Files under `raw/` carry a provenance header (warning): frontmatter present, a
49
+ non-empty `source:`, and `quality:` — if set — reading exactly `high` or `low`.
50
+ `source` is what makes a bad extraction recoverable, so a raw file without one is
51
+ uncatalogable, not merely untidy.
52
+
53
+ ### OKF conformance
54
+
55
+ - L9. Concept frontmatter is the machine-readable half of a concept, so it is validated
56
+ as data rather than glanced at:
57
+ - frontmatter block present, and **parses as YAML** (error) — a malformed block means
58
+ every field is silently lost.
59
+ - non-empty `type` (error) — the one OKF v0.1 §9 requirement. Its *value* stays
60
+ free-form: `Metric`, `Playbook`, `Runbook`, anything the domain needs.
61
+ - `title` and `description` present (warning) — indexes and index generators read them.
62
+ - `tags`, if present, is a YAML list of strings (error). `tags: "a, b"` is a string and
63
+ filters as one opaque value; `tags: [a, b]` is two tags.
64
+ - `timestamp`, if present, parses as an ISO-8601 datetime (warning).
65
+ - unknown top-level keys (warning). The known set is `type`, `title`, `description`,
66
+ `resource`, `tags`, `timestamp` — `resource` is optional and unvalidated, but it is
67
+ known, so it costs no warning. OKF is permissive and extra keys are legal, but `titel:`
68
+ is a typo that silently drops the field, and one warning line is cheaper than a field
69
+ nobody notices is missing.
@@ -0,0 +1,25 @@
1
+ ---
2
+ name: new-bundle
3
+ description: Create a new KHB bundle — a logical unit owned by a person, team, project or client. Use when the user wants to add a new owner/area to the knowledge base.
4
+ ---
5
+
6
+ # New bundle
7
+
8
+ A bundle is a **logical unit, defined by whoever owns its material** — a person, a team, a
9
+ project, a client. It is not a subject classification. One bundle holds as many topics as
10
+ its owner has; topics are organized *inside* it with subdirectories, not by making more
11
+ bundles.
12
+
13
+ Creating a bundle is always a human decision. Never create one because material looks like
14
+ it belongs to a new subject, and never split an existing one on your own initiative — an
15
+ agent splits a bundle only when explicitly told to.
16
+
17
+ 1. Pick a name: lowercase, digits, hyphens. Name the owner or the context, not the subject
18
+ (`team-payments`, `client-acme`, `notes`), and expect its scope line to list several
19
+ topics — that is correct, not a signal to split.
20
+ 2. Run `khb new-bundle <name> "<one-line scope>"` — scaffolds from
21
+ `.bundle_template/` and registers in `outer.index.md`.
22
+ 3. Fill in the "Route here when" column in `outer.index.md`. Write it as a trigger for
23
+ *whose* material this is, since one bundle answers for many topics.
24
+ 4. Declare inputs in `sources.yaml` (see the ingest skill to pull them).
25
+ 5. `khb lint`.