@msareen/knowledge-hub-builder 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/catalog/SKILL.md +7 -0
- package/.agents/skills/export/SKILL.md +7 -0
- package/.agents/skills/ingest/SKILL.md +7 -0
- package/.agents/skills/lint/SKILL.md +7 -0
- package/.agents/skills/new-bundle/SKILL.md +7 -0
- package/.agents/skills/query/SKILL.md +7 -0
- package/.agents/skills/visualize/SKILL.md +7 -0
- package/.bundle_template/index.md +9 -0
- package/.bundle_template/log.md +10 -0
- package/.bundle_template/raw/.gitkeep +15 -0
- package/.bundle_template/refs.md +6 -0
- package/.bundle_template/sources.yaml +13 -0
- package/.claude/skills/catalog/SKILL.md +7 -0
- package/.claude/skills/export/SKILL.md +7 -0
- package/.claude/skills/ingest/SKILL.md +7 -0
- package/.claude/skills/lint/SKILL.md +7 -0
- package/.claude/skills/new-bundle/SKILL.md +7 -0
- package/.claude/skills/query/SKILL.md +7 -0
- package/.claude/skills/visualize/SKILL.md +7 -0
- package/AGENTS.md +167 -0
- package/CLAUDE.md +13 -0
- package/README.md +289 -0
- package/SPEC.md +354 -0
- package/document/faq.md +156 -0
- package/package.json +52 -0
- package/scripts/cli.ts +66 -0
- package/scripts/export.ts +42 -0
- package/scripts/ingest/acquire.ts +189 -0
- package/scripts/ingest/exts.ts +29 -0
- package/scripts/ingest/files.ts +29 -0
- package/scripts/ingest/folder.ts +44 -0
- package/scripts/ingest/index.ts +125 -0
- package/scripts/ingest/protect.ts +42 -0
- package/scripts/ingest/web.ts +54 -0
- package/scripts/init.ts +93 -0
- package/scripts/lib/args.ts +8 -0
- package/scripts/lib/extract.ts +384 -0
- package/scripts/lib/graph-page.ts +477 -0
- package/scripts/lib/graph.ts +117 -0
- package/scripts/lib/ledger.ts +136 -0
- package/scripts/lib/log.ts +53 -0
- package/scripts/lib/paths.ts +55 -0
- package/scripts/lib/scaffold.ts +56 -0
- package/scripts/lib/util.ts +154 -0
- package/scripts/lint.ts +165 -0
- package/scripts/new-bundle.ts +17 -0
- package/scripts/visualize.ts +104 -0
- package/skills/catalog/SKILL.md +164 -0
- package/skills/export/SKILL.md +31 -0
- package/skills/ingest/SKILL.md +229 -0
- package/skills/lint/SKILL.md +69 -0
- package/skills/new-bundle/SKILL.md +25 -0
- package/skills/query/SKILL.md +114 -0
- package/skills/visualize/SKILL.md +33 -0
- package/templates/hub/gitattributes +12 -0
- package/templates/hub/gitignore +12 -0
- package/templates/hub/outer.index.md +13 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// khb visualize — scan bundles + refs + concept links, serve an interactive graph from a
|
|
2
|
+
// local server. Two zoom levels: bundles (outer graph, refs.md edges) and, click a bundle,
|
|
3
|
+
// its concepts (inner graph, markdown-link edges) with a panel to view a concept's body.
|
|
4
|
+
// Unpinned, it picks a random free port (and retries on another if that one's taken)
|
|
5
|
+
// rather than fighting over one fixed number; it also exits on its own once the browser
|
|
6
|
+
// tab goes away, since a background `khb visualize` nobody's looking at is just a stray
|
|
7
|
+
// process. `--port N` pins a specific port.
|
|
8
|
+
import { buildGraphData, readConceptFile } from "./lib/graph";
|
|
9
|
+
import { renderGraphPage } from "./lib/graph-page";
|
|
10
|
+
|
|
11
|
+
const argv = process.argv.slice(2);
|
|
12
|
+
// port 0 asks the OS for any free port — no fixed default to collide with something else
|
|
13
|
+
// already running on the machine. Both `--port N` and `--port=N` pin it, since the docs
|
|
14
|
+
// have always shown the spaced form.
|
|
15
|
+
const eqArg = argv.find((a) => a.startsWith("--port="));
|
|
16
|
+
const spacedArg = argv[argv.indexOf("--port") + 1];
|
|
17
|
+
const requestedPort = eqArg
|
|
18
|
+
? Number(eqArg.slice("--port=".length))
|
|
19
|
+
: argv.includes("--port") && spacedArg
|
|
20
|
+
? Number(spacedArg)
|
|
21
|
+
: 0;
|
|
22
|
+
const noOpen = argv.includes("--no-open");
|
|
23
|
+
|
|
24
|
+
function summarize(data: ReturnType<typeof buildGraphData>) {
|
|
25
|
+
const concepts = Object.values(data.bundleGraphs).reduce((n, g) => n + g.concepts.length, 0);
|
|
26
|
+
const links = Object.values(data.bundleGraphs).reduce((n, g) => n + g.edges.length, 0);
|
|
27
|
+
return `${data.bundles.length} bundles, ${data.bundleEdges.length} refs, ${concepts} concepts, ${links} concept links`;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
let data = buildGraphData();
|
|
31
|
+
|
|
32
|
+
// The page pings /api/heartbeat every few seconds and beacons /api/close on unload.
|
|
33
|
+
// Nothing marks the server "connected" until the first heartbeat, so a slow first
|
|
34
|
+
// page-load never races the idle timeout below.
|
|
35
|
+
let connected = false;
|
|
36
|
+
let lastHeartbeat = Date.now();
|
|
37
|
+
const IDLE_TIMEOUT_MS = 10_000;
|
|
38
|
+
const HEARTBEAT_INTERVAL_MS = 3_000;
|
|
39
|
+
|
|
40
|
+
const fetchHandler = (req: Request) => {
|
|
41
|
+
const url = new URL(req.url);
|
|
42
|
+
if (url.pathname === "/")
|
|
43
|
+
return new Response(renderGraphPage(data), { headers: { "content-type": "text/html; charset=utf-8" } });
|
|
44
|
+
if (url.pathname === "/api/graph") {
|
|
45
|
+
if (url.searchParams.get("rebuild")) data = buildGraphData();
|
|
46
|
+
return Response.json(data);
|
|
47
|
+
}
|
|
48
|
+
if (url.pathname === "/api/file") {
|
|
49
|
+
const bundle = url.searchParams.get("bundle") ?? "";
|
|
50
|
+
const path = url.searchParams.get("path") ?? "";
|
|
51
|
+
const body = readConceptFile(bundle, path);
|
|
52
|
+
return body === undefined
|
|
53
|
+
? new Response("not found", { status: 404 })
|
|
54
|
+
: new Response(body, { headers: { "content-type": "text/plain; charset=utf-8" } });
|
|
55
|
+
}
|
|
56
|
+
if (url.pathname === "/api/heartbeat") {
|
|
57
|
+
connected = true;
|
|
58
|
+
lastHeartbeat = Date.now();
|
|
59
|
+
return new Response("ok");
|
|
60
|
+
}
|
|
61
|
+
if (url.pathname === "/api/close" && req.method === "POST") {
|
|
62
|
+
console.log("khb visualize — browser closed, shutting down.");
|
|
63
|
+
setTimeout(() => process.exit(0), 50); // let the response flush first
|
|
64
|
+
return new Response("ok");
|
|
65
|
+
}
|
|
66
|
+
return new Response("not found", { status: 404 });
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
let server;
|
|
70
|
+
try {
|
|
71
|
+
server = Bun.serve({ port: requestedPort, fetch: fetchHandler });
|
|
72
|
+
} catch (e) {
|
|
73
|
+
if (requestedPort && (e as { code?: string }).code === "EADDRINUSE") {
|
|
74
|
+
console.warn(`Port ${requestedPort} is busy — picking a free one instead.`);
|
|
75
|
+
server = Bun.serve({ port: 0, fetch: fetchHandler });
|
|
76
|
+
} else throw e;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
setInterval(() => {
|
|
80
|
+
if (connected && Date.now() - lastHeartbeat > IDLE_TIMEOUT_MS) {
|
|
81
|
+
console.log("khb visualize — browser tab gone, shutting down.");
|
|
82
|
+
process.exit(0);
|
|
83
|
+
}
|
|
84
|
+
}, HEARTBEAT_INTERVAL_MS).unref();
|
|
85
|
+
|
|
86
|
+
const url = `http://localhost:${server.port}`;
|
|
87
|
+
console.log(`khb visualize → ${url} (${summarize(data)})`);
|
|
88
|
+
|
|
89
|
+
// Open the default browser. If that fails the URL is already printed above, so a headless
|
|
90
|
+
// or locked-down box just falls back to copy-paste rather than erroring out.
|
|
91
|
+
if (!noOpen) {
|
|
92
|
+
const cmd =
|
|
93
|
+
process.platform === "win32"
|
|
94
|
+
? ["cmd", "/c", "start", "", url]
|
|
95
|
+
: process.platform === "darwin"
|
|
96
|
+
? ["open", url]
|
|
97
|
+
: ["xdg-open", url];
|
|
98
|
+
try {
|
|
99
|
+
Bun.spawn(cmd, { stdout: "ignore", stderr: "ignore" }).unref();
|
|
100
|
+
} catch {
|
|
101
|
+
console.warn("Could not launch a browser — open the URL above yourself.");
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
console.log(`The server exits on its own once you close the tab. Ctrl+C also works.`);
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: catalog
|
|
3
|
+
description: Turn one bundle's raw/ material into OKF concept docs — read each raw file, split it into sub-topics, label each with type/title/description/tags, link them, and register them in index.md. Uses parallel subagents when the runtime supports them. Use after khb ingest, or when the user asks to curate, catalog, organize, or write up a bundle.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Catalog a bundle
|
|
7
|
+
|
|
8
|
+
The second half of the pipeline: `khb ingest` produced faithful text in `raw/`; catalog
|
|
9
|
+
turns that text into **knowledge**. One bundle at a time, always.
|
|
10
|
+
|
|
11
|
+
The unit of output is the **concept** — one idea, one markdown file, OKF frontmatter,
|
|
12
|
+
registered in an index. A raw file is not a concept. A 40-page contract is a dozen concepts;
|
|
13
|
+
three meeting transcripts about the same decision are one. Splitting and merging is the
|
|
14
|
+
whole job, and it is why this pass needs a model and `khb ingest` does not.
|
|
15
|
+
|
|
16
|
+
`khb` has no `catalog` command. Nothing here is mechanical enough to script.
|
|
17
|
+
|
|
18
|
+
**Catalog classifies concepts and links them. It never reorganizes bundles.** The splitting
|
|
19
|
+
it does is *within* a bundle — raw files into concepts, concepts into subdirectories. A
|
|
20
|
+
bundle is a logical unit its owner defined (a person, a team, a project) and it holds many
|
|
21
|
+
topics by design, so finding three unrelated subjects in one bundle is the expected case,
|
|
22
|
+
not a problem to fix. Never create a bundle, move material to another bundle, or propose a
|
|
23
|
+
split because the contents look heterogeneous. If material clearly belongs to someone else,
|
|
24
|
+
it gets a `refs.md` line and stays where it is.
|
|
25
|
+
|
|
26
|
+
## 1. Scope the work
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
khb ingest <bundle> # if the user asked to catalog something not yet ingested
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
That only moves bytes for sources already declared in the bundle's `sources.yaml`; if the
|
|
33
|
+
material isn't declared yet, you are at the start of the [ingest skill](../ingest/SKILL.md),
|
|
34
|
+
not this one.
|
|
35
|
+
|
|
36
|
+
The worklist is `bundles/<bundle>/log.md`: **every row with a `raw` path and an empty
|
|
37
|
+
`curated` column.** Nothing else records what is outstanding, so work from the ledger rather
|
|
38
|
+
than from a directory listing — a `raw/` file whose row is already filled has been done.
|
|
39
|
+
|
|
40
|
+
Rows with an empty `raw` were never extracted; they are ingest's problem, not yours. If
|
|
41
|
+
there are many, say so and offer to fix ingest first.
|
|
42
|
+
|
|
43
|
+
## 2. Read the map before writing anything
|
|
44
|
+
|
|
45
|
+
Open the bundle's `index.md` and any nested indexes, and list the existing concepts: path,
|
|
46
|
+
`title`, `type`, `tags`. This is the **vocabulary**, and it does two jobs — it stops you
|
|
47
|
+
coining `billing` when `invoices` already exists, and it is what lets a new concept link to
|
|
48
|
+
an old one. Hold it in this thread; every subagent gets a copy.
|
|
49
|
+
|
|
50
|
+
Also decide the **grouping** now, before any file is written: which subdirectories the new
|
|
51
|
+
concepts go in (`contracts/`, `metrics/`, `notes/`, `playbooks/` — whatever fits this
|
|
52
|
+
domain; structure carries no fixed meaning). Reuse existing directories wherever they fit.
|
|
53
|
+
Inventing a new subdirectory per raw file is the classic failure here.
|
|
54
|
+
|
|
55
|
+
## 3. Seed wave — one file, checked by hand
|
|
56
|
+
|
|
57
|
+
Pick the most representative raw file and delegate it to **one subagent** with the prompt
|
|
58
|
+
below. Use a fast, economical model when the runtime exposes model selection; otherwise use
|
|
59
|
+
its default subagent. If subagents are unavailable, do the same work in the current thread.
|
|
60
|
+
Then read what it wrote. You are checking the shape, not the facts: right granularity,
|
|
61
|
+
frontmatter complete, titles that read like knowledge rather than filenames. Correct the
|
|
62
|
+
prompt if it is off, and take the concept titles it produced into your vocabulary before
|
|
63
|
+
going wide.
|
|
64
|
+
|
|
65
|
+
## 4. Bulk wave — the rest in parallel
|
|
66
|
+
|
|
67
|
+
Use one subagent per remaining raw file and start them concurrently when the runtime
|
|
68
|
+
supports parallel delegation. Group several small files into one task when they are
|
|
69
|
+
obviously the same subject; never split one file across two tasks. Without subagents,
|
|
70
|
+
process the files sequentially in the current thread and keep the same single-writer rules.
|
|
71
|
+
|
|
72
|
+
Parallel subagents cannot see each other's output, so two rules are absolute:
|
|
73
|
+
|
|
74
|
+
- **Every prompt carries the vocabulary.** Otherwise you get `q3-budget`, `budget-q3` and
|
|
75
|
+
`quarterly-budget` as three concepts.
|
|
76
|
+
- **Subagents write concept docs only.** `index.md`, `log.md` and `refs.md` have exactly one
|
|
77
|
+
writer — you, in step 5. Concurrent edits to a shared index silently lose rows.
|
|
78
|
+
|
|
79
|
+
Prompt each subagent with, substituting the bracketed parts:
|
|
80
|
+
|
|
81
|
+
> Read `bundles/<bundle>/raw/<file>.md`. It is ingested source material with a provenance
|
|
82
|
+
> header naming the original file.
|
|
83
|
+
>
|
|
84
|
+
> Split its content into distinct **concepts** — one idea per concept. A concept is
|
|
85
|
+
> something someone would ask a question about on its own. Do not summarize the document as
|
|
86
|
+
> a whole, and do not create a concept per section heading.
|
|
87
|
+
>
|
|
88
|
+
> Write each concept as its own markdown file in `bundles/<bundle>/<group>/`, named
|
|
89
|
+
> `<kebab-case-title>.md`. If that filename already exists and is about something else,
|
|
90
|
+
> append `-2`. Each file starts with OKF frontmatter:
|
|
91
|
+
>
|
|
92
|
+
> ```yaml
|
|
93
|
+
> ---
|
|
94
|
+
> type: <Table | Metric | Playbook | Decision Log | Reference | Contract | …>
|
|
95
|
+
> title: <display name>
|
|
96
|
+
> description: <one line, under 20 words>
|
|
97
|
+
> tags: [<lowercase kebab-case>]
|
|
98
|
+
> timestamp: <ISO from the source if it has one, else today>
|
|
99
|
+
> ---
|
|
100
|
+
> ```
|
|
101
|
+
>
|
|
102
|
+
> Then the body: the actual knowledge, in your own words, structured with headings. End with
|
|
103
|
+
> a `# Citations` section listing the raw file path and the `source:` value from its header.
|
|
104
|
+
>
|
|
105
|
+
> Link to related concepts with plain markdown links using bundle-root paths
|
|
106
|
+
> (`/metrics/churn.md`). You may link to any of these existing concepts:
|
|
107
|
+
> `<vocabulary: path — title — description, one per line>`
|
|
108
|
+
> Reuse an existing concept's subject rather than restating it: if this file only adds a
|
|
109
|
+
> detail to one of the above, say so in your reply instead of writing a near-duplicate.
|
|
110
|
+
> Never link outside this bundle.
|
|
111
|
+
>
|
|
112
|
+
> If the provenance header says `quality: low`, the text came from OCR or a transcript and
|
|
113
|
+
> may be garbled. Do not invent through it — quote what is legible, and flag what is not in
|
|
114
|
+
> your reply.
|
|
115
|
+
>
|
|
116
|
+
> If the file contains nothing worth keeping (a receipt, boilerplate, a duplicate), write
|
|
117
|
+
> nothing and say so.
|
|
118
|
+
>
|
|
119
|
+
> Reply with one line per concept you wrote: `<path> | <title> | <description>`, then any
|
|
120
|
+
> flags. Do not edit `index.md`, `log.md` or `refs.md`.
|
|
121
|
+
|
|
122
|
+
## 5. Merge — you, single writer
|
|
123
|
+
|
|
124
|
+
Everything in this step happens in this thread, sequentially, because every file it touches
|
|
125
|
+
is shared.
|
|
126
|
+
|
|
127
|
+
1. **Review and dedupe.** Read the concepts the wave produced. Two files covering the same
|
|
128
|
+
idea get merged into the better one and the loser deleted. A concept that only restates
|
|
129
|
+
an existing one gets folded into it instead.
|
|
130
|
+
2. **Cross-link.** Subagents could link to pre-existing concepts but not to each other's
|
|
131
|
+
output. Add the sibling links now — that is where most of a bundle's value ends up.
|
|
132
|
+
3. **Register every concept in `index.md`**, in the OKF form
|
|
133
|
+
`* [Title](path.md) - description`. **An unindexed concept is invisible to every query**
|
|
134
|
+
and lint will flag it. Add a nested `index.md` if a subdirectory grew past ~10 concepts.
|
|
135
|
+
4. **Fill the `curated` column in `log.md`** for every row you worked. Use the concept
|
|
136
|
+
paths, comma-separated. For a raw file you deliberately declined, write `declined` — a
|
|
137
|
+
row that stays empty will be offered as backlog forever.
|
|
138
|
+
5. **Foreign material → `refs.md`.** A raw file that turns out to belong to a different
|
|
139
|
+
bundle is not curated here: note the target bundle and the reason in `refs.md` and, if
|
|
140
|
+
the user agrees, add the source to that bundle's `sources.yaml`. Never inline-link across
|
|
141
|
+
bundles, and never copy the content over.
|
|
142
|
+
|
|
143
|
+
## 6. Verify
|
|
144
|
+
|
|
145
|
+
```
|
|
146
|
+
khb lint # index coverage, frontmatter, ref targets, no cross-bundle links
|
|
147
|
+
khb visualize # optional: refresh visualizer/graph.html
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Fix every error before finishing. Then report: concepts created, raw files declined, rows
|
|
151
|
+
still outstanding, and anything you flagged as low-quality and worth re-reading from source.
|
|
152
|
+
|
|
153
|
+
## Judgement notes
|
|
154
|
+
|
|
155
|
+
- **Curate selectively.** Raw is bulk; concepts are distilled. Most corpora are 80% receipts
|
|
156
|
+
and boilerplate. Declining is a real outcome, not a failure.
|
|
157
|
+
- **Granularity.** If two concepts are always read together, they are one. If one concept
|
|
158
|
+
has two `# Schema` sections, it is two.
|
|
159
|
+
- **Write knowledge, not summaries.** "This document discusses the retention policy" is
|
|
160
|
+
useless; the retention policy is what belongs in the file.
|
|
161
|
+
- **Never put content in an index.** Indexes route. Knowledge goes in a concept doc.
|
|
162
|
+
- **`quality: low` sources** deserve a look at the original before you commit their claims —
|
|
163
|
+
the `source:` path in the raw header is exactly for this. Reading a chart or a scanned
|
|
164
|
+
table with vision recovers what OCR dropped.
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: export
|
|
3
|
+
description: Export a KHB bundle as a standalone shareable folder with the common patterns injected. Use when the user wants to share or ship a single bundle.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Export a bundle
|
|
7
|
+
|
|
8
|
+
Bundles stay lean inside the hub because the common patterns live at hub root; export
|
|
9
|
+
injects those patterns so the folder works alone with any agent.
|
|
10
|
+
|
|
11
|
+
1. Run `khb export <bundle> [dest]` (default dest: `export/<bundle>/`). The command
|
|
12
|
+
**refuses to write into an existing destination** — re-exporting means removing the old
|
|
13
|
+
folder first, or passing a new `dest`.
|
|
14
|
+
2. The result is a miniature hub, with the bundle itself one level down:
|
|
15
|
+
|
|
16
|
+
```
|
|
17
|
+
<dest>/bundle/ the bundle, copied whole
|
|
18
|
+
<dest>/outer.index.md single-bundle router pointing at bundle/index.md
|
|
19
|
+
<dest>/AGENTS.md the contract, plus CLAUDE.md
|
|
20
|
+
<dest>/skills/ the canonical protocols (query, ingest, lint, …)
|
|
21
|
+
<dest>/.claude/skills/ discovery adapters, and .agents/skills/ likewise
|
|
22
|
+
<dest>/README.md provenance: what this is and when it was exported
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
3. Tell the user two things before they send it anywhere:
|
|
26
|
+
- `refs.md` entries pointing at other bundles **will not resolve** — the export is one
|
|
27
|
+
bundle, and its cross-bundle pointers now dangle.
|
|
28
|
+
- The copy is literal, so `raw/` and `log.md` go with it. `log.md` records **absolute
|
|
29
|
+
source paths** from the machine that ingested them, and `raw/` is uncurated source
|
|
30
|
+
material that was never written for an outside reader. Check both before sharing
|
|
31
|
+
outside the team, and prune if they say more than intended.
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: ingest
|
|
3
|
+
description: Acquire external material (folders, files, web pages, Confluence, ADO, git) into a KHB bundle's raw/ folder as markdown with provenance — one flat mechanical phase, no interpretation. Use when the user wants to add, import, dump, pull, or refresh source data in the knowledge base.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Ingest into KHB
|
|
7
|
+
|
|
8
|
+
**Ingest gets bytes into `bundles/<bundle>/raw/` as markdown with a provenance header.
|
|
9
|
+
That is all it does.** It is one flat phase, it is mechanical, and it ends the moment the
|
|
10
|
+
text exists. Deciding what the text *means* — splitting it into concepts, titling,
|
|
11
|
+
tagging, linking, indexing — is the [catalog skill](../catalog/SKILL.md), a separate step
|
|
12
|
+
you run afterwards.
|
|
13
|
+
|
|
14
|
+
Do not curate here. Do not create, split, or merge bundles here — routing material to a
|
|
15
|
+
bundle that exists is fine, reshaping the hub is not. If you find yourself reading a document
|
|
16
|
+
to understand it, you have left this skill.
|
|
17
|
+
|
|
18
|
+
## 1. Declare the sources
|
|
19
|
+
|
|
20
|
+
Ingest is bundle-first: material lands in one bundle, and you say where it comes from. Edit
|
|
21
|
+
`bundles/<bundle>/sources.yaml`:
|
|
22
|
+
|
|
23
|
+
```yaml
|
|
24
|
+
sources:
|
|
25
|
+
- type: folder # walk a directory tree
|
|
26
|
+
path: /abs/path/to/project-x
|
|
27
|
+
- type: files # a scattered, explicitly named set
|
|
28
|
+
paths:
|
|
29
|
+
- /abs/path/to/one.pdf
|
|
30
|
+
- /abs/path/to/two.xlsx
|
|
31
|
+
- type: web
|
|
32
|
+
urls:
|
|
33
|
+
- https://example.com/design-doc
|
|
34
|
+
# Types with no scripted ingester are still declared here, for the record —
|
|
35
|
+
# you pull them yourself in step 3.
|
|
36
|
+
- type: confluence
|
|
37
|
+
space: PROJX
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
If the user has not explicitly named the source locations, inspect the bundle's current
|
|
41
|
+
`sources.yaml`, then ask which files, folders, URLs, or services to ingest. Include any
|
|
42
|
+
existing declarations in the question so the user can confirm or replace them. Do not
|
|
43
|
+
infer sources from nearby files, edit `sources.yaml`, or run `khb ingest` until the user
|
|
44
|
+
answers.
|
|
45
|
+
|
|
46
|
+
Nothing is copied by declaring a source.
|
|
47
|
+
|
|
48
|
+
**Which bundle — take the first of these that applies, and do not go further:**
|
|
49
|
+
|
|
50
|
+
1. **The user named a bundle** → use it. A named bundle that does not exist is an error,
|
|
51
|
+
not an invitation to create one.
|
|
52
|
+
2. **The hub has bundles and exactly one plainly owns the material** → use it, and say
|
|
53
|
+
which you picked. If several could own it, ask which — this is the only bundle question
|
|
54
|
+
ingest ever asks.
|
|
55
|
+
3. **Anything else** — no bundle named, or the hub has no bundles at all → `default`,
|
|
56
|
+
created on the spot, without asking.
|
|
57
|
+
|
|
58
|
+
Never ask the user to name or create a bundle *for the ingest to land in*. `default` exists
|
|
59
|
+
so that question never has to be asked at this stage: bytes always have somewhere to go, and
|
|
60
|
+
which bundle owns them is a cheaper decision later, once the text exists and the user can see
|
|
61
|
+
what they actually have.
|
|
62
|
+
|
|
63
|
+
**The `default` bundle.** When no bundle is named, ingest targets `default` and creates it
|
|
64
|
+
if the hub has none — a first `khb ingest` never fails for want of a destination. It is not a
|
|
65
|
+
way around step 2: when a bundle in the hub plainly owns the material, that bundle wins. What
|
|
66
|
+
lands there is ordinary bundle content: catalog it like any other. Do **not** graduate it into new
|
|
67
|
+
bundles on your own — a bundle is a logical unit the user defines (a person, a team, a
|
|
68
|
+
project), so material leaves `default` only when the user says which bundle owns it. An
|
|
69
|
+
explicitly named bundle that doesn't exist is still an error — only `default` is conjured.
|
|
70
|
+
|
|
71
|
+
## 2. Run it
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
khb ingest # no bundle named → the 'default' bundle
|
|
75
|
+
khb ingest <bundle> # incremental: unchanged content hashes are skipped
|
|
76
|
+
khb ingest <bundle> --force # re-acquire everything
|
|
77
|
+
khb ingest <bundle> --skip-ocr # leave scans and images unread
|
|
78
|
+
khb ingest <bundle> --skip-audio # leave audio/video untranscribed
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
One command handles every scripted source in `sources.yaml` and extracts everything it can,
|
|
82
|
+
locally. Read the summary it prints — the counts are the state of the world:
|
|
83
|
+
|
|
84
|
+
| line | meaning |
|
|
85
|
+
|---|---|
|
|
86
|
+
| `unchanged, skipped` | already acquired at this exact content hash |
|
|
87
|
+
| `extracted` / `reused from the extraction cache` | converted now / converted by an earlier run or another bundle |
|
|
88
|
+
| `read by OCR` / `transcribed` | lossy routes — see quality, below |
|
|
89
|
+
| `marked quality: low` | verify these against the source when cataloging |
|
|
90
|
+
| `not extracted` | got a ledger row with an empty `raw`; the per-file line says why |
|
|
91
|
+
|
|
92
|
+
Above that summary, every file gets its own trace — announced *before* the work starts, so a
|
|
93
|
+
run that is taking minutes always names the file it is taking them on:
|
|
94
|
+
|
|
95
|
+
```
|
|
96
|
+
[ 7/94] D:\corpus\board-pack.pdf
|
|
97
|
+
extracting pdf …
|
|
98
|
+
no text layer, 12p — scanned, running OCR (seconds per page)
|
|
99
|
+
page 1/12 — 1843 chars
|
|
100
|
+
…
|
|
101
|
+
extracted → raw/folder/board-pack.pdf.md [tesseract.js @ 216dpi, quality: low] (48.1s)
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
There is no quiet mode, and this is deliberate: the trace is the audit trail for a pass that
|
|
105
|
+
rewrites `raw/`. When something looks wrong later, that line — tool, quality, elapsed — is
|
|
106
|
+
what tells you which file to distrust and which extractor to blame.
|
|
107
|
+
|
|
108
|
+
### What khb extracts
|
|
109
|
+
|
|
110
|
+
All of it runs locally and none of it contacts a model — that is the `AGENTS.md` division of
|
|
111
|
+
labor. khb converts bytes to text as cheaply as possible; your judgement is spent on
|
|
112
|
+
curation, not transcription.
|
|
113
|
+
|
|
114
|
+
| Format | Tool | Quality |
|
|
115
|
+
|---|---|---|
|
|
116
|
+
| `.md .txt .rst .adoc .html .csv .json .yaml` | copied verbatim | high |
|
|
117
|
+
| `.pdf` (born-digital) | `unpdf`, then `pdftotext` if on PATH | high |
|
|
118
|
+
| `.docx` / `.odt` / `.pptx` | `mammoth` / `fflate` / `fflate`, `pandoc` if on PATH | high |
|
|
119
|
+
| `.xlsx` | `fflate` → one markdown table per sheet | high |
|
|
120
|
+
| `.pdf` (scanned, no text layer) | `pdfium` + `tesseract.js`, automatically | **low** |
|
|
121
|
+
| `.png .jpg .webp .tif .gif` | `tesseract.js`, automatically | **low** |
|
|
122
|
+
| `.mp3 .wav .m4a .mp4 .mov .mkv` | local `whisper` / `faster-whisper` | **low** |
|
|
123
|
+
|
|
124
|
+
Extracted text is cached hub-wide by content hash at `inbox/extracted/<sha256>.md`, so the
|
|
125
|
+
same file appearing in two bundles converts once.
|
|
126
|
+
|
|
127
|
+
OCR and transcription need optional dependencies. When they are missing khb says so once and
|
|
128
|
+
records the affected files as pending rather than failing the run:
|
|
129
|
+
|
|
130
|
+
```
|
|
131
|
+
bun add @hyzyla/pdfium sharp tesseract.js # OCR — ~75 MB WASM, no system binary
|
|
132
|
+
pip install -U openai-whisper # transcription (faster-whisper also works)
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
Install them where `khb` resolves modules from — for a global install that is the khb
|
|
136
|
+
package directory, not your hub. khb prints the exact `cd … && bun add …` to use.
|
|
137
|
+
|
|
138
|
+
## 3. Sources khb cannot reach
|
|
139
|
+
|
|
140
|
+
Anything behind an authenticated API has no scripted ingester, because maintaining API
|
|
141
|
+
wrappers is not what this tool is for. Pull those yourself with the site's MCP server or
|
|
142
|
+
official CLI, and write the result into `raw/<type>/` **in exactly the format khb
|
|
143
|
+
produces** — same folder shape, same header — so the catalog pass cannot tell the
|
|
144
|
+
difference:
|
|
145
|
+
|
|
146
|
+
| Source | How |
|
|
147
|
+
|---|---|
|
|
148
|
+
| Confluence | MCP server or `confluence` CLI → `raw/confluence/<page>.md` |
|
|
149
|
+
| Azure DevOps | MCP/CLI → wiki pages, work items → `raw/ado/<item>.md` |
|
|
150
|
+
| GitHub / GitLab issues, PRs, wikis | `gh` / `glab` CLI → `raw/git/<thing>.md` |
|
|
151
|
+
| Source code repository | do **not** copy — record the location in `sources.yaml` and read it in place |
|
|
152
|
+
| A diagram or chart no OCR can read | vision read the image → `raw/images/<file>.md`, `extract_tool: claude-vision` |
|
|
153
|
+
|
|
154
|
+
Then add the row to `log.md` yourself (`source`, `sha256` if you have it, `fetched`, `raw`),
|
|
155
|
+
so the ledger stays the complete record regardless of who did the fetching.
|
|
156
|
+
|
|
157
|
+
### The provenance header — the contract
|
|
158
|
+
|
|
159
|
+
Every file in `raw/` starts with it. This is the reason ingest is a separate phase from
|
|
160
|
+
catalog: extraction is sometimes lossy, and curation must always be able to walk back to the
|
|
161
|
+
original bytes.
|
|
162
|
+
|
|
163
|
+
```yaml
|
|
164
|
+
---
|
|
165
|
+
source: /abs/path/to/original.pdf # or the url, or the tool query
|
|
166
|
+
fetched: 2026-07-23T09:14:02Z
|
|
167
|
+
sha256: db2ee470c95d
|
|
168
|
+
extract_tool: tesseract.js # what produced the text below
|
|
169
|
+
quality: low # high = real text; low = OCR or a transcript
|
|
170
|
+
---
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
`quality: low` is a standing invitation to distrust the body. When cataloging one of these
|
|
174
|
+
and the text reads thin, garbled, or contradictory, **open the `source:` file and read it
|
|
175
|
+
directly** — a vision pass over a chart or a scanned table recovers what OCR drops. Rewrite
|
|
176
|
+
the `raw/` file with `extract_tool: claude-vision` and `quality: high` when you do.
|
|
177
|
+
|
|
178
|
+
## 4. The ledger — `log.md`
|
|
179
|
+
|
|
180
|
+
Every bundle keeps its ingest ledger in `log.md` (OKF-reserved, so it is never mistaken for
|
|
181
|
+
a concept doc, and committed, so it survives `raw/` being deleted and re-derived).
|
|
182
|
+
|
|
183
|
+
| column | owner | meaning |
|
|
184
|
+
|---|---|---|
|
|
185
|
+
| `source` | khb | origin URI: absolute path, url, or tool query |
|
|
186
|
+
| `sha256` | khb | content hash (12-char prefix) — drives skip-unchanged, move detection and dedup |
|
|
187
|
+
| `fetched` | khb | ISO timestamp of last acquisition |
|
|
188
|
+
| `raw` | khb | bundle-relative `raw/` path; **empty = never extracted** |
|
|
189
|
+
| `curated` | agent | concept doc(s) distilled from it; **empty = catalog backlog** |
|
|
190
|
+
|
|
191
|
+
`khb ingest` maintains the first four and never touches `curated`.
|
|
192
|
+
|
|
193
|
+
### Moved and renamed sources
|
|
194
|
+
|
|
195
|
+
The bytes are a source's identity; the path is only where they live today. When a file
|
|
196
|
+
appears at a path the ledger has not seen, khb looks for an existing row with the same hash
|
|
197
|
+
whose own path has since disappeared. If exactly one matches, that row is **re-pointed** at
|
|
198
|
+
the new path: same `raw` file (concepts cite it by name, so it is never renamed), same
|
|
199
|
+
`curated`, and the raw file's `source:` provenance header is corrected. Nothing is
|
|
200
|
+
re-extracted and nothing re-enters the backlog.
|
|
201
|
+
|
|
202
|
+
Two cases are deliberately *not* treated as moves, because both would silently rewire
|
|
203
|
+
provenance:
|
|
204
|
+
|
|
205
|
+
- **Copy** — the twin row's path still exists, so these are two real sources with the same
|
|
206
|
+
bytes. Both are ingested; the run says so, and folding or declining the second is a
|
|
207
|
+
cataloging judgement.
|
|
208
|
+
- **Ambiguous** — several vanished rows share the hash, so which one moved here is
|
|
209
|
+
unknowable. The new file is ingested as its own source and the run says why.
|
|
210
|
+
|
|
211
|
+
Still on you, not khb: a source **modified in place** keeps its `curated` value, so the
|
|
212
|
+
concept derived from it does not re-enter the backlog even though its material changed.
|
|
213
|
+
Watch for `raw/` files whose content shifted and re-catalog them deliberately.
|
|
214
|
+
|
|
215
|
+
## Hand off
|
|
216
|
+
|
|
217
|
+
Ingest is done when the summary shows nothing unexpectedly pending. Report to the user what
|
|
218
|
+
landed, what didn't and why, and how many rows are uncurated — then continue with the
|
|
219
|
+
[catalog skill](../catalog/SKILL.md) to turn `raw/` into concept docs.
|
|
220
|
+
|
|
221
|
+
## Hygiene
|
|
222
|
+
|
|
223
|
+
- `raw/` is gitignored, derived, and **never canonical**. Never cite it in an answer.
|
|
224
|
+
- Never copy a bulk corpus into a bundle wholesale. Extraction shrinks documents to text;
|
|
225
|
+
source-code repos and media libraries stay where they are and get a `sources.yaml` entry.
|
|
226
|
+
- The same file in two bundles: one bundle owns it, the other gets a `refs.md` entry. Never
|
|
227
|
+
two copies. The content hash in `log.md` is how you spot it.
|
|
228
|
+
- `log.md` records absolute source paths. If those paths are themselves sensitive, gitignore
|
|
229
|
+
it before the first commit.
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: lint
|
|
3
|
+
description: Validate KHB structure (routing integrity, bundle shape, OKF conformance). Use after any structural edit, or when the user asks to check/validate/fix the knowledge base.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Lint KHB
|
|
7
|
+
|
|
8
|
+
1. Run `khb lint` from anywhere inside the hub — it walks up to `khb.json` to find the root.
|
|
9
|
+
2. Fix every ERROR (structure, routing, OKF frontmatter); judge warnings case by case
|
|
10
|
+
(broken index links may be intentional not-yet-written knowledge).
|
|
11
|
+
3. Re-run until 0 errors. If a fix changes root files **and the hub has a `meta` bundle**,
|
|
12
|
+
log it in `bundles/meta/notes/decisions.md`. No meta bundle means no decision log — do
|
|
13
|
+
not create one to have somewhere to write.
|
|
14
|
+
|
|
15
|
+
## The rules (L1–L9)
|
|
16
|
+
|
|
17
|
+
Enforced by `khb lint`. Combines KHB routing rules with
|
|
18
|
+
OKF v0.1 conformance (see the OKF spec). Reserved filenames: `index.md`, `log.md`
|
|
19
|
+
(OKF) and `refs.md` (KHB). Every other `.md` in a bundle — outside `raw/` —
|
|
20
|
+
is a **concept document**.
|
|
21
|
+
|
|
22
|
+
### Bundle shape
|
|
23
|
+
|
|
24
|
+
- L1. Every `bundles/<name>/` has: `index.md`, `refs.md`, `sources.yaml`. Concept docs
|
|
25
|
+
live in whatever subdirectory grouping fits the domain. No per-bundle AGENTS.md —
|
|
26
|
+
root `AGENTS.md` is the common contract; `khb export` injects it for standalone
|
|
27
|
+
sharing.
|
|
28
|
+
- L2. Bundle names: lowercase, digits, hyphens (`^[a-z0-9][a-z0-9-]*$`).
|
|
29
|
+
|
|
30
|
+
### Routing integrity
|
|
31
|
+
|
|
32
|
+
- L3. Every bundle is listed in `outer.index.md`; every bundle linked from
|
|
33
|
+
`outer.index.md` exists on disk.
|
|
34
|
+
- L4. Every concept doc is listed in at least one of the bundle's `index.md` files
|
|
35
|
+
(error). Index links pointing at missing files are a warning only — OKF treats
|
|
36
|
+
broken links as not-yet-written knowledge.
|
|
37
|
+
- L5. Index files contain routing only: headings, bullet/table link lines, one-line
|
|
38
|
+
descriptions. Paragraph-length prose is a violation (warning).
|
|
39
|
+
|
|
40
|
+
### Independence
|
|
41
|
+
|
|
42
|
+
- L6. No markdown link from a concept doc into another bundle's files. Cross-bundle
|
|
43
|
+
pointers live in `refs.md` only.
|
|
44
|
+
- L7. Every target bundle named in `refs.md` exists.
|
|
45
|
+
|
|
46
|
+
### Provenance
|
|
47
|
+
|
|
48
|
+
- L8. Files under `raw/` carry a provenance header (warning): frontmatter present, a
|
|
49
|
+
non-empty `source:`, and `quality:` — if set — reading exactly `high` or `low`.
|
|
50
|
+
`source` is what makes a bad extraction recoverable, so a raw file without one is
|
|
51
|
+
uncatalogable, not merely untidy.
|
|
52
|
+
|
|
53
|
+
### OKF conformance
|
|
54
|
+
|
|
55
|
+
- L9. Concept frontmatter is the machine-readable half of a concept, so it is validated
|
|
56
|
+
as data rather than glanced at:
|
|
57
|
+
- frontmatter block present, and **parses as YAML** (error) — a malformed block means
|
|
58
|
+
every field is silently lost.
|
|
59
|
+
- non-empty `type` (error) — the one OKF v0.1 §9 requirement. Its *value* stays
|
|
60
|
+
free-form: `Metric`, `Playbook`, `Runbook`, anything the domain needs.
|
|
61
|
+
- `title` and `description` present (warning) — indexes and index generators read them.
|
|
62
|
+
- `tags`, if present, is a YAML list of strings (error). `tags: "a, b"` is a string and
|
|
63
|
+
filters as one opaque value; `tags: [a, b]` is two tags.
|
|
64
|
+
- `timestamp`, if present, parses as an ISO-8601 datetime (warning).
|
|
65
|
+
- unknown top-level keys (warning). The known set is `type`, `title`, `description`,
|
|
66
|
+
`resource`, `tags`, `timestamp` — `resource` is optional and unvalidated, but it is
|
|
67
|
+
known, so it costs no warning. OKF is permissive and extra keys are legal, but `titel:`
|
|
68
|
+
is a typo that silently drops the field, and one warning line is cheaper than a field
|
|
69
|
+
nobody notices is missing.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: new-bundle
|
|
3
|
+
description: Create a new KHB bundle — a logical unit owned by a person, team, project or client. Use when the user wants to add a new owner/area to the knowledge base.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# New bundle
|
|
7
|
+
|
|
8
|
+
A bundle is a **logical unit, defined by whoever owns its material** — a person, a team, a
|
|
9
|
+
project, a client. It is not a subject classification. One bundle holds as many topics as
|
|
10
|
+
its owner has; topics are organized *inside* it with subdirectories, not by making more
|
|
11
|
+
bundles.
|
|
12
|
+
|
|
13
|
+
Creating a bundle is always a human decision. Never create one because material looks like
|
|
14
|
+
it belongs to a new subject, and never split an existing one on your own initiative — an
|
|
15
|
+
agent splits a bundle only when explicitly told to.
|
|
16
|
+
|
|
17
|
+
1. Pick a name: lowercase, digits, hyphens. Name the owner or the context, not the subject
|
|
18
|
+
(`team-payments`, `client-acme`, `notes`), and expect its scope line to list several
|
|
19
|
+
topics — that is correct, not a signal to split.
|
|
20
|
+
2. Run `khb new-bundle <name> "<one-line scope>"` — scaffolds from
|
|
21
|
+
`.bundle_template/` and registers in `outer.index.md`.
|
|
22
|
+
3. Fill in the "Route here when" column in `outer.index.md`. Write it as a trigger for
|
|
23
|
+
*whose* material this is, since one bundle answers for many topics.
|
|
24
|
+
4. Declare inputs in `sources.yaml` (see the ingest skill to pull them).
|
|
25
|
+
5. `khb lint`.
|