@panaversity/ksor 0.0.2 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/CHANGELOG.md +87 -0
  2. package/README.md +6 -4
  3. package/dist/cli.mjs +6348 -8
  4. package/dist/index.d.mts +3 -2
  5. package/dist/index.mjs +1 -36
  6. package/dist/src-CpDIVudJ.mjs +43 -0
  7. package/docs/index.md +10 -4
  8. package/package.json +18 -5
  9. package/schema/schema.sql +316 -0
  10. package/templates/scaffold/.agents/skills/add-sources/SKILL.md +4 -1
  11. package/templates/scaffold/.agents/skills/format-checker/SKILL.md +8 -1
  12. package/templates/scaffold/.agents/skills/format-checker/check.mjs +241 -9
  13. package/templates/scaffold/.agents/skills/intake-interview/SKILL.md +27 -7
  14. package/templates/scaffold/.claude/skills/add-sources/SKILL.md +4 -1
  15. package/templates/scaffold/.claude/skills/format-checker/SKILL.md +8 -1
  16. package/templates/scaffold/.claude/skills/format-checker/check.mjs +241 -9
  17. package/templates/scaffold/.claude/skills/intake-interview/SKILL.md +27 -7
  18. package/templates/scaffold/AGENTS.md +137 -14
  19. package/templates/scaffold/README.md +55 -3
  20. package/templates/scaffold/gitignore +3 -0
  21. package/templates/scaffold/instance.md +3 -2
  22. package/templates/scaffold/package.json +8 -1
  23. package/templates/scaffold/pnpm-workspace.yaml +21 -5
  24. package/templates/scaffold/system/site/app/(home)/page.tsx +2 -2
  25. package/templates/scaffold/system/site/app/docs/layout.tsx +2 -2
  26. package/templates/scaffold/system/site/components/footer-mark.tsx +22 -0
  27. package/templates/scaffold/system/site/lib/audience.ts +178 -0
  28. package/templates/scaffold/system/site/lib/shared.ts +14 -5
  29. package/templates/scaffold/system/site/lib/stage-knowledge.ts +301 -0
  30. package/templates/scaffold/system/site/source.config.ts +7 -1
  31. package/templates/scaffold/vercel.json +8 -0
@@ -23,6 +23,7 @@ const ALLOWED_KEYS = new Set([
23
23
  "title",
24
24
  "description",
25
25
  "status",
26
+ "visibility",
26
27
  "owner",
27
28
  "provenance",
28
29
  "effective",
@@ -96,9 +97,10 @@ function unquote(value) {
96
97
 
97
98
  /**
98
99
  * The frontmatter block, two levels deep (`ksor:` has children; `provenance:`
99
- * has list items). Returns null when there is no block at all, and collects
100
- * every line that is neither `key: value`, a list item, an indented
101
- * continuation, nor blank those mean the block was never closed.
100
+ * and `audiences:` have list items, collected under the key above them).
101
+ * Returns null when there is no block at all, and collects every line that is
102
+ * neither `key: value`, a list item, an indented continuation, nor blank —
103
+ * those mean the block was never closed.
102
104
  */
103
105
  function parseFrontmatter(text) {
104
106
  // An editor's byte-order mark is invisible to the author; it must not be
@@ -108,6 +110,7 @@ function parseFrontmatter(text) {
108
110
  if (!match) return null;
109
111
  const keys = new Map();
110
112
  const children = new Map();
113
+ const lists = new Map();
111
114
  const quoted = new Set();
112
115
  const malformedQuote = new Map();
113
116
  const malformed = [];
@@ -139,6 +142,7 @@ function parseFrontmatter(text) {
139
142
  }
140
143
  keys.set(current, unquote(top[2]));
141
144
  children.set(current, new Map());
145
+ lists.set(current, []);
142
146
  continue;
143
147
  }
144
148
  const nested = /^[ \t]+([A-Za-z_][\w-]*)\s*:\s*(.*)$/.exec(line);
@@ -146,10 +150,40 @@ function parseFrontmatter(text) {
146
150
  children.get(current).set(nested[1], unquote(nested[2]));
147
151
  continue;
148
152
  }
153
+ // A list item belongs to the key above it: audiences: is a list of
154
+ // audiences, and the rules that read it need the entries, not their count.
155
+ const item = /^[ \t]*-[ \t]+(.*)$/.exec(line);
156
+ if (item && current !== null) {
157
+ // YAML ends a plain scalar at ` #` — the comment is not part of the
158
+ // entry, and reading it as one refuses the documents instead of the
159
+ // list (found live 2026-08-18: `- public # the default` made every
160
+ // public document's visibility undeclared).
161
+ const value = /^["']/.test(item[1]) ? item[1] : item[1].replace(/\s+#.*$/, "");
162
+ lists.get(current).push(unquote(value));
163
+ continue;
164
+ }
165
+ // A dash glued to its value (`-internal`) is a list item to nobody —
166
+ // the indented-continuation escape below swallowed it while the build
167
+ // scanners stopped reading the list there: one green record, two
168
+ // different audience lists (review finding, 2026-08-19).
169
+ if (/^[ \t]*-\S/.test(line)) {
170
+ malformed.push(line.trim());
171
+ continue;
172
+ }
149
173
  if (/^[ \t]*-([ \t]|$)/.test(line) || /^[ \t]+\S/.test(line)) continue;
150
174
  malformed.push(line.trim());
151
175
  }
152
- return { keys, children, quoted, malformedQuote, duplicates, malformed, tightColons, tabIndents };
176
+ return {
177
+ keys,
178
+ children,
179
+ lists,
180
+ quoted,
181
+ malformedQuote,
182
+ duplicates,
183
+ malformed,
184
+ tightColons,
185
+ tabIndents,
186
+ };
153
187
  }
154
188
 
155
189
  /**
@@ -218,13 +252,14 @@ function linkTargets(body) {
218
252
  return raw.map((t) => (t.startsWith("<") && t.endsWith(">") ? t.slice(1, -1).trim() : t));
219
253
  }
220
254
 
255
+ /** Reports a broken target; returns the record document it resolves to, if any. */
221
256
  function checkLinkTarget(rel, docPath, target) {
222
257
  // Anything with a URI scheme (https:, mailto:, tel:, ftp:, …) or a
223
258
  // protocol-relative // host leaves the record on purpose — only relative
224
259
  // paths are the record's own links (review finding 2026-08-18: tel: was
225
260
  // reported as a dead file and //host as an escape).
226
- if (target === "" || target.startsWith("#") || target.startsWith("//")) return;
227
- if (/^[a-z][a-z0-9+.-]*:/i.test(target)) return;
261
+ if (target === "" || target.startsWith("#") || target.startsWith("//")) return null;
262
+ if (/^[a-z][a-z0-9+.-]*:/i.test(target)) return null;
228
263
  const resolved = path.resolve(path.dirname(docPath), target.split("#")[0]);
229
264
  if (!resolved.startsWith(knowledgeDir + path.sep) && resolved !== knowledgeDir) {
230
265
  problem(
@@ -240,9 +275,46 @@ function checkLinkTarget(rel, docPath, target) {
240
275
  "a record with dead internal links serves different truths by path",
241
276
  "fix the path or remove the link",
242
277
  );
278
+ } else if (resolved.endsWith(".md")) {
279
+ return resolved;
243
280
  }
281
+ return null;
282
+ }
283
+
284
+ // ---------------------------------------------------------------------------
285
+ // the audience model: who may read a document, declared in instance.md
286
+ // ---------------------------------------------------------------------------
287
+ const instanceMd = path.join(root, "instance.md");
288
+
289
+ /**
290
+ * The declared audiences, ordered least- to most-restricted, or null when the
291
+ * record declares none — and then every visibility rule below stays inert, so
292
+ * a record without an audience model behaves exactly as it did before the key
293
+ * existed. Read before the record itself: who may read what is a property of
294
+ * the whole record, which no single document can answer.
295
+ */
296
+ function readAudienceModel() {
297
+ if (!existsSync(instanceMd)) return null;
298
+ const fm = parseFrontmatter(readFileSync(instanceMd, "utf8"));
299
+ const audiences = fm?.lists.get("audiences") ?? [];
300
+ if (audiences.length === 0) return null;
301
+ return { audiences, defaultVisibility: scalarValue(fm, "default_visibility") ?? "" };
302
+ }
303
+
304
+ /**
305
+ * A plain scalar ends at ` #` — the rule the list items above already follow
306
+ * and both build scanners apply. The checker not applying it to values let
307
+ * `default_visibility: public # the default` build fine and fail `pnpm check`
308
+ * (review finding, 2026-08-19).
309
+ */
310
+ function scalarValue(fm, key) {
311
+ const value = fm.keys.get(key);
312
+ if (value === undefined || fm.quoted.has(key)) return value;
313
+ return value.replace(/\s+#.*$/, "").trim();
244
314
  }
245
315
 
316
+ const audienceModel = readAudienceModel();
317
+
246
318
  if (!existsSync(knowledgeDir)) {
247
319
  problem(
248
320
  "knowledge/",
@@ -389,6 +461,8 @@ if (!existsSync(knowledgeDir)) {
389
461
  }
390
462
 
391
463
  // frontmatter + links per document
464
+ const visibilityByPath = new Map();
465
+ const crossings = [];
392
466
  for (const p of mdFiles) {
393
467
  const rel = path.relative(root, p);
394
468
  const text = readFileSync(p, "utf8");
@@ -552,22 +626,116 @@ if (!existsSync(knowledgeDir)) {
552
626
  "a replaced document must hand the reader its successor — a broken pointer dead-ends them on stale truth",
553
627
  "fix the path (it resolves relative to this document), or write the successor first",
554
628
  );
629
+ } else {
630
+ crossings.push({ kind: "superseded_by", rel, from: p, to: resolved, target: successor });
631
+ }
632
+ }
633
+ // visibility: one audience per document, from the set instance.md declares
634
+ const visibility = fm.keys.get("visibility");
635
+ const listed = fm.lists.get("visibility") ?? [];
636
+ // A flow list ([a, b]) is already named by the shape rule above.
637
+ const flowList = !fm.quoted.has("visibility") && /^\[.*\]$/.test(visibility ?? "");
638
+ if (listed.length > 0) {
639
+ problem(
640
+ rel,
641
+ "visibility is one value, not a list",
642
+ "a list makes every document a set-membership question, and set intersection is where access-control bugs live — one document belongs to exactly one audience",
643
+ `write a single audience: visibility: ${audienceModel?.audiences.at(-1) ?? "<audience>"}`,
644
+ );
645
+ } else if (visibility !== undefined && !flowList) {
646
+ if (audienceModel === null) {
647
+ problem(
648
+ rel,
649
+ `visibility: ${visibility} — the record declares no audience model`,
650
+ "who may read a document is governance, not a comment: with no audiences: in instance.md nothing constrains this value, and every surface publishes the document to everyone regardless",
651
+ "add audiences: to instance.md (ordered least- to most-restricted, public first) with default_visibility:, or remove the visibility: key",
652
+ );
653
+ } else if (!audienceModel.audiences.includes(visibility)) {
654
+ problem(
655
+ rel,
656
+ `visibility "${visibility}" is not a declared audience`,
657
+ "the audience set is closed in instance.md — a value outside it names a build that does not exist, so the document reaches either nobody or everybody",
658
+ `use one of: ${audienceModel.audiences.join(", ")} — or remove the key to take the default (${audienceModel.defaultVisibility})`,
659
+ );
555
660
  }
556
661
  }
662
+ if (audienceModel !== null) {
663
+ visibilityByPath.set(p, visibility ?? audienceModel.defaultVisibility);
664
+ }
557
665
  }
558
666
  // links: resolve, and never escape the record
559
- for (const target of linkTargets(stripCode(text))) checkLinkTarget(rel, p, target);
667
+ for (const target of linkTargets(stripCode(text))) {
668
+ const to = checkLinkTarget(rel, p, target);
669
+ if (to !== null) crossings.push({ kind: "link", rel, from: p, to, target });
670
+ }
671
+ }
672
+
673
+ // Pointers across audiences: the leak no single build can catch, because the
674
+ // build that publishes the pointer has already dropped its target and cannot
675
+ // know it ever existed. Only the whole record sees both ends.
676
+ if (audienceModel !== null) {
677
+ const audienceOf = (file) => visibilityByPath.get(file) ?? audienceModel.defaultVisibility;
678
+ const tier = (file) => audienceModel.audiences.indexOf(audienceOf(file));
679
+ for (const { kind, rel, from, to, target } of crossings) {
680
+ const here = tier(from);
681
+ const there = tier(to);
682
+ // An undeclared audience at either end is already reported; comparing
683
+ // against a tier that does not exist would invent a second problem.
684
+ if (here === -1 || there === -1 || there <= here) continue;
685
+ const relTo = path.relative(root, to);
686
+ const both = `${relTo} is ${audienceOf(to)}, this document is ${audienceOf(from)}`;
687
+ if (kind === "link") {
688
+ problem(
689
+ rel,
690
+ `link to a more restricted document: ${target} — ${both}`,
691
+ "the build that publishes this link has already dropped its target: the link text and URL ship to readers who cannot open them, naming a document they were never meant to know exists",
692
+ `raise this document to ${audienceOf(to)}, widen ${relTo} to ${audienceOf(from)}, or remove the link`,
693
+ );
694
+ } else {
695
+ problem(
696
+ rel,
697
+ `superseded_by points at a more restricted document: ${target} — ${both}`,
698
+ "it strands the very readers the supersession exists to redirect: they are told this document is replaced, by a successor their build does not contain",
699
+ `widen ${relTo} to ${audienceOf(from)}, raise this document to ${audienceOf(to)}, or supersede it with a document its readers can reach`,
700
+ );
701
+ }
702
+ }
560
703
  }
561
704
  }
562
705
 
563
706
  // ---------------------------------------------------------------------------
564
707
  // instance.md: the identity of this SoR — format 1, closed key set
565
708
  // ---------------------------------------------------------------------------
566
- const INSTANCE_KEYS = new Set(["format", "name", "ksor", "site"]);
709
+ // The identity/site keys this checker owns, PLUS the four kernel config groups
710
+ // (database/embedding/retrieval/budgets) that `ksor serve`/`ingest` read. The
711
+ // groups are optional — a level-0 project that only runs `pnpm dev` declares
712
+ // none, and the default scaffold ships none (init stays database-free). They
713
+ // must be ALLOWED here so a project climbing to the served rung is not fought
714
+ // by its own CI: the kernel's instance parser (packages/content/src/instance.ts)
715
+ // REQUIRES `database:` to serve and is authoritative for the VALUES inside each
716
+ // group (dsn_env grammar, dim range, the vector_floor states). This checker
717
+ // only guards the key NAMES so a misspelled group or field is still caught.
718
+ const INSTANCE_KEYS = new Set([
719
+ "format",
720
+ "name",
721
+ "ksor",
722
+ "site",
723
+ "audiences",
724
+ "default_visibility",
725
+ "database",
726
+ "embedding",
727
+ "retrieval",
728
+ "budgets",
729
+ ]);
567
730
  const INSTANCE_KSOR_KEYS = new Set(["requires", "scaffolded"]);
568
731
  const INSTANCE_SITE_KEYS = new Set(["url"]);
732
+ // Nested field names mirror the kernel's instance schema; the kernel validates
733
+ // their values (this checker stays dependency-free and cannot import it).
734
+ const INSTANCE_DATABASE_KEYS = new Set(["dsn_env", "tenant_id"]);
735
+ const INSTANCE_EMBEDDING_KEYS = new Set(["provider", "model", "dim"]);
736
+ const INSTANCE_RETRIEVAL_KEYS = new Set(["vector_floor", "keyword_floor"]);
737
+ const INSTANCE_BUDGETS_KEYS = new Set(["maximum_response_characters"]);
569
738
 
570
- const instanceMd = path.join(root, "instance.md");
571
739
  if (!existsSync(instanceMd)) {
572
740
  problem(
573
741
  "instance.md",
@@ -673,9 +841,70 @@ if (!existsSync(instanceMd)) {
673
841
  );
674
842
  }
675
843
  }
844
+ // the audience model: ordered, public first, and never without its default
845
+ const audiences = fm.lists.get("audiences") ?? [];
846
+ const defaultVisibility = scalarValue(fm, "default_visibility") ?? "";
847
+ if (fm.keys.has("audiences") && audiences.length === 0) {
848
+ const value = fm.keys.get("audiences");
849
+ problem(
850
+ "instance.md",
851
+ value === ""
852
+ ? "audiences: declares no audiences"
853
+ : `audiences is a list, not a value: ${value}`,
854
+ "the audience list is the record's whole access model, ordered least- to most-restricted — with nothing in it, no document's visibility: can be answered",
855
+ "write it as list items:\n audiences:\n - public\n - internal",
856
+ );
857
+ } else if (audiences.length > 0) {
858
+ if (audiences[0] !== "public") {
859
+ problem(
860
+ "instance.md",
861
+ `audiences: does not start with public (it starts with "${audiences[0]}")`,
862
+ "the order is the restriction level, and that ordering is what makes an internal build mean public-and-internal with no further configuration — public is the least restricted tier by definition",
863
+ "list public first, then each narrower audience in turn",
864
+ );
865
+ }
866
+ const seen = new Set();
867
+ for (const audience of audiences) {
868
+ if (seen.has(audience)) {
869
+ problem(
870
+ "instance.md",
871
+ `duplicate audience: ${audience}`,
872
+ "an audience's position in the list is its restriction level — named twice, it has two levels and neither can be trusted",
873
+ `keep one ${audience} entry`,
874
+ );
875
+ }
876
+ seen.add(audience);
877
+ }
878
+ if (defaultVisibility === "") {
879
+ problem(
880
+ "instance.md",
881
+ "audiences: without default_visibility:",
882
+ "there is no safe inference for a document that declares no audience: guessing the widest leaks the first document whose key is forgotten, guessing the narrowest hides the record from everyone it was written for",
883
+ `add default_visibility: — one of ${audiences.join(", ")} — the audience a document belongs to when it declares none`,
884
+ );
885
+ } else if (!audiences.includes(defaultVisibility)) {
886
+ problem(
887
+ "instance.md",
888
+ `default_visibility "${defaultVisibility}" is not one of the declared audiences`,
889
+ "every document without a visibility: key takes this value — a default outside the list puts most of the record in an audience that does not exist",
890
+ `use one of: ${audiences.join(", ")}`,
891
+ );
892
+ }
893
+ } else if (fm.keys.has("default_visibility")) {
894
+ problem(
895
+ "instance.md",
896
+ "default_visibility: without audiences:",
897
+ "a default audience with no audience list is a setting with nothing to select from — the owner believes the record has a visibility model while every surface publishes every document to everyone",
898
+ "add audiences: (ordered least- to most-restricted, public first), or remove default_visibility:",
899
+ );
900
+ }
676
901
  for (const [parent, allowed] of [
677
902
  ["ksor", INSTANCE_KSOR_KEYS],
678
903
  ["site", INSTANCE_SITE_KEYS],
904
+ ["database", INSTANCE_DATABASE_KEYS],
905
+ ["embedding", INSTANCE_EMBEDDING_KEYS],
906
+ ["retrieval", INSTANCE_RETRIEVAL_KEYS],
907
+ ["budgets", INSTANCE_BUDGETS_KEYS],
679
908
  ]) {
680
909
  for (const key of fm.children.get(parent)?.keys() ?? []) {
681
910
  if (!allowed.has(key)) {
@@ -757,6 +986,9 @@ if (existsSync(siteDir)) {
757
986
  (p) =>
758
987
  !p.includes(`${path.sep}.next${path.sep}`) &&
759
988
  !p.includes(`${path.sep}.source${path.sep}`) &&
989
+ // The per-audience stage: generated copies of the record a build makes
990
+ // for one audience, never authored content (specs/ksor/visibility).
991
+ !p.includes(`${path.sep}.staged-knowledge${path.sep}`) &&
760
992
  !p.includes(`${path.sep}out${path.sep}`),
761
993
  )
762
994
  .filter((p) => p.toLowerCase().endsWith(".md") || p.toLowerCase().endsWith(".mdx"));
@@ -1,18 +1,18 @@
1
1
  ---
2
2
  name: intake-interview
3
- description: The first conversation with the owner of this Knowledge System of Record — five questions that define what it is authoritative for, then write instance.md together. Use when the owner asks to set up, configure, or "get started with" this project, when instance.md still contains its scaffold placeholder text, or when the scope of the corpus is unclear.
3
+ description: The first conversation with the owner of this Knowledge System of Record — six questions that define what it is authoritative for and who may read it, then write instance.md together. Use when the owner asks to set up, configure, or "get started with" this project, when instance.md still contains its scaffold placeholder text, or when the scope of the corpus is unclear.
4
4
  metadata:
5
- version: "1.1.0"
5
+ version: "1.3.0"
6
6
  ---
7
7
 
8
8
  # Intake interview
9
9
 
10
10
  `instance.md` is the identity of this Knowledge System of Record, and its
11
- prose will one day be the agent surface's system prompt. Do not draft it from
12
- guesses — interview the owner, one question at a time, and write down what
13
- they actually say.
11
+ prose IS the agent surface's system prompt (`ksor serve` wires it into the MCP
12
+ server's instructions). Do not draft it from guesses — interview the owner, one
13
+ question at a time, and write down what they actually say.
14
14
 
15
- ## The five questions
15
+ ## The six questions
16
16
 
17
17
  Ask these one at a time; follow up until each answer is concrete enough to
18
18
  act on:
@@ -30,6 +30,18 @@ act on:
30
30
  5. **Strictness** — "When the record doesn't cover a question, how firmly
31
31
  should it decline? ('Not in this corpus' is a correct answer here —
32
32
  confirm the owner wants that behavior and where they want it softened.)"
33
+ 6. **Audiences** — "Does every reader of this record see every document? If
34
+ not, what are the audiences, from most public to most restricted?" A yes
35
+ is the common answer and the whole answer: write no `audiences:` key and
36
+ nothing about the project changes. A list means writing it into
37
+ `instance.md`'s frontmatter — ordered least- to most-restricted with
38
+ `public` first, plus `default_visibility:` naming the audience a document
39
+ takes when it says nothing (there is no safe guess, so the checker
40
+ requires it). Tell the owner what the key does and does not do:
41
+ documents carry `visibility:` and builds are made per audience, but
42
+ anyone who can clone the repository reads everything in it — if someone
43
+ must not read a document and can clone, that document belongs in a
44
+ different repository.
33
45
 
34
46
  ## Then write
35
47
 
@@ -37,7 +49,15 @@ act on:
37
49
  record's **display title**, the human name every page will lead with
38
50
  ("Acme Operations Handbook", not the slug) — then the authority sentence,
39
51
  boundary, audience, and strictness — plain prose, written for a reader
40
- who must act on it. Do not touch the frontmatter keys.
52
+ who must act on it. Leave the identity frontmatter keys alone; two things
53
+ are written there when they apply: an audience model from question 6, as
54
+ `audiences:` (a list) and `default_visibility:` (`pnpm check` holds the
55
+ record to it from that moment on); and — only when the owner stands up the
56
+ served MCP rung — the `database:`/`embedding:`/`retrieval:` blocks (see
57
+ `AGENTS.md` → "Serving to agents"; that is a later climb, not part of this
58
+ interview). The strictness answer from question 5 is the intent behind the
59
+ `retrieval.vector_floor` on that climb, measured by `ksor calibrate` — capture
60
+ it in the prose now so it is ready.
41
61
  - Restart `pnpm dev` afterwards so the site picks the new title up, and
42
62
  show the owner their name on the page.
43
63
  - Offer to capture the source list from question 4 as the first real
@@ -5,19 +5,25 @@ here; every coding agent reads this file first.
5
5
 
6
6
  ## The two worlds
7
7
 
8
- | Path | What it is |
9
- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
10
- | `knowledge/` | **the record** — governed markdown, the owner's world, the product |
11
- | `system/` | **the system** — all code that serves the record |
12
- | `instance.md` | what this SoR is authoritative for; its prose is the future agent surface's system prompt. Its `name:` is the machine identity (llms.txt, future citations) and its body `# H1` is the DISPLAY TITLE every page leads with — both read when the server or build STARTS, so restart `pnpm dev` after changing either (found live 2026-08-18) |
8
+ | Path | What it is |
9
+ | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
10
+ | `knowledge/` | **the record** — governed markdown, the owner's world, the product |
11
+ | `system/` | **the system** — all code that serves the record |
12
+ | `instance.md` | what this SoR is authoritative for; its prose IS the agent surface's system prompt (`ksor serve` wires the body into the MCP server's instructions). Its `name:` is the machine identity (llms.txt, citations) and its body `# H1` is the DISPLAY TITLE every page leads with — both read when the server or build STARTS, so restart `pnpm dev` after changing either (found live 2026-08-18) |
13
13
 
14
14
  The record survives the system: `knowledge/` must stay readable and complete
15
15
  even if `system/` is deleted. Dependency flows one way — the system reads the
16
16
  record; the record never references the system.
17
17
 
18
- `instance.md` carries a closed key set — `format`, `name`, `ksor`, `site` —
19
- and everything that matters about it is the prose below that frontmatter;
20
- `pnpm check` names any other key rather than ignoring it.
18
+ `instance.md` carries a closed key set — `format`, `name`, `ksor`, `site`,
19
+ the optional pair `audiences` + `default_visibility` (the record's reader
20
+ audiences, ordered least- to most-restricted with `public` first, and the one
21
+ a document takes when it names none — declared together or not at all), and
22
+ the four serve-config blocks `database` / `embedding` / `retrieval` / `budgets`
23
+ (present only once you climb to the served MCP rung — see "Serving to agents"
24
+ below; a `pnpm dev`-only project declares none). Everything else that matters
25
+ about the instance is the prose below the frontmatter; `pnpm check` names any
26
+ other key rather than ignoring it.
21
27
 
22
28
  ## Critical rules
23
29
 
@@ -27,17 +33,121 @@ and everything that matters about it is the prose below that frontmatter;
27
33
  2. **`knowledge/` is CommonMark `.md` only.** No `.mdx`, no `meta.json`, no
28
34
  framework files. A document must read cleanly in any markdown viewer.
29
35
  3. **Never edit generated files** — `system/site/.source/`, `.next/`, `out/`,
36
+ `system/site/.staged-knowledge/` (a build's per-audience copy of the
37
+ record — edit `knowledge/`, or the next build erases the change),
30
38
  lockfiles by hand.
31
39
 
32
40
  ## Commands (run at the repo root)
33
41
 
34
42
  ```sh
35
- pnpm install # once, after cloning or scaffolding
43
+ pnpm install # once, after cloning or scaffolding (also fetches the pinned `ksor` tool)
36
44
  pnpm dev # the site, hot-reloading, at http://localhost:3000
37
45
  pnpm build # static site into system/site/out/
38
46
  pnpm check # the format checker — run before handing off any knowledge change
39
47
  ```
40
48
 
49
+ ## Serving to agents — the MCP rung (needs Postgres + a provider key)
50
+
51
+ `ksor serve` runs an MCP server over the record so agents get cited retrieval
52
+ with honest abstention. It is the climbed rung — not required for `pnpm dev`.
53
+ Stand it up in this order (each step's errors explain how to fix themselves):
54
+
55
+ 1. **Configure `instance.md`.** Add the serve blocks to the frontmatter
56
+ (`pnpm check` accepts them; the kernel validates their values):
57
+
58
+ ```yaml
59
+ database:
60
+ dsn_env: KSOR_DB_URL # the NAME of the env var holding the DSN — never the DSN itself
61
+ embedding:
62
+ provider: gemini # default; the seam, not the vendor, is the contract
63
+ model: gemini-embedding-001
64
+ dim: 1536 # ≤ 2000 for the pgvector HNSW index
65
+ retrieval:
66
+ vector_floor: uncalibrated # see step 6; `uncalibrated` REFUSES every serve until you paste a number
67
+ ```
68
+
69
+ 2. **Provision Postgres** with the `vector` extension (`CREATE EXTENSION vector`),
70
+ e.g. a Neon database. Export the DSN under the name `dsn_env` chose, plus the
71
+ provider key:
72
+
73
+ ```sh
74
+ export KSOR_DB_URL='postgresql://…' # the var instance.md names
75
+ export GEMINI_API_KEY='…' # the embedding provider key
76
+ ```
77
+
78
+ 3. **Apply the schema:** `pnpm schema` (creates tables, indexes, and the
79
+ ingest role).
80
+
81
+ 4. **Authorize ingest:** `pnpm grant` — writes the one row row-level security
82
+ requires before any write to this corpus is allowed. Idempotent, and
83
+ `pnpm exec ksor grant --instance instance.md --revoke` withdraws it.
84
+
85
+ This is a separate, named act on purpose: applying the schema and
86
+ authorizing writes are different decisions, and a schema step that granted
87
+ itself write access would make the tool its own authorizer. Apply the schema
88
+ and ingest as the SAME Postgres login (the ingest role is granted to whoever
89
+ applied the DDL).
90
+
91
+ 5. **Ingest:** `pnpm ingest` — embeds `knowledge/` into a fresh generation and
92
+ activates it (`--flip`). Safe to re-run (see the generation model below).
93
+
94
+ 6. **Calibrate the abstention floor** (only if `vector_floor: uncalibrated`):
95
+ `pnpm exec ksor calibrate --instance instance.md` prints a recommended
96
+ `vector_floor` measurement; paste the number into `instance.md`'s `retrieval:`
97
+ block and re-run. A corpus that declares no `retrieval:` block serves with the
98
+ gate OFF (honest: it will not refuse out-of-corpus questions).
99
+
100
+ 7. **Serve:** `pnpm serve`.
101
+
102
+ ```sh
103
+ pnpm schema # apply the DDL (once)
104
+ pnpm grant # authorize ingest for this corpus (once)
105
+ pnpm ingest # embed knowledge/ into a generation and activate it
106
+ pnpm serve # run the MCP server; any other verb: pnpm exec ksor <verb>
107
+ ```
108
+
109
+ ### How serving updates work (the generation model)
110
+
111
+ Each `ksor ingest` builds a **fresh generation** (invisible until activated) and
112
+ carries every unchanged embedding forward from the last complete generation —
113
+ so **re-ingest is safe and cheap**; only changed or failed chunks re-embed.
114
+ `--flip` swaps the active pointer, guarded by a catastrophic-shrink check
115
+ (`KSOR_MAX_SHRINK`, default `0.15` — a flip that drops more than 15% of nodes
116
+ refuses; override with `KSOR_ALLOW_SHRINK=1` when the shrink is intended). The
117
+ previous generation stays as a rollback target; `pnpm exec ksor gc` collects
118
+ abandoned ones.
119
+
120
+ ### Serving safely (fail-closed posture)
121
+
122
+ `pnpm serve` binds **loopback with auth off** — safe for local use. A **public**
123
+ bind refuses to boot unless auth is configured (`KSOR_SSO_URL` +
124
+ `KSOR_MCP_RESOURCE_URL` + `KSOR_JWT_ALLOWED_AUDIENCES`, making it an OAuth
125
+ Resource Server) OR you deliberately set `KSOR_ALLOW_PUBLIC_UNAUTHENTICATED=1`.
126
+ Never let a dropped auth variable silently ship an open door. On a non-loopback
127
+ bind, set `KSOR_ALLOWED_HOSTS` / `KSOR_ALLOWED_ORIGINS`; on more than one
128
+ replica, set a shared `KSOR_SNAPSHOT_KEYS` (unset ⇒ a per-process key, so a
129
+ search token minted by one replica fails on another).
130
+
131
+ Two things worth being deliberate about:
132
+
133
+ - **`KSOR_ALLOW_PUBLIC_UNAUTHENTICATED=1` serves your whole record to anyone
134
+ who can reach the port.** It exists for deployments fronted by your own
135
+ gateway or network policy. If nothing else is in front, do not set it.
136
+ - **Set `KSOR_SSO_ISSUER` when your SSO stamps a stable `iss`.** Audience is
137
+ always enforced against `KSOR_JWT_ALLOWED_AUDIENCES`; naming the issuer adds
138
+ one more check for the cost of one variable.
139
+
140
+ ## Publishing
141
+
142
+ `pnpm build` emits a fully static site (`system/site/out/`) deployable to
143
+ any host — Vercel reads the shipped `vercel.json` (deploy from the repo
144
+ ROOT, never `system/site/`), and every other host just serves the folder.
145
+ `KSOR_BASE_PATH=/repo pnpm build` targets sub-path hosting. With
146
+ `audiences:` declared, plain `pnpm build` is always the public tier;
147
+ `KSOR_AUDIENCE=<audience> pnpm build` builds a wider tier that belongs
148
+ behind that audience's own access control, never on a public host.
149
+ Details in README → Deploying.
150
+
41
151
  ## Writing knowledge
42
152
 
43
153
  - One document per file under `knowledge/`; the path is the document's
@@ -51,10 +161,23 @@ pnpm check # the format checker — run before handing off any knowledge c
51
161
  - Frontmatter: `title` and `status` (`draft | review | approved | superseded`)
52
162
  are required. `owner` and `provenance` (a list naming real sources) are
53
163
  strongly encouraged — they become required as this project climbs the
54
- governance ladder. `description`, `order` (sidebar position), `effective`
55
- (the date the document takes effect) and `superseded` (a legacy marker —
56
- prefer `status`) are available. No other keys; never `id:` or `name:` — the
57
- path is the identity.
164
+ governance ladder. `description`, `visibility` (below), `order` (sidebar
165
+ position), `effective` (the date the document takes effect) and `superseded`
166
+ (a legacy marker — prefer `status`) are available. No other keys; never
167
+ `id:` or `name:` — the path is the identity.
168
+ - `visibility:` names the one audience a document belongs to — a single value
169
+ from `instance.md`'s `audiences:`, never a list, and orthogonal to `status:`
170
+ (an approved document can be restricted, and a draft is not hidden). Leave
171
+ it off and the document takes `default_visibility`. The key does nothing
172
+ until `instance.md` declares `audiences:`; once it does, `pnpm check`
173
+ refuses any link or `superseded_by:` pointing from a wider audience at a
174
+ narrower one — the leak no single build can catch, because the build that
175
+ publishes the link has already dropped its target.
176
+
177
+ **Publication, not authorship: anyone who can clone the repository reads
178
+ every document regardless of frontmatter; if someone must not read a
179
+ document and can clone, the answer is a second repository.**
180
+
58
181
  - A replaced document is marked `status: superseded` with `superseded_by:`
59
182
  pointing at its successor — superseded documents are never deleted.
60
183
  - Images and assets live in `knowledge/` beside the document that uses them,
@@ -94,7 +217,7 @@ You own `system/site/` outright — these are the seams, cheapest first:
94
217
  and the home-page mark are the same file.
95
218
  - **Anything deeper** — edit the site like the Next.js app it is; the only
96
219
  rule that survives customization is critical rule 1. The whole shell is
97
- replaceable behind a four-clause contract (a themed Docusaurus shell with
220
+ replaceable behind a five-clause contract (a themed Docusaurus shell with
98
221
  a swap recipe lives in the ksor repository under `workbench/shells/`).
99
222
 
100
223
  ## What this project owns