@panaversity/ksor 0.0.39 → 0.0.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +965 -0
- package/README.md +29 -9
- package/dist/checker/check-main.mjs +14049 -0
- package/dist/cli.mjs +11528 -5213
- package/dist/gateway-api-CF4ED9_g-BQusM_dK.mjs +10895 -0
- package/dist/gateway.d.mts +52 -13
- package/dist/gateway.mjs +2 -2
- package/dist/index.d.mts +1 -1
- package/dist/index.mjs +1 -1
- package/dist/{src-pl4aOpVs.mjs → src-dqpI-p1a.mjs} +1 -0
- package/docs/authorization.md +8 -6
- package/docs/deploying.md +36 -25
- package/docs/index.md +26 -13
- package/docs/ingesting.md +70 -22
- package/docs/tool-surface.md +69 -16
- package/package.json +4 -3
- package/schema/migrations/2.4-2.5__okf-profile.sql +114 -0
- package/schema/schema.sql +77 -14
- package/templates/scaffold/.agents/skills/add-sources/SKILL.md +63 -18
- package/templates/scaffold/.agents/skills/format-checker/SKILL.md +42 -33
- package/templates/scaffold/.agents/skills/format-checker/check.mjs +13827 -1314
- package/templates/scaffold/.agents/skills/intake-interview/SKILL.md +65 -27
- package/templates/scaffold/.agents/skills/make-slides/SKILL.md +7 -5
- package/templates/scaffold/.agents/skills/make-summary/SKILL.md +13 -6
- package/templates/scaffold/.claude/skills/add-sources/SKILL.md +63 -18
- package/templates/scaffold/.claude/skills/format-checker/SKILL.md +42 -33
- package/templates/scaffold/.claude/skills/format-checker/check.mjs +13827 -1314
- package/templates/scaffold/.claude/skills/intake-interview/SKILL.md +65 -27
- package/templates/scaffold/.claude/skills/make-slides/SKILL.md +7 -5
- package/templates/scaffold/.claude/skills/make-summary/SKILL.md +13 -6
- package/templates/scaffold/.github/workflows/validate.yml +9 -1
- package/templates/scaffold/.ksor/governance.yaml +17 -0
- package/templates/scaffold/AGENTS.md +293 -119
- package/templates/scaffold/Dockerfile +5 -1
- package/templates/scaffold/README.md +160 -42
- package/templates/scaffold/env.example +37 -6
- package/templates/scaffold/gitignore +14 -6
- package/templates/scaffold/instance.md +21 -17
- package/templates/scaffold/knowledge/governance-ladder.md +11 -2
- package/templates/scaffold/knowledge/index.md +9 -0
- package/templates/scaffold/knowledge/surfaces/for-agents.md +13 -6
- package/templates/scaffold/knowledge/surfaces/for-people.md +7 -6
- package/templates/scaffold/knowledge/surfaces/index.md +4 -20
- package/templates/scaffold/knowledge/surfaces/overview.md +25 -0
- package/templates/scaffold/knowledge/what-is-a-ksor.md +6 -5
- package/templates/scaffold/knowledge/what-is-a-ksor.summary.md +4 -0
- package/templates/scaffold/package.json +3 -4
- package/templates/scaffold/pnpm-lock.yaml +3 -0
- package/templates/scaffold/system/gateways/content.ts +13 -0
- package/templates/scaffold/system/site/app/(home)/page.tsx +2 -2
- package/templates/scaffold/system/site/app/.well-known/mcp/server.json/route.ts +10 -0
- package/templates/scaffold/system/site/app/docs/[[...slug]]/page.tsx +134 -91
- package/templates/scaffold/system/site/app/global.css +198 -6
- package/templates/scaffold/system/site/app/layout.tsx +8 -3
- package/templates/scaffold/system/site/app/llms-full.txt/route.ts +13 -7
- package/templates/scaffold/system/site/app/llms.txt/route.ts +12 -7
- package/templates/scaffold/system/site/app/md/[[...slug]]/route.ts +26 -25
- package/templates/scaffold/system/site/components/code-block.tsx +87 -0
- package/templates/scaffold/system/site/components/embed.tsx +276 -0
- package/templates/scaffold/system/site/components/footer-mark.tsx +3 -2
- package/templates/scaffold/system/site/components/governance.tsx +219 -87
- package/templates/scaffold/system/site/components/mdx.tsx +10 -2
- package/templates/scaffold/system/site/components/record-index.tsx +5 -5
- package/templates/scaffold/system/site/components/record-stack.tsx +10 -9
- package/templates/scaffold/system/site/components/record-views.tsx +6 -17
- package/templates/scaffold/system/site/components/sidebar-status.tsx +19 -18
- package/templates/scaffold/system/site/lib/alert-rule.ts +214 -0
- package/templates/scaffold/system/site/lib/attachment-rule.ts +6 -1
- package/templates/scaffold/system/site/lib/attachments.ts +0 -28
- package/templates/scaffold/system/site/lib/audience-rule.ts +15 -21
- package/templates/scaffold/system/site/lib/audience.ts +42 -146
- package/templates/scaffold/system/site/lib/embed-rule.ts +255 -0
- package/templates/scaffold/system/site/lib/governance.ts +339 -225
- package/templates/scaffold/system/site/lib/index-routes.ts +125 -0
- package/templates/scaffold/system/site/lib/lifecycle-rule.ts +52 -0
- package/templates/scaffold/system/site/lib/lock.ts +282 -0
- package/templates/scaffold/system/site/lib/order-rule.ts +37 -0
- package/templates/scaffold/system/site/lib/record-href.ts +68 -0
- package/templates/scaffold/system/site/lib/record-link.tsx +26 -0
- package/templates/scaffold/system/site/lib/rules-version.ts +11 -0
- package/templates/scaffold/system/site/lib/shared.ts +67 -104
- package/templates/scaffold/system/site/lib/sim-rule.ts +49 -0
- package/templates/scaffold/system/site/lib/source.ts +256 -186
- package/templates/scaffold/system/site/lib/stage-knowledge.ts +638 -467
- package/templates/scaffold/system/site/lib/stage-manifest.ts +128 -0
- package/templates/scaffold/system/site/package.json +1 -0
- package/templates/scaffold/system/site/record/actor.ts +23 -0
- package/templates/scaffold/system/site/record/check.ts +571 -0
- package/templates/scaffold/system/site/record/citations.ts +312 -0
- package/templates/scaffold/system/site/record/frontmatter.ts +134 -0
- package/templates/scaffold/system/site/record/git-ledger.ts +171 -0
- package/templates/scaffold/system/site/record/hygiene.ts +320 -0
- package/templates/scaffold/system/site/record/index-file.ts +150 -0
- package/templates/scaffold/system/site/record/index.ts +103 -0
- package/templates/scaffold/system/site/record/instance.ts +257 -0
- package/templates/scaffold/system/site/record/instant.ts +43 -0
- package/templates/scaffold/system/site/record/ledger.ts +694 -0
- package/templates/scaffold/system/site/record/load.ts +129 -0
- package/templates/scaffold/system/site/record/lock.ts +306 -0
- package/templates/scaffold/system/site/record/near-miss.ts +37 -0
- package/templates/scaffold/system/site/record/policy.ts +414 -0
- package/templates/scaffold/system/site/record/profile.ts +535 -0
- package/templates/scaffold/system/site/record/refusal.ts +106 -0
- package/templates/scaffold/system/site/record/yaml-file.ts +103 -0
- package/templates/scaffold/system/site/source.config.ts +109 -23
- package/dist/gateway-api-CmIthmJS-IUA9qS-T.mjs +0 -3225
- package/templates/scaffold/system/site/lib/denial-rule.ts +0 -220
- package/templates/scaffold/system/site/lib/page-order.ts +0 -93
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a concept's body says about other things: GFM footnotes (the one
|
|
3
|
+
* extension to CommonMark, record spec §2.3 — per-claim citation keyed on
|
|
4
|
+
* `sources[].id`) and links in both OKF §6.1 forms. The code-stripping is
|
|
5
|
+
* the scaffold checker's, carried because its two review findings (links in
|
|
6
|
+
* fenced and indented code checked as real; a document-wide span strip
|
|
7
|
+
* pairing stray backticks pages apart) are ours to keep closed.
|
|
8
|
+
*/
|
|
9
|
+
import type { Refusal } from "./refusal";
|
|
10
|
+
|
|
11
|
+
const FOOTNOTE_REF = /\[\^([^\]\s]+)\](?!:)/g;
|
|
12
|
+
const FOOTNOTE_DEF = /^[ \t]{0,3}\[\^([^\]\s]+)\]:/gm;
|
|
13
|
+
|
|
14
|
+
/** Every footnote reference and definition must be keyed on a `sources[].id`. */
|
|
15
|
+
export function checkFootnotes(
|
|
16
|
+
path: string,
|
|
17
|
+
body: string,
|
|
18
|
+
sourceIds: readonly string[],
|
|
19
|
+
): Refusal[] {
|
|
20
|
+
const prose = stripCode(body);
|
|
21
|
+
const known = new Set(sourceIds);
|
|
22
|
+
const seen = new Set<string>();
|
|
23
|
+
const refusals: Refusal[] = [];
|
|
24
|
+
const report = (label: string, form: string): void => {
|
|
25
|
+
if (known.has(label) || seen.has(label)) return;
|
|
26
|
+
seen.add(label);
|
|
27
|
+
refusals.push({
|
|
28
|
+
slug: "ksor-footnote-unkeyed",
|
|
29
|
+
path,
|
|
30
|
+
why: `the footnote ${form} \`[^${label}]\` matches no \`sources[].id\` — a citation must trace to a declared source`,
|
|
31
|
+
fix: `add a source with \`id: ${label}\`, or change the label to one of: ${[...known].join(", ") || "(none declared)"}`,
|
|
32
|
+
});
|
|
33
|
+
};
|
|
34
|
+
for (const m of prose.matchAll(FOOTNOTE_DEF)) report(m[1] ?? "", "definition");
|
|
35
|
+
for (const m of prose.matchAll(FOOTNOTE_REF)) report(m[1] ?? "", "reference");
|
|
36
|
+
return refusals;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// The reference definitions that inline `[text][label]` links point at.
|
|
40
|
+
const REFERENCE_DEFINITION =
|
|
41
|
+
/^[ \t]{0,3}\[[^\]^]+\]:[ \t]*(<[^<>\n]*>|\S+)[ \t]*(?:"[^"]*"|'[^']*'|\([^)]*\))?[ \t]*$/gm;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Every inline destination in a body, scanned rather than matched, because
|
|
45
|
+
* link TEXT nests and a regex character class cannot.
|
|
46
|
+
*
|
|
47
|
+
* The class this replaces was `\[[^\]]*\]\(`, which stops at the FIRST `]`.
|
|
48
|
+
* On `[](secret/plan.md)` it therefore saw the image and
|
|
49
|
+
* never the link around it — so a public document could point at a restricted
|
|
50
|
+
* target and escape `ksor-link-widens`, `ksor-link-dead` and
|
|
51
|
+
* `ksor-link-escapes` at once, silently, because no rule ever received the
|
|
52
|
+
* link. A plain `[the [2026] policy](hr/leave.md)` disappeared the same way.
|
|
53
|
+
* Both found in review, 2026-08-25.
|
|
54
|
+
*
|
|
55
|
+
* Every destination is reported, inner and outer: an image inside a link is
|
|
56
|
+
* one link with two targets, and BOTH are governed — the asset is staged and
|
|
57
|
+
* the destination decides the audience.
|
|
58
|
+
*/
|
|
59
|
+
function inlineDestinations(prose: string): string[] {
|
|
60
|
+
const found: string[] = [];
|
|
61
|
+
const open: number[] = [];
|
|
62
|
+
let i = 0;
|
|
63
|
+
while (i < prose.length) {
|
|
64
|
+
const ch = prose[i];
|
|
65
|
+
if (ch === "\\") {
|
|
66
|
+
i += 2;
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
if (ch === "[") {
|
|
70
|
+
open.push(i);
|
|
71
|
+
i += 1;
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
if (ch !== "]") {
|
|
75
|
+
i += 1;
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
// A `]` with no `[` before it is ordinary text, and so is one whose `(`
|
|
79
|
+
// never closes: neither names a destination, so neither is reported.
|
|
80
|
+
open.pop();
|
|
81
|
+
const parsed = prose[i + 1] === "(" ? destinationAt(prose, i + 2) : null;
|
|
82
|
+
if (parsed === null) {
|
|
83
|
+
i += 1;
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
found.push(parsed.target);
|
|
87
|
+
i = parsed.end;
|
|
88
|
+
}
|
|
89
|
+
return found;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** The destination that starts just after `](`, or null when it does not close. */
|
|
93
|
+
function destinationAt(prose: string, start: number): { target: string; end: number } | null {
|
|
94
|
+
let i = start;
|
|
95
|
+
while (i < prose.length && /\s/.test(prose[i] ?? "")) i += 1;
|
|
96
|
+
let target = "";
|
|
97
|
+
if (prose[i] === "<") {
|
|
98
|
+
const close = prose.indexOf(">", i + 1);
|
|
99
|
+
const newline = prose.indexOf("\n", i + 1);
|
|
100
|
+
if (close === -1 || (newline !== -1 && newline < close)) return null;
|
|
101
|
+
target = prose.slice(i, close + 1);
|
|
102
|
+
i = close + 1;
|
|
103
|
+
} else {
|
|
104
|
+
const from = i;
|
|
105
|
+
while (i < prose.length && !/[\s)]/.test(prose[i] ?? "")) i += 1;
|
|
106
|
+
target = prose.slice(from, i);
|
|
107
|
+
if (target === "") return null;
|
|
108
|
+
}
|
|
109
|
+
// An optional "double", 'single' or (paren) title, then the closing paren.
|
|
110
|
+
while (i < prose.length && /\s/.test(prose[i] ?? "")) i += 1;
|
|
111
|
+
const quote = prose[i];
|
|
112
|
+
if (quote === '"' || quote === "'" || quote === "(") {
|
|
113
|
+
const closer = quote === "(" ? ")" : quote;
|
|
114
|
+
const close = prose.indexOf(closer, i + 1);
|
|
115
|
+
if (close === -1) return null;
|
|
116
|
+
i = close + 1;
|
|
117
|
+
while (i < prose.length && /\s/.test(prose[i] ?? "")) i += 1;
|
|
118
|
+
}
|
|
119
|
+
if (prose[i] !== ")") return null;
|
|
120
|
+
return { target, end: i + 1 };
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** The record's own link destinations in a body: no scheme, no `//host`, no bare fragment. */
|
|
124
|
+
export function linkTargets(body: string): string[] {
|
|
125
|
+
const prose = stripCode(body);
|
|
126
|
+
const raw: string[] = [];
|
|
127
|
+
raw.push(...inlineDestinations(prose));
|
|
128
|
+
for (const m of prose.matchAll(REFERENCE_DEFINITION)) raw.push(m[1] ?? "");
|
|
129
|
+
return raw
|
|
130
|
+
.map((t) => (t.startsWith("<") && t.endsWith(">") ? t.slice(1, -1).trim() : t))
|
|
131
|
+
.filter(
|
|
132
|
+
(t) =>
|
|
133
|
+
t !== "" && !t.startsWith("#") && !t.startsWith("//") && !/^[a-z][a-z0-9+.-]*:/i.test(t),
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* The bundle-relative id a link resolves to — bundle-absolute (`/x/y`,
|
|
139
|
+
* against `knowledge/`) or relative (against the source's directory), `.md`
|
|
140
|
+
* optional, fragment and trailing slash dropped. Null when it escapes the
|
|
141
|
+
* bundle.
|
|
142
|
+
*/
|
|
143
|
+
export function resolveLink(sourceId: string, target: string): string | null {
|
|
144
|
+
const raw = target.split("#")[0] ?? "";
|
|
145
|
+
let decoded = raw;
|
|
146
|
+
try {
|
|
147
|
+
decoded = decodeURIComponent(raw);
|
|
148
|
+
} catch {
|
|
149
|
+
// A malformed %-escape names nothing; it resolves to a path that does not exist.
|
|
150
|
+
}
|
|
151
|
+
const clean = decoded.replace(/\.md$/, "");
|
|
152
|
+
const base = clean.startsWith("/") ? [] : sourceId.split("/").slice(0, -1);
|
|
153
|
+
const segments = [...base];
|
|
154
|
+
for (const seg of clean.replace(/^\/+/, "").split("/")) {
|
|
155
|
+
if (seg === "" || seg === ".") continue;
|
|
156
|
+
if (seg === "..") {
|
|
157
|
+
if (segments.length === 0) return null;
|
|
158
|
+
segments.pop();
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
segments.push(seg);
|
|
162
|
+
}
|
|
163
|
+
return segments.join("/");
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* The column a line's first non-space character sits at, tabs advancing to the
|
|
168
|
+
* next four-column stop. CommonMark measures block structure in COLUMNS, and a
|
|
169
|
+
* tab-indented continuation paragraph is the same paragraph as a
|
|
170
|
+
* four-space-indented one.
|
|
171
|
+
*/
|
|
172
|
+
function indentOf(line: string): number {
|
|
173
|
+
let col = 0;
|
|
174
|
+
for (const ch of line) {
|
|
175
|
+
if (ch === " ") col += 1;
|
|
176
|
+
else if (ch === "\t") col += 4 - (col % 4);
|
|
177
|
+
else break;
|
|
178
|
+
}
|
|
179
|
+
return col;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
const LIST_MARKER = /^[ \t]*([-*+]|\d{1,9}[.)])([ \t]*)(.?)/;
|
|
183
|
+
/** `* * *` and `- - -`: a thematic break, which takes precedence over the item it looks like. */
|
|
184
|
+
const THEMATIC_BREAK = /^[ \t]*([-*_])(?:[ \t]*\1){2,}[ \t]*$/;
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* The column at which the list item this line opens holds its CONTENT, or null
|
|
188
|
+
* when the line opens no item. Everything indented to that column belongs to
|
|
189
|
+
* the item, and code inside the item starts four columns further right — which
|
|
190
|
+
* is the whole of why the indent below is measured against it.
|
|
191
|
+
*/
|
|
192
|
+
function itemContentColumn(line: string, indent: number): number | null {
|
|
193
|
+
if (THEMATIC_BREAK.test(line)) return null;
|
|
194
|
+
const m = LIST_MARKER.exec(line);
|
|
195
|
+
if (m === null) return null;
|
|
196
|
+
const marker = m[1] ?? "";
|
|
197
|
+
const spaces = m[2] ?? "";
|
|
198
|
+
const rest = m[3] ?? "";
|
|
199
|
+
// `-x` is a word, not a bullet: a marker is followed by space or line end.
|
|
200
|
+
if (spaces === "" && rest !== "") return null;
|
|
201
|
+
const after = indent + marker.length;
|
|
202
|
+
let padding = 0;
|
|
203
|
+
for (const ch of spaces) padding += ch === "\t" ? 4 - ((after + padding) % 4) : 1;
|
|
204
|
+
// One to four spaces of padding set the content column; five or more open an
|
|
205
|
+
// indented code block INSIDE the item, whose content starts one past the marker.
|
|
206
|
+
return rest === "" || padding > 4 ? after + 1 : after + padding;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
const FENCE_OPEN = /^[ \t]*(`{3,}|~{3,})/;
|
|
210
|
+
const FENCE_CLOSE = /^[ \t]*(`{3,}|~{3,})[ \t]*$/;
|
|
211
|
+
|
|
212
|
+
/** The line closing a fence opened with `bar` inside a container at `content`, or null. */
|
|
213
|
+
function fenceCloses(
|
|
214
|
+
lines: readonly string[],
|
|
215
|
+
from: number,
|
|
216
|
+
bar: string,
|
|
217
|
+
content: number,
|
|
218
|
+
): number | null {
|
|
219
|
+
for (let i = from; i < lines.length; i += 1) {
|
|
220
|
+
const line = lines[i] ?? "";
|
|
221
|
+
const run = FENCE_CLOSE.exec(line)?.[1];
|
|
222
|
+
if (run === undefined) continue;
|
|
223
|
+
if (run[0] === bar[0] && run.length >= bar.length && indentOf(line) - content <= 3) return i;
|
|
224
|
+
}
|
|
225
|
+
return null;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* Code is prose about links, never links. Strips fenced blocks (``` and ~~~),
|
|
230
|
+
* indented code, and code spans per paragraph — and NOTHING else, because a
|
|
231
|
+
* line this misreads as code is a line no rule judges. `checkFootnotes` and
|
|
232
|
+
* `checkLinks` both read what comes back, so anything wrongly swallowed here
|
|
233
|
+
* escapes `ksor-footnote-unkeyed`, `ksor-link-widens`, `ksor-link-dead` and
|
|
234
|
+
* `ksor-link-escapes` at once, silently — a public document pointing at a
|
|
235
|
+
* restricted one with nothing red. Two shapes did exactly that (review,
|
|
236
|
+
* 2026-08-25), and both are answered by reading indentation the way CommonMark
|
|
237
|
+
* does: from the CONTAINER's content column rather than the line start.
|
|
238
|
+
*
|
|
239
|
+
* Where the exact rule is out of reach the line is KEPT. A link checked inside
|
|
240
|
+
* something that was really code is a false refusal an author can see and fix;
|
|
241
|
+
* a link never checked is a governance hole nothing reports.
|
|
242
|
+
*/
|
|
243
|
+
export function stripCode(text: string): string {
|
|
244
|
+
const lines = text.replace(/\r\n?/g, "\n").split("\n");
|
|
245
|
+
const kept: string[] = [];
|
|
246
|
+
/** The content columns of the list items currently open, outermost first. */
|
|
247
|
+
const items: number[] = [];
|
|
248
|
+
let blank = true;
|
|
249
|
+
let indented = false;
|
|
250
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
251
|
+
const line = lines[i] ?? "";
|
|
252
|
+
if (line.trim() === "") {
|
|
253
|
+
blank = true;
|
|
254
|
+
kept.push(line);
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
const indent = indentOf(line);
|
|
258
|
+
const opens = itemContentColumn(line, indent);
|
|
259
|
+
// A block starting left of the open item's content leaves the item, and so
|
|
260
|
+
// does a new marker. A line that merely continues a paragraph — no blank
|
|
261
|
+
// before it, no marker of its own — is a lazy continuation and closes nothing.
|
|
262
|
+
while (items.length > 0 && indent < (items.at(-1) ?? 0) && (blank || opens !== null)) {
|
|
263
|
+
items.pop();
|
|
264
|
+
}
|
|
265
|
+
const content = items.at(-1) ?? 0;
|
|
266
|
+
const relative = indent - content;
|
|
267
|
+
|
|
268
|
+
// A fence opens up to three columns past its container's content, so a
|
|
269
|
+
// fenced sample inside a list item is still a fence.
|
|
270
|
+
const open = relative <= 3 ? FENCE_OPEN.exec(line)?.[1] : undefined;
|
|
271
|
+
if (open !== undefined) {
|
|
272
|
+
// An unclosed fence is a stray backtick run, not a block. The state used
|
|
273
|
+
// to survive to end of input, so ONE stray ``` in prose took every link
|
|
274
|
+
// and footnote after it out of both checks — half a document unjudged,
|
|
275
|
+
// with no signal at all. Only the line itself is dropped now, and dropped
|
|
276
|
+
// rather than kept so its backticks cannot pair with a later run in the
|
|
277
|
+
// code-span pass below.
|
|
278
|
+
const close = fenceCloses(lines, i + 1, open, content);
|
|
279
|
+
if (close !== null) {
|
|
280
|
+
i = close;
|
|
281
|
+
} else {
|
|
282
|
+
// That dropped line is still TEXT, so it ends whatever ran before it:
|
|
283
|
+
// an indented code block cannot interrupt a paragraph, and reading the
|
|
284
|
+
// next indented line as code would hide it all over again.
|
|
285
|
+
blank = false;
|
|
286
|
+
indented = false;
|
|
287
|
+
}
|
|
288
|
+
continue;
|
|
289
|
+
}
|
|
290
|
+
// Indented code starts four columns past the CONTAINER's content, never
|
|
291
|
+
// four past the line start: a four-space-indented continuation paragraph is
|
|
292
|
+
// what CommonMark REQUIRES inside a list item, and reading it as code hid
|
|
293
|
+
// its links from every rule. A marker line is never read as code, which
|
|
294
|
+
// only ever checks more.
|
|
295
|
+
if (relative >= 4 && opens === null) {
|
|
296
|
+
if (blank || indented) {
|
|
297
|
+
indented = true;
|
|
298
|
+
continue;
|
|
299
|
+
}
|
|
300
|
+
} else {
|
|
301
|
+
indented = false;
|
|
302
|
+
}
|
|
303
|
+
if (opens !== null) items.push(opens);
|
|
304
|
+
blank = false;
|
|
305
|
+
kept.push(line);
|
|
306
|
+
}
|
|
307
|
+
return kept
|
|
308
|
+
.join("\n")
|
|
309
|
+
.split(/\n{2,}/)
|
|
310
|
+
.map((paragraph) => paragraph.replace(/(`+)[^`]*?\1/g, " "))
|
|
311
|
+
.join("\n\n");
|
|
312
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Split a document into its YAML frontmatter and its body — with a real YAML
|
|
3
|
+
* parser. The line scanners it will replace (research/okf-native.md §1) could
|
|
4
|
+
* not read a nested `ksor:` block and failed silently on it.
|
|
5
|
+
*/
|
|
6
|
+
import { parseAllDocuments, YAMLParseError } from "yaml";
|
|
7
|
+
|
|
8
|
+
import type { Refusal } from "./refusal";
|
|
9
|
+
import { firstNonPlain, isPlainMapping } from "./yaml-file";
|
|
10
|
+
|
|
11
|
+
export type Split =
|
|
12
|
+
| {
|
|
13
|
+
readonly ok: true;
|
|
14
|
+
readonly frontmatter: Readonly<Record<string, unknown>> | null;
|
|
15
|
+
/** The YAML between the fences, byte-exact and comments intact — what `ksor migrate` rewrites. */
|
|
16
|
+
readonly block: string;
|
|
17
|
+
readonly body: string;
|
|
18
|
+
}
|
|
19
|
+
| { readonly ok: false; readonly refusal: Refusal };
|
|
20
|
+
|
|
21
|
+
const SLUG = "ksor-frontmatter-invalid";
|
|
22
|
+
/** A fence line: three dashes, trailing blanks tolerated, on the opening and the closing line alike. */
|
|
23
|
+
const FENCE = /^---[ \t]*$/;
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* An editor's byte-order mark is invisible to the author; CR and CRLF are the
|
|
27
|
+
* checkout's, not the record's. The BOM is why this runs before the fence is
|
|
28
|
+
* looked for at all: a BOM-prefixed file whose opening fence is therefore not
|
|
29
|
+
* at byte 0 reads as having no frontmatter, and serves its YAML as content
|
|
30
|
+
* (review finding, 2026-08-19).
|
|
31
|
+
*/
|
|
32
|
+
export function normalizeText(text: string): string {
|
|
33
|
+
return text.replace(/^\uFEFF/, "").replace(/\r\n?/g, "\n");
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* `frontmatter` is null when the text has no fence at all, `{}` for an empty
|
|
38
|
+
* block. The body is everything after the closing fence's newline, byte-exact.
|
|
39
|
+
*
|
|
40
|
+
* The block ends at the first fence LINE after the opening one — found by
|
|
41
|
+
* walking real `\n` boundaries, never a multiline regex, because JS `^`/`$`
|
|
42
|
+
* also break on U+2028/U+2029 and YAML 1.2 does not (found in review: a scalar
|
|
43
|
+
* containing U+2028 was cut mid-line and the real fence published as body).
|
|
44
|
+
* The consequence is a rule of the profile: a frontmatter may not contain a
|
|
45
|
+
* bare `---` line, block scalars included (record spec §2).
|
|
46
|
+
*/
|
|
47
|
+
export function splitFrontmatter(text: string, path: string): Split {
|
|
48
|
+
const normalized = normalizeText(text);
|
|
49
|
+
const lines = normalized.split("\n");
|
|
50
|
+
if (!FENCE.test(lines[0] ?? "")) {
|
|
51
|
+
return { ok: true, frontmatter: null, block: "", body: normalized };
|
|
52
|
+
}
|
|
53
|
+
const closeAt = lines.findIndex((line, i) => i > 0 && FENCE.test(line));
|
|
54
|
+
if (closeAt === -1) {
|
|
55
|
+
return refuse(
|
|
56
|
+
path,
|
|
57
|
+
"the frontmatter opens with `---` but no closing `---` line follows",
|
|
58
|
+
"add a `---` line after the last frontmatter key",
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
const block = lines.slice(1, closeAt).join("\n");
|
|
62
|
+
const body = lines.slice(closeAt + 1).join("\n");
|
|
63
|
+
|
|
64
|
+
let value: unknown;
|
|
65
|
+
try {
|
|
66
|
+
const docs = parseAllDocuments(block, {
|
|
67
|
+
schema: "core",
|
|
68
|
+
uniqueKeys: true,
|
|
69
|
+
logLevel: "silent",
|
|
70
|
+
});
|
|
71
|
+
if (docs.length > 1) {
|
|
72
|
+
return refuse(
|
|
73
|
+
path,
|
|
74
|
+
"the frontmatter is not valid YAML: a second document marker (`---` or `...`) sits inside the block",
|
|
75
|
+
"one document per frontmatter — remove the marker",
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
const doc = docs[0];
|
|
79
|
+
const problem = doc?.errors[0] ?? doc?.warnings[0];
|
|
80
|
+
if (problem !== undefined) throw problem;
|
|
81
|
+
value = doc?.toJS();
|
|
82
|
+
} catch (error) {
|
|
83
|
+
return refuse(
|
|
84
|
+
path,
|
|
85
|
+
`the frontmatter is not valid YAML: ${reasonOf(error)}`,
|
|
86
|
+
"fix the YAML — a stray colon, an unclosed bracket, a duplicated key, tab indentation or an unknown `!!tag` are the usual causes",
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
if (value === null || value === undefined) return { ok: true, frontmatter: {}, block, body };
|
|
90
|
+
if (!isPlainMapping(value)) {
|
|
91
|
+
return refuse(
|
|
92
|
+
path,
|
|
93
|
+
"the frontmatter is not a mapping (a list or a bare scalar was found)",
|
|
94
|
+
"write `key: value` pairs between the fences",
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
const tagged = firstNonPlain(value, "");
|
|
98
|
+
if (tagged !== null) {
|
|
99
|
+
return refuse(
|
|
100
|
+
path,
|
|
101
|
+
`the frontmatter value at \`${tagged}\` is not plain data — a \`!!tag\` turned it into something no reader expects`,
|
|
102
|
+
"write strings, numbers, booleans, lists and mappings only; timestamps stay quoted strings",
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
return { ok: true, frontmatter: value, block, body };
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** The message as an author reads it: the first line, minus any `SomeError:` class prefix. */
|
|
109
|
+
function reasonOf(error: unknown): string {
|
|
110
|
+
const first = error instanceof YAMLParseError ? error.message : String(error);
|
|
111
|
+
return first.split("\n")[0]?.replace(/^\w*Error: /, "") ?? "unreadable";
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function refuse(path: string, why: string, fix: string): Split {
|
|
115
|
+
return { ok: false, refusal: { slug: SLUG, path, why, fix } };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* The document's frontmatter as the file holds it — the bytes between the
|
|
120
|
+
* fences, unparsed — or null when it has none or an unclosed one.
|
|
121
|
+
*
|
|
122
|
+
* The markdown twin serves this INTACT rather than a re-serialisation of the
|
|
123
|
+
* parsed object, and so does the door's `read` tool: the profile preserves
|
|
124
|
+
* unknown keys (record spec §2.6), and anything this codebase re-emits can
|
|
125
|
+
* only carry the keys it thought of. It
|
|
126
|
+
* reads the block `splitFrontmatter` already found, so there is exactly one
|
|
127
|
+
* walk deciding where a fence ends — a second one that disagreed would
|
|
128
|
+
* publish a different document from the one the checker read.
|
|
129
|
+
*/
|
|
130
|
+
export function frontmatterText(text: string): string | null {
|
|
131
|
+
const split = splitFrontmatter(text, "(twin)");
|
|
132
|
+
if (!split.ok || split.frontmatter === null) return null;
|
|
133
|
+
return split.block;
|
|
134
|
+
}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The takedown ledger's OTHER baseline: every entry any COMMITTED version of
|
|
3
|
+
* `.ksor/takedowns.yaml` has ever carried.
|
|
4
|
+
*
|
|
5
|
+
* The committed lock is a baseline too, and a good one — it holds each entry's
|
|
6
|
+
* digest, so an entry retargeted in place is caught. What it cannot do is prove
|
|
7
|
+
* that an entry was never deleted, because the lock travels in the SAME change
|
|
8
|
+
* as the ledger: delete the row, recompute `ledger_sha256`, empty
|
|
9
|
+
* `ledger_entries`, and the two agree with each other about a denial that is
|
|
10
|
+
* gone. Only history remembers.
|
|
11
|
+
*
|
|
12
|
+
* The baseline may be INCOMPLETE only if it SAYS so: every version history
|
|
13
|
+
* holds is read, or `entries` comes back null and the caller reports that it
|
|
14
|
+
* could not verify. A version silently skipped would contribute neither
|
|
15
|
+
* digests nor ids while the answer still read "verified"
|
|
16
|
+
* (`git-ledger.integration.test.ts`).
|
|
17
|
+
*
|
|
18
|
+
* This lives in the record module because THREE surfaces need the same answer —
|
|
19
|
+
* `ksor build`, the emitted checker, and the site's stage (decision 19: a
|
|
20
|
+
* surface that refuses must refuse on both surfaces). Plain `git log` / `git
|
|
21
|
+
* show`, so nothing here needs installing.
|
|
22
|
+
*/
|
|
23
|
+
import { spawnSync } from "node:child_process";
|
|
24
|
+
|
|
25
|
+
import { entryDigest, parseLedger, type LedgerBaselineEntry } from "./ledger";
|
|
26
|
+
|
|
27
|
+
const LEDGER = ".ksor/takedowns.yaml";
|
|
28
|
+
|
|
29
|
+
export interface HistoricLedger {
|
|
30
|
+
/** False outside any repository, and false without a git binary. */
|
|
31
|
+
readonly repository: boolean;
|
|
32
|
+
readonly shallow: boolean;
|
|
33
|
+
/** Null when history could not be read; empty in a repository with no commit. */
|
|
34
|
+
readonly entries: readonly LedgerBaselineEntry[] | null;
|
|
35
|
+
/**
|
|
36
|
+
* Why `entries` is null, in the caller's words. Two different states reached
|
|
37
|
+
* one refusal that asserted "this is a shallow clone" as fact — a diagnosis
|
|
38
|
+
* stated for a state nobody distinguished (27352a4).
|
|
39
|
+
*/
|
|
40
|
+
readonly unreadable: "shallow" | "unreadable" | null;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* `spawnSync` defaults to a 1 MB stdout buffer, and past it the child is KILLED
|
|
45
|
+
* — `status` comes back null, so the query reads as a failure. A ledger with a
|
|
46
|
+
* few thousand entries, or one entry carrying a long reason, clears 1 MB
|
|
47
|
+
* easily, and the version was then dropped from the baseline while the caller
|
|
48
|
+
* was still told history had been verified. The ceiling stays finite on
|
|
49
|
+
* purpose: past it this returns null, which is a state the caller SAYS.
|
|
50
|
+
*/
|
|
51
|
+
const MAX_BUFFER = 64 * 1024 * 1024;
|
|
52
|
+
|
|
53
|
+
/** One git query, read-only. Null on any non-zero exit, including no git at all. */
|
|
54
|
+
export function git(root: string, args: readonly string[]): string | null {
|
|
55
|
+
const r = spawnSync("git", [...args], { cwd: root, encoding: "utf8", maxBuffer: MAX_BUFFER });
|
|
56
|
+
return r.status === 0 ? r.stdout : null;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function historicLedger(root: string): HistoricLedger {
|
|
60
|
+
const inside = git(root, ["rev-parse", "--is-inside-work-tree"]);
|
|
61
|
+
if (inside === null || inside.trim() !== "true") {
|
|
62
|
+
return { repository: false, shallow: false, entries: null, unreadable: null };
|
|
63
|
+
}
|
|
64
|
+
const shallow = (git(root, ["rev-parse", "--is-shallow-repository"]) ?? "").trim() === "true";
|
|
65
|
+
// `ksor init` runs `git init`, so a fresh scaffold IS a repository — with no
|
|
66
|
+
// commit in it. `git log` exits non-zero there, which read as "history is
|
|
67
|
+
// unreadable" and refused the first build an adopter ever runs with a message
|
|
68
|
+
// about a shallow clone (found live). A repository with no commits has no
|
|
69
|
+
// history for a ledger id to disappear from, so its baseline is empty and
|
|
70
|
+
// verified, not missing.
|
|
71
|
+
const born = git(root, ["rev-parse", "--verify", "--quiet", "HEAD"]) !== null;
|
|
72
|
+
const entries = shallow ? null : born ? historicEntries(root) : [];
|
|
73
|
+
return {
|
|
74
|
+
repository: true,
|
|
75
|
+
shallow,
|
|
76
|
+
entries,
|
|
77
|
+
unreadable: entries !== null ? null : shallow ? "shallow" : "unreadable",
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Every id history has ever recorded, each with the text it carried the FIRST
|
|
83
|
+
* time it was written. A version that parses contributes each entry's digest,
|
|
84
|
+
* so an entry EDITED in place is caught, not only one deleted; a version that
|
|
85
|
+
* no longer parses still contributes its ids, read permissively — the point
|
|
86
|
+
* there is that an id once written never disappears.
|
|
87
|
+
*
|
|
88
|
+
* FIRST, not every: keying this by `id\tdigest` kept one baseline entry per
|
|
89
|
+
* version an id ever had, so a tamper that was COMMITTED and then UNDONE left
|
|
90
|
+
* two digests for one id, the restored entry matched only one of them, and
|
|
91
|
+
* `ksor-ledger-amended` fired for good. The record became permanently
|
|
92
|
+
* unbuildable — by a tamper that had already been put right — and the only
|
|
93
|
+
* escape was rewriting git history, which is not a remedy a refusal may
|
|
94
|
+
* demand (found in review, 2026-08-25).
|
|
95
|
+
*
|
|
96
|
+
* Taking the OLDEST is what makes the guarantee both enforceable and
|
|
97
|
+
* escapable. It still refuses a committed tamper (the baseline is what the
|
|
98
|
+
* entry said when it was written, so committing the edit does not launder
|
|
99
|
+
* it), and the remedy it names — put the entry back — now actually clears
|
|
100
|
+
* it. Taking the NEWEST would have done the opposite on both counts.
|
|
101
|
+
*/
|
|
102
|
+
function historicEntries(root: string): readonly LedgerBaselineEntry[] | null {
|
|
103
|
+
// TWO paths for one file, because git means two different things by "path"
|
|
104
|
+
// here and each call is right on its own:
|
|
105
|
+
//
|
|
106
|
+
// `git show <rev>:<path>` and `ls-tree --full-tree` read a path relative to
|
|
107
|
+
// the REPOSITORY ROOT — hence `atRoot`, built from `--show-prefix`.
|
|
108
|
+
//
|
|
109
|
+
// a `git log -- <pathspec>` is relative to the CWD, and the cwd is already
|
|
110
|
+
// the record root — hence `LEDGER`, bare.
|
|
111
|
+
//
|
|
112
|
+
// Prefixing both asked `git log` for `docs-sor/docs-sor/.ksor/takedowns.yaml`
|
|
113
|
+
// whenever a record sat below its repository root, which is an ordinary
|
|
114
|
+
// shape. A pathspec matching nothing is not an ERROR: `git log` exits 0 and
|
|
115
|
+
// prints nothing, so the baseline came back EMPTY and non-null — verified,
|
|
116
|
+
// holding nothing — and a deleted denial republished with the build exiting
|
|
117
|
+
// 0. Every case in the test file was `mkdtemp` + `git init` in ONE directory,
|
|
118
|
+
// where the prefix is always empty, so no number of cases in that shape could
|
|
119
|
+
// have caught it (found in review, 2026-08-25).
|
|
120
|
+
const prefix = (git(root, ["rev-parse", "--show-prefix"]) ?? "").trim();
|
|
121
|
+
const atRoot = `${prefix}${LEDGER}`;
|
|
122
|
+
// `--full-history` because `git log -- <path>` SIMPLIFIES by default: a merge
|
|
123
|
+
// TREESAME to a parent is followed through that parent alone, so a branch
|
|
124
|
+
// whose net effect on the ledger was nil is pruned whole — a denial added and
|
|
125
|
+
// withdrawn inside one pull request never entered the baseline, and
|
|
126
|
+
// `ksor-ledger-shrank` could not fire for it. That is precisely the deletion
|
|
127
|
+
// the committed lock cannot catch either, since the lock travels in the same
|
|
128
|
+
// pull request; if history does not remember it, nothing does.
|
|
129
|
+
//
|
|
130
|
+
// `--topo-order` because the walk below lets an OLDER version overwrite a
|
|
131
|
+
// newer one, and across branches the default ordering is by commit DATE,
|
|
132
|
+
// which need not follow ancestry. Topological order does, so "oldest" means
|
|
133
|
+
// the ancestor rather than whichever machine's clock was behind.
|
|
134
|
+
const commits = git(root, ["log", "--full-history", "--topo-order", "--format=%H", "--", LEDGER]);
|
|
135
|
+
if (commits === null) return null;
|
|
136
|
+
const seen = new Map<string, LedgerBaselineEntry>();
|
|
137
|
+
for (const sha of commits.split("\n").filter((s) => s !== "")) {
|
|
138
|
+
const text = git(root, ["show", `${sha}:${atRoot}`]);
|
|
139
|
+
if (text === null) {
|
|
140
|
+
// `git show` failing means one of two things, and they used to be one
|
|
141
|
+
// silent `continue`: the commit DELETED the ledger (expected — it has no
|
|
142
|
+
// version to contribute), or the bytes could not be read (a killed query,
|
|
143
|
+
// a missing object). `ls-tree` separates them — it exits 0 either way and
|
|
144
|
+
// prints nothing when the path is absent — so an unreadable version stops
|
|
145
|
+
// being counted as a verified empty one. Asked only on failure, so the
|
|
146
|
+
// ordinary walk still costs one git call per commit.
|
|
147
|
+
const listed = git(root, ["ls-tree", "--full-tree", "--name-only", sha, "--", atRoot]);
|
|
148
|
+
if (listed !== null && listed.trim() === "") continue;
|
|
149
|
+
return null;
|
|
150
|
+
}
|
|
151
|
+
const where = sha.slice(0, 7);
|
|
152
|
+
const parsed = parseLedger(text, LEDGER);
|
|
153
|
+
// The walk is newest-first in topological order, so an OLDER version
|
|
154
|
+
// overwriting a newer one leaves the oldest — the text the id was written
|
|
155
|
+
// with.
|
|
156
|
+
if (parsed.ok) {
|
|
157
|
+
for (const entry of parsed.ledger.entries) {
|
|
158
|
+
seen.set(entry.id, { id: entry.id, digest: entryDigest(entry), entry, where });
|
|
159
|
+
}
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
for (const m of text.matchAll(/^\s*(?:-\s+)?id:\s*["']?([^\s"']+)/gm)) {
|
|
163
|
+
const id = m[1] ?? "";
|
|
164
|
+
// Presence only, and never over a digest: a version that stopped
|
|
165
|
+
// parsing proves the id existed, and proves nothing about its text, so
|
|
166
|
+
// it must not erase what an older readable version already said.
|
|
167
|
+
if (!seen.has(id)) seen.set(id, { id, digest: null, where });
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
return [...seen.values()].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
171
|
+
}
|