@docsworth/cli 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +55 -0
- package/README.md +238 -2
- package/dist/cli.d.ts +11 -0
- package/dist/cli.js +458 -0
- package/dist/core/acl.d.ts +37 -0
- package/dist/core/acl.js +136 -0
- package/dist/core/adopt.d.ts +68 -0
- package/dist/core/adopt.js +665 -0
- package/dist/core/audit-log.d.ts +133 -0
- package/dist/core/audit-log.js +290 -0
- package/dist/core/blocks.d.ts +35 -0
- package/dist/core/blocks.js +417 -0
- package/dist/core/codeowners.d.ts +62 -0
- package/dist/core/codeowners.js +133 -0
- package/dist/core/config.d.ts +101 -0
- package/dist/core/config.js +175 -0
- package/dist/core/coverage.d.ts +25 -0
- package/dist/core/coverage.js +41 -0
- package/dist/core/duration.d.ts +20 -0
- package/dist/core/duration.js +60 -0
- package/dist/core/edits.d.ts +13 -0
- package/dist/core/edits.js +20 -0
- package/dist/core/errors.d.ts +11 -0
- package/dist/core/errors.js +13 -0
- package/dist/core/frontmatter.d.ts +29 -0
- package/dist/core/frontmatter.js +49 -0
- package/dist/core/gate.d.ts +17 -0
- package/dist/core/gate.js +25 -0
- package/dist/core/gfm.d.ts +13 -0
- package/dist/core/gfm.js +244 -0
- package/dist/core/github.d.ts +152 -0
- package/dist/core/github.js +234 -0
- package/dist/core/host.d.ts +125 -0
- package/dist/core/host.js +22 -0
- package/dist/core/identity.d.ts +39 -0
- package/dist/core/identity.js +127 -0
- package/dist/core/init.d.ts +25 -0
- package/dist/core/init.js +86 -0
- package/dist/core/links.d.ts +82 -0
- package/dist/core/links.js +345 -0
- package/dist/core/lint.d.ts +16 -0
- package/dist/core/lint.js +149 -0
- package/dist/core/mcp/audit-transport.d.ts +16 -0
- package/dist/core/mcp/audit-transport.js +238 -0
- package/dist/core/mcp/server.d.ts +19 -0
- package/dist/core/mcp/server.js +216 -0
- package/dist/core/mcp/text.d.ts +14 -0
- package/dist/core/mcp/text.js +43 -0
- package/dist/core/mirror-view.d.ts +68 -0
- package/dist/core/mirror-view.js +367 -0
- package/dist/core/new-page.d.ts +52 -0
- package/dist/core/new-page.js +49 -0
- package/dist/core/policy.d.ts +10 -0
- package/dist/core/policy.js +30 -0
- package/dist/core/polish.d.ts +42 -0
- package/dist/core/polish.js +1000 -0
- package/dist/core/printable.d.ts +4 -0
- package/dist/core/printable.js +16 -0
- package/dist/core/references.d.ts +87 -0
- package/dist/core/references.js +1744 -0
- package/dist/core/repo.d.ts +46 -0
- package/dist/core/repo.js +118 -0
- package/dist/core/schema.d.ts +50 -0
- package/dist/core/schema.js +51 -0
- package/dist/core/search.d.ts +32 -0
- package/dist/core/search.js +132 -0
- package/dist/core/serve/auth.d.ts +23 -0
- package/dist/core/serve/auth.js +38 -0
- package/dist/core/serve/server.d.ts +21 -0
- package/dist/core/serve/server.js +138 -0
- package/dist/core/serve/sessions.d.ts +40 -0
- package/dist/core/serve/sessions.js +203 -0
- package/dist/core/serve/token.d.ts +10 -0
- package/dist/core/serve/token.js +96 -0
- package/dist/core/service.d.ts +473 -0
- package/dist/core/service.js +973 -0
- package/dist/core/sla.d.ts +16 -0
- package/dist/core/sla.js +20 -0
- package/dist/core/stale.d.ts +16 -0
- package/dist/core/stale.js +24 -0
- package/dist/core/tool-registry.d.ts +31 -0
- package/dist/core/tool-registry.js +80 -0
- package/dist/core/url.d.ts +4 -0
- package/dist/core/url.js +31 -0
- package/dist/core/verify-pages.d.ts +19 -0
- package/dist/core/verify-pages.js +110 -0
- package/dist/core/version.d.ts +22 -0
- package/dist/core/version.js +32 -0
- package/dist/core/writes.d.ts +58 -0
- package/dist/core/writes.js +113 -0
- package/dist/web/assets/index-8J-FUbb0.js +29 -0
- package/dist/web/assets/index-BGpAZzeM.css +1 -0
- package/dist/web/index.html +14 -0
- package/docs/format-spec.md +349 -0
- package/package.json +75 -4
- package/plugin/.claude-plugin/plugin.json +8 -0
- package/plugin/.mcp.json +8 -0
- package/plugin/bin/docsworth-mcp.mjs +46 -0
- package/plugin/skills/draft-runbook/SKILL.md +75 -0
- package/plugin/skills/polish-page/SKILL.md +64 -0
- package/plugin/skills/update-after-incident/SKILL.md +59 -0
- package/plugin/skills/verify-page/SKILL.md +46 -0
- package/templates/github/knowledge.yml +51 -0
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { type Codeowners } from "./codeowners.js";
|
|
2
|
+
import { type Config } from "./config.js";
|
|
3
|
+
import { type FieldIssue, type Frontmatter } from "./schema.js";
|
|
4
|
+
export interface Page extends Frontmatter {
|
|
5
|
+
/** Repo-relative posix path, e.g. `knowledge/runbooks/deploy.md`. */
|
|
6
|
+
path: string;
|
|
7
|
+
body: string;
|
|
8
|
+
headings: string[];
|
|
9
|
+
/** Relative markdown link targets as written (anchors stripped). */
|
|
10
|
+
links: string[];
|
|
11
|
+
sha256: string;
|
|
12
|
+
unknownFields: string[];
|
|
13
|
+
/** Its frontmatter `slug`, if a string: where a docs site serves it (`siteRoute`). Not in the format; read for redaction and lint. */
|
|
14
|
+
slug?: string;
|
|
15
|
+
}
|
|
16
|
+
export interface InvalidPage {
|
|
17
|
+
path: string;
|
|
18
|
+
/** The `id` field if present and a string, so lint can still report duplicates. */
|
|
19
|
+
id?: string;
|
|
20
|
+
/** The `slug` field if a string: redaction hides the route it gives the page too. */
|
|
21
|
+
slug?: string;
|
|
22
|
+
issues: FieldIssue[];
|
|
23
|
+
}
|
|
24
|
+
export interface Repo {
|
|
25
|
+
root: string;
|
|
26
|
+
config: Config;
|
|
27
|
+
codeowners: Codeowners;
|
|
28
|
+
pages: Page[];
|
|
29
|
+
invalid: InvalidPage[];
|
|
30
|
+
byId: Map<string, Page>;
|
|
31
|
+
byPath: Map<string, Page>;
|
|
32
|
+
}
|
|
33
|
+
/** Relative link targets (inline and reference definitions), without query or fragment. */
|
|
34
|
+
export declare function extractLinks(body: string): string[];
|
|
35
|
+
export declare function extractHeadings(body: string): string[];
|
|
36
|
+
/** Resolve a relative link from a page to a repo-relative posix path. */
|
|
37
|
+
export declare function resolveLink(fromPath: string, target: string): string;
|
|
38
|
+
export declare function parsePage(path: string, text: string): Page | InvalidPage;
|
|
39
|
+
export declare function isPage(p: Page | InvalidPage): p is Page;
|
|
40
|
+
/**
|
|
41
|
+
* Load a knowledge repo from disk. Pages with invalid frontmatter are kept out of
|
|
42
|
+
* `pages` (nothing downstream may serve them) and reported in `invalid` for the linter.
|
|
43
|
+
*/
|
|
44
|
+
export declare function loadRepo(root: string, config?: Config): Repo;
|
|
45
|
+
/** Latest mtime under the knowledge dir; the MCP server uses it to detect edits. */
|
|
46
|
+
export declare function knowledgeMtime(root: string, config: Config): number;
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readdirSync, readFileSync, statSync } from "node:fs";
|
|
3
|
+
import { join, posix, relative, sep } from "node:path";
|
|
4
|
+
import { loadCodeowners } from "./codeowners.js";
|
|
5
|
+
import { loadConfig } from "./config.js";
|
|
6
|
+
import { splitFrontmatter } from "./frontmatter.js";
|
|
7
|
+
import { validateFrontmatter } from "./schema.js";
|
|
8
|
+
// Linear without bounds on what a link may hold (main had none, and lint reads every link):
|
|
9
|
+
// each span stops where the next of its kind could start. Link text stops at a `[`, so `[a`
|
|
10
|
+
// repeated can't make each `[` scan on (to the end of the page: 1.1 s for 64 KB); the
|
|
11
|
+
// destination is matched once, in a lookahead, and never given back character by
|
|
12
|
+
// character (`[a](` repeated retried every length); an angle destination stops at `<`
|
|
13
|
+
// (CommonMark), a plain one at `[`. A title runs to its own closer; one in parentheses
|
|
14
|
+
// holds no unescaped `(` (CommonMark), so it stops there too. Only a definition's label is
|
|
15
|
+
// bounded, at CommonMark's 999 characters.
|
|
16
|
+
const TITLE = String.raw `(?:\s+(?:"[^"]*"|'[^']*'|\((?:[^()\\]|\\.)*\)))?`;
|
|
17
|
+
const LINK_RE = new RegExp(String.raw `!?\[[^\][]*\]\(\s*(?=(<[^<>\n]*>|[^)\s[]+))\1${TITLE}\s*\)`, "g");
|
|
18
|
+
const REF_DEF_RE = new RegExp(String.raw `^ {0,3}\[[^\][\n]{1,999}\]:[ \t]*(?=(<[^<>\n]*>|\S+))\1${TITLE}[ \t]*$`, "gm");
|
|
19
|
+
const EXTERNAL = /^(?:[a-z][a-z0-9+.-]*:|\/\/|#)/i;
|
|
20
|
+
/** Relative link targets (inline and reference definitions), without query or fragment. */
|
|
21
|
+
export function extractLinks(body) {
|
|
22
|
+
const out = [];
|
|
23
|
+
for (const m of [...body.matchAll(LINK_RE), ...body.matchAll(REF_DEF_RE)]) {
|
|
24
|
+
const target = m[1].replace(/^<|>$/g, "");
|
|
25
|
+
if (EXTERNAL.test(target))
|
|
26
|
+
continue;
|
|
27
|
+
const path = target.split(/[?#]/)[0];
|
|
28
|
+
if (path)
|
|
29
|
+
out.push(path);
|
|
30
|
+
}
|
|
31
|
+
return out;
|
|
32
|
+
}
|
|
33
|
+
export function extractHeadings(body) {
|
|
34
|
+
return body
|
|
35
|
+
.split(/\r?\n/)
|
|
36
|
+
.filter((l) => /^#{1,6}\s/.test(l))
|
|
37
|
+
.map((l) => l.replace(/^#{1,6}\s+/, "").trim());
|
|
38
|
+
}
|
|
39
|
+
/** Resolve a relative link from a page to a repo-relative posix path. */
|
|
40
|
+
export function resolveLink(fromPath, target) {
|
|
41
|
+
let t = target;
|
|
42
|
+
try {
|
|
43
|
+
t = decodeURI(target);
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
/* malformed escape: resolve as written */
|
|
47
|
+
}
|
|
48
|
+
return t.startsWith("/") ? posix.normalize(t.slice(1)) : posix.normalize(posix.join(posix.dirname(fromPath), t));
|
|
49
|
+
}
|
|
50
|
+
function walkMarkdown(dir) {
|
|
51
|
+
let entries;
|
|
52
|
+
try {
|
|
53
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
return [];
|
|
57
|
+
}
|
|
58
|
+
return entries.flatMap((e) => {
|
|
59
|
+
const abs = join(dir, e.name);
|
|
60
|
+
if (e.isDirectory())
|
|
61
|
+
return e.name.startsWith(".") ? [] : walkMarkdown(abs);
|
|
62
|
+
return e.isFile() && e.name.endsWith(".md") ? [abs] : [];
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
export function parsePage(path, text) {
|
|
66
|
+
const doc = splitFrontmatter(text);
|
|
67
|
+
if (!doc.data)
|
|
68
|
+
return { path, issues: [{ field: "(frontmatter)", message: doc.error ?? "missing frontmatter" }] };
|
|
69
|
+
const rawId = typeof doc.data.id === "string" ? doc.data.id : undefined;
|
|
70
|
+
const slug = typeof doc.data.slug === "string" ? { slug: doc.data.slug } : {};
|
|
71
|
+
const v = validateFrontmatter(doc.data);
|
|
72
|
+
if (!v.ok)
|
|
73
|
+
return { path, id: rawId, ...slug, issues: v.issues };
|
|
74
|
+
return {
|
|
75
|
+
...v.value,
|
|
76
|
+
...slug,
|
|
77
|
+
path,
|
|
78
|
+
body: doc.body,
|
|
79
|
+
headings: extractHeadings(doc.body),
|
|
80
|
+
links: extractLinks(doc.body),
|
|
81
|
+
sha256: createHash("sha256").update(text).digest("hex"),
|
|
82
|
+
unknownFields: v.unknownFields,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
export function isPage(p) {
|
|
86
|
+
return p.sha256 !== undefined;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Load a knowledge repo from disk. Pages with invalid frontmatter are kept out of
|
|
90
|
+
* `pages` (nothing downstream may serve them) and reported in `invalid` for the linter.
|
|
91
|
+
*/
|
|
92
|
+
export function loadRepo(root, config = loadConfig(root)) {
|
|
93
|
+
const knowledgeDir = join(root, config.knowledge_root);
|
|
94
|
+
const pages = [];
|
|
95
|
+
const invalid = [];
|
|
96
|
+
for (const abs of walkMarkdown(knowledgeDir).sort()) {
|
|
97
|
+
const path = relative(root, abs).split(sep).join("/");
|
|
98
|
+
const parsed = parsePage(path, readFileSync(abs, "utf8"));
|
|
99
|
+
(isPage(parsed) ? pages : invalid).push(parsed);
|
|
100
|
+
}
|
|
101
|
+
const byId = new Map();
|
|
102
|
+
for (const p of pages)
|
|
103
|
+
if (!byId.has(p.id))
|
|
104
|
+
byId.set(p.id, p);
|
|
105
|
+
return {
|
|
106
|
+
root,
|
|
107
|
+
config,
|
|
108
|
+
codeowners: loadCodeowners(root),
|
|
109
|
+
pages,
|
|
110
|
+
invalid,
|
|
111
|
+
byId,
|
|
112
|
+
byPath: new Map(pages.map((p) => [p.path, p])),
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
/** Latest mtime under the knowledge dir; the MCP server uses it to detect edits. */
|
|
116
|
+
export function knowledgeMtime(root, config) {
|
|
117
|
+
return walkMarkdown(join(root, config.knowledge_root)).reduce((max, f) => Math.max(max, statSync(f).mtimeMs), 0);
|
|
118
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export declare const PAGE_TYPES: readonly ["runbook", "adr", "service", "guide", "policy"];
|
|
3
|
+
export declare const STATUSES: readonly ["canonical", "draft", "deprecated"];
|
|
4
|
+
export declare const REQUIRED_FIELDS: readonly ["title", "id", "type", "status", "owner_team", "last_verified", "review_every"];
|
|
5
|
+
export declare const OPTIONAL_FIELDS: readonly ["supersedes", "applies_to", "acl", "description"];
|
|
6
|
+
/** Default ACL: anyone with read access to the knowledge repo. */
|
|
7
|
+
export declare const ACL_INTERNAL = "internal";
|
|
8
|
+
/** A team slug, or `internal` (owner_team, acl). */
|
|
9
|
+
export declare const SLUG_RE: RegExp;
|
|
10
|
+
/** A page id: lowercase letters, digits and dashes. */
|
|
11
|
+
export declare const ID_RE: RegExp;
|
|
12
|
+
export declare const FrontmatterSchema: z.ZodObject<{
|
|
13
|
+
title: z.ZodString;
|
|
14
|
+
id: z.ZodString;
|
|
15
|
+
type: z.ZodEnum<{
|
|
16
|
+
adr: "adr";
|
|
17
|
+
guide: "guide";
|
|
18
|
+
policy: "policy";
|
|
19
|
+
runbook: "runbook";
|
|
20
|
+
service: "service";
|
|
21
|
+
}>;
|
|
22
|
+
status: z.ZodEnum<{
|
|
23
|
+
canonical: "canonical";
|
|
24
|
+
deprecated: "deprecated";
|
|
25
|
+
draft: "draft";
|
|
26
|
+
}>;
|
|
27
|
+
owner_team: z.ZodString;
|
|
28
|
+
last_verified: z.ZodString;
|
|
29
|
+
review_every: z.ZodString;
|
|
30
|
+
supersedes: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
31
|
+
applies_to: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
32
|
+
acl: z.ZodDefault<z.ZodString>;
|
|
33
|
+
description: z.ZodOptional<z.ZodString>;
|
|
34
|
+
}, z.core.$strip>;
|
|
35
|
+
export type Frontmatter = z.infer<typeof FrontmatterSchema>;
|
|
36
|
+
export type PageType = (typeof PAGE_TYPES)[number];
|
|
37
|
+
export type PageStatus = (typeof STATUSES)[number];
|
|
38
|
+
export interface FieldIssue {
|
|
39
|
+
field: string;
|
|
40
|
+
message: string;
|
|
41
|
+
}
|
|
42
|
+
/** Validate raw frontmatter, returning issues keyed by field for the linter. */
|
|
43
|
+
export declare function validateFrontmatter(data: Record<string, unknown>): {
|
|
44
|
+
ok: true;
|
|
45
|
+
value: Frontmatter;
|
|
46
|
+
unknownFields: string[];
|
|
47
|
+
} | {
|
|
48
|
+
ok: false;
|
|
49
|
+
issues: FieldIssue[];
|
|
50
|
+
};
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { DURATION_RE, parseDate } from "./duration.js";
|
|
3
|
+
// The frontmatter contract from docs/format-spec.md (format: 1).
|
|
4
|
+
export const PAGE_TYPES = ["runbook", "adr", "service", "guide", "policy"];
|
|
5
|
+
export const STATUSES = ["canonical", "draft", "deprecated"];
|
|
6
|
+
export const REQUIRED_FIELDS = [
|
|
7
|
+
"title",
|
|
8
|
+
"id",
|
|
9
|
+
"type",
|
|
10
|
+
"status",
|
|
11
|
+
"owner_team",
|
|
12
|
+
"last_verified",
|
|
13
|
+
"review_every",
|
|
14
|
+
];
|
|
15
|
+
export const OPTIONAL_FIELDS = ["supersedes", "applies_to", "acl", "description"];
|
|
16
|
+
/** Default ACL: anyone with read access to the knowledge repo. */
|
|
17
|
+
export const ACL_INTERNAL = "internal";
|
|
18
|
+
/** A team slug, or `internal` (owner_team, acl). */
|
|
19
|
+
export const SLUG_RE = /^[a-z0-9][a-z0-9_-]*$/;
|
|
20
|
+
/** A page id: lowercase letters, digits and dashes. */
|
|
21
|
+
export const ID_RE = /^[a-z0-9][a-z0-9-]*$/;
|
|
22
|
+
export const FrontmatterSchema = z.object({
|
|
23
|
+
title: z.string().trim().min(1),
|
|
24
|
+
id: z.string().regex(ID_RE, "lowercase letters, digits and dashes"),
|
|
25
|
+
type: z.enum(PAGE_TYPES),
|
|
26
|
+
status: z.enum(STATUSES),
|
|
27
|
+
// A bare team slug; the org comes from config (github.org). Never `@user`.
|
|
28
|
+
owner_team: z.string().regex(SLUG_RE, "a team slug like `platform` (no @, no org)"),
|
|
29
|
+
last_verified: z
|
|
30
|
+
.string()
|
|
31
|
+
.refine((s) => parseDate(s) !== null, "a real calendar date as YYYY-MM-DD"),
|
|
32
|
+
review_every: z.string().regex(DURATION_RE, "a period like 30d, 12w, 6m or 1y"),
|
|
33
|
+
supersedes: z.array(z.string().regex(ID_RE)).optional(),
|
|
34
|
+
applies_to: z.array(z.string().min(1)).optional(),
|
|
35
|
+
acl: z.string().regex(SLUG_RE, "`internal` or a team slug").default(ACL_INTERNAL),
|
|
36
|
+
description: z.string().optional(),
|
|
37
|
+
});
|
|
38
|
+
/** Validate raw frontmatter, returning issues keyed by field for the linter. */
|
|
39
|
+
export function validateFrontmatter(data) {
|
|
40
|
+
const result = FrontmatterSchema.safeParse(data);
|
|
41
|
+
const known = new Set([...REQUIRED_FIELDS, ...OPTIONAL_FIELDS]);
|
|
42
|
+
const unknownFields = Object.keys(data).filter((k) => !known.has(k));
|
|
43
|
+
if (result.success)
|
|
44
|
+
return { ok: true, value: result.data, unknownFields };
|
|
45
|
+
const issues = result.error.issues.map((issue) => {
|
|
46
|
+
const field = String(issue.path[0] ?? "(root)");
|
|
47
|
+
const missing = data[field] === undefined;
|
|
48
|
+
return { field, message: missing ? `missing required field \`${field}\`` : `\`${field}\`: ${issue.message}` };
|
|
49
|
+
});
|
|
50
|
+
return { ok: false, issues };
|
|
51
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export interface SearchDoc {
|
|
2
|
+
title: string;
|
|
3
|
+
headings?: string[];
|
|
4
|
+
description?: string;
|
|
5
|
+
body: string;
|
|
6
|
+
/** Extra exact-ish tokens such as ids or service names. */
|
|
7
|
+
keywords?: string[];
|
|
8
|
+
}
|
|
9
|
+
export interface Hit {
|
|
10
|
+
index: number;
|
|
11
|
+
score: number;
|
|
12
|
+
}
|
|
13
|
+
export declare function tokenize(text: string): string[];
|
|
14
|
+
export declare class SearchIndex {
|
|
15
|
+
private postings;
|
|
16
|
+
private lengths;
|
|
17
|
+
private avgLength;
|
|
18
|
+
constructor(docs: SearchDoc[]);
|
|
19
|
+
get size(): number;
|
|
20
|
+
/** Top hits for a query; `accept` filters candidates before ranking (ACL, filters). */
|
|
21
|
+
search(query: string, limit: number, accept?: (index: number) => boolean): Hit[];
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* A page's markdown as a reader sees it, to cut snippets from: its words, code as written,
|
|
25
|
+
* an image as its label, a table's cells (`a | b`); no markup, link targets or raw HTML.
|
|
26
|
+
* It reads the tree the web app renders from (`parseGfm`), so a snippet cut from it holds
|
|
27
|
+
* exactly the words the page shows, and never ends inside markup (search's
|
|
28
|
+
* `snippets: "text"`).
|
|
29
|
+
*/
|
|
30
|
+
export declare function plainText(markdown: string): string;
|
|
31
|
+
/** A ~maxLen window of `body` around the densest run of query terms. */
|
|
32
|
+
export declare function snippet(body: string, query: string, maxLen?: number): string;
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
// In-memory BM25 (BM25F-lite: per-field boosts folded into term frequency).
|
|
2
|
+
// The plan says "BM25 first, embeddings later"; this keeps search dependency-free and
|
|
3
|
+
// fast enough for 5,000 pages (scripts/bench-search.ts holds it to p95 < 500 ms).
|
|
4
|
+
import { isTableCell, parseGfm, textIn } from "./gfm.js";
|
|
5
|
+
const STOPWORDS = new Set("a an and are as at be by can do does for from has have how i if in into is it its of on or our should that the their then there this to was we what when where which who why will with you your".split(" "));
|
|
6
|
+
const BOOST = { title: 3, keywords: 3, headings: 2, description: 2, body: 1 };
|
|
7
|
+
const K1 = 1.2;
|
|
8
|
+
const B = 0.75;
|
|
9
|
+
export function tokenize(text) {
|
|
10
|
+
return (text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [])
|
|
11
|
+
.filter((t) => !STOPWORDS.has(t))
|
|
12
|
+
.map(stem);
|
|
13
|
+
}
|
|
14
|
+
/** Deliberately light stemming: plural and common verb suffixes only. */
|
|
15
|
+
function stem(t) {
|
|
16
|
+
if (t.length > 5 && t.endsWith("ing"))
|
|
17
|
+
return t.slice(0, -3);
|
|
18
|
+
if (t.length > 4 && t.endsWith("ies"))
|
|
19
|
+
return `${t.slice(0, -3)}y`;
|
|
20
|
+
if (t.length > 4 && t.endsWith("ed"))
|
|
21
|
+
return t.slice(0, -2);
|
|
22
|
+
if (t.length > 3 && t.endsWith("s") && !t.endsWith("ss"))
|
|
23
|
+
return t.slice(0, -1);
|
|
24
|
+
return t;
|
|
25
|
+
}
|
|
26
|
+
export class SearchIndex {
|
|
27
|
+
postings = new Map();
|
|
28
|
+
lengths = [];
|
|
29
|
+
avgLength = 0;
|
|
30
|
+
constructor(docs) {
|
|
31
|
+
docs.forEach((doc, i) => {
|
|
32
|
+
const tf = new Map();
|
|
33
|
+
const add = (text, boost) => {
|
|
34
|
+
for (const t of tokenize(text ?? ""))
|
|
35
|
+
tf.set(t, (tf.get(t) ?? 0) + boost);
|
|
36
|
+
};
|
|
37
|
+
add(doc.title, BOOST.title);
|
|
38
|
+
add(doc.keywords?.join(" "), BOOST.keywords);
|
|
39
|
+
add(doc.headings?.join(" "), BOOST.headings);
|
|
40
|
+
add(doc.description, BOOST.description);
|
|
41
|
+
add(doc.body, BOOST.body);
|
|
42
|
+
let len = 0;
|
|
43
|
+
for (const [term, f] of tf) {
|
|
44
|
+
len += f;
|
|
45
|
+
let list = this.postings.get(term);
|
|
46
|
+
if (!list)
|
|
47
|
+
this.postings.set(term, (list = []));
|
|
48
|
+
list.push({ doc: i, tf: f });
|
|
49
|
+
}
|
|
50
|
+
this.lengths.push(len);
|
|
51
|
+
});
|
|
52
|
+
this.avgLength = this.lengths.reduce((a, b) => a + b, 0) / Math.max(1, this.lengths.length);
|
|
53
|
+
}
|
|
54
|
+
get size() {
|
|
55
|
+
return this.lengths.length;
|
|
56
|
+
}
|
|
57
|
+
/** Top hits for a query; `accept` filters candidates before ranking (ACL, filters). */
|
|
58
|
+
search(query, limit, accept = () => true) {
|
|
59
|
+
const n = this.lengths.length;
|
|
60
|
+
const scores = new Map();
|
|
61
|
+
for (const term of new Set(tokenize(query))) {
|
|
62
|
+
const list = this.postings.get(term);
|
|
63
|
+
if (!list)
|
|
64
|
+
continue;
|
|
65
|
+
const idf = Math.log(1 + (n - list.length + 0.5) / (list.length + 0.5));
|
|
66
|
+
for (const { doc, tf } of list) {
|
|
67
|
+
const norm = tf + K1 * (1 - B + (B * this.lengths[doc]) / this.avgLength);
|
|
68
|
+
scores.set(doc, (scores.get(doc) ?? 0) + idf * ((tf * (K1 + 1)) / norm));
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return [...scores.entries()]
|
|
72
|
+
.filter(([doc]) => accept(doc))
|
|
73
|
+
.map(([index, score]) => ({ index, score: Math.round(score * 1000) / 1000 }))
|
|
74
|
+
.sort((a, b) => b.score - a.score || a.index - b.index)
|
|
75
|
+
.slice(0, limit);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
// Blocks whose text a reader sees apart from what's next to them.
|
|
79
|
+
const BLOCKS = new Set(["paragraph", "heading", "item", "block_quote", "code_block", "custom_block", "thematic_break"]);
|
|
80
|
+
/**
|
|
81
|
+
* A page's markdown as a reader sees it, to cut snippets from: its words, code as written,
|
|
82
|
+
* an image as its label, a table's cells (`a | b`); no markup, link targets or raw HTML.
|
|
83
|
+
* It reads the tree the web app renders from (`parseGfm`), so a snippet cut from it holds
|
|
84
|
+
* exactly the words the page shows, and never ends inside markup (search's
|
|
85
|
+
* `snippets: "text"`).
|
|
86
|
+
*/
|
|
87
|
+
export function plainText(markdown) {
|
|
88
|
+
let out = "";
|
|
89
|
+
const walker = parseGfm(markdown).walker();
|
|
90
|
+
for (let e = walker.next(); e; e = walker.next()) {
|
|
91
|
+
const n = e.node;
|
|
92
|
+
if (!e.entering)
|
|
93
|
+
continue;
|
|
94
|
+
if (n.type === "text" || n.type === "code")
|
|
95
|
+
out += n.literal ?? "";
|
|
96
|
+
else if (n.type === "code_block")
|
|
97
|
+
out += ` ${n.literal ?? ""} `;
|
|
98
|
+
else if (isTableCell(n))
|
|
99
|
+
out += n.prev ? " | " : " ";
|
|
100
|
+
// As the web app labels an image (packages/ui/src/markdown.ts): `image: ` and its words,
|
|
101
|
+
// or its address when it has none, once; what's inside it (an image too) isn't shown.
|
|
102
|
+
else if (n.type === "image") {
|
|
103
|
+
out += `image: ${textIn(n) || n.destination || ""}`;
|
|
104
|
+
walker.resumeAt(n, false);
|
|
105
|
+
}
|
|
106
|
+
else if (n.type === "softbreak" || n.type === "linebreak" || BLOCKS.has(n.type))
|
|
107
|
+
out += " ";
|
|
108
|
+
}
|
|
109
|
+
return out.replace(/\s+/g, " ").trim();
|
|
110
|
+
}
|
|
111
|
+
/** A ~maxLen window of `body` around the densest run of query terms. */
|
|
112
|
+
export function snippet(body, query, maxLen = 240) {
|
|
113
|
+
const text = body.replace(/\s+/g, " ").trim();
|
|
114
|
+
if (text.length <= maxLen)
|
|
115
|
+
return text;
|
|
116
|
+
const terms = new Set(tokenize(query));
|
|
117
|
+
const words = text.split(" ");
|
|
118
|
+
let best = 0;
|
|
119
|
+
let bestScore = -1;
|
|
120
|
+
const window = 30;
|
|
121
|
+
for (let i = 0; i < words.length; i += 5) {
|
|
122
|
+
const score = words.slice(i, i + window).filter((w) => terms.has(tokenize(w)[0] ?? "")).length;
|
|
123
|
+
if (score > bestScore)
|
|
124
|
+
[best, bestScore] = [i, score];
|
|
125
|
+
}
|
|
126
|
+
// Start a few words before the first match in the winning window, so the matches
|
|
127
|
+
// themselves land inside the visible length.
|
|
128
|
+
const firstMatch = words.findIndex((w, i) => i >= best && terms.has(tokenize(w)[0] ?? ""));
|
|
129
|
+
const start = firstMatch === -1 ? best : Math.max(0, firstMatch - 4);
|
|
130
|
+
const out = words.slice(start).join(" ").slice(0, maxLen);
|
|
131
|
+
return `${start > 0 ? "…" : ""}${out}${out.length >= maxLen ? "…" : ""}`;
|
|
132
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { IncomingMessage } from "node:http";
|
|
2
|
+
/** A Chromium extension id: 32 letters a to p. */
|
|
3
|
+
export declare const EXTENSION_ID: RegExp;
|
|
4
|
+
export interface AccessRules {
|
|
5
|
+
port: number;
|
|
6
|
+
token: string;
|
|
7
|
+
/** Extension ids allowed to call /mcp from `chrome-extension://<id>`. */
|
|
8
|
+
extensions: ReadonlySet<string>;
|
|
9
|
+
}
|
|
10
|
+
export type Verdict = {
|
|
11
|
+
ok: true; /** The origin to answer with CORS headers, for an allowed extension. */
|
|
12
|
+
cors?: string;
|
|
13
|
+
} | {
|
|
14
|
+
ok: false;
|
|
15
|
+
status: 401 | 403;
|
|
16
|
+
message: string;
|
|
17
|
+
};
|
|
18
|
+
/** Host and Origin, checked on every request, static files included. */
|
|
19
|
+
export declare function checkOrigin(req: IncomingMessage, rules: AccessRules): Verdict;
|
|
20
|
+
/** The bearer token, checked on every /mcp request after checkOrigin. */
|
|
21
|
+
export declare function checkToken(req: IncomingMessage, rules: AccessRules): Verdict;
|
|
22
|
+
/** CORS headers for an allowed extension origin, on preflights and on responses. */
|
|
23
|
+
export declare function corsHeaders(origin: string): Record<string, string>;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { tokenMatches } from "./token.js";
|
|
2
|
+
// Who may talk to `docsworth serve`. It listens on 127.0.0.1 only, and on top of that
|
|
3
|
+
// every request must name this server as its Host (a DNS-rebound page names its own),
|
|
4
|
+
// and a browser request's Origin must be this server or an extension the user allowed.
|
|
5
|
+
// /mcp also needs the bearer token. There are no cookies: nothing a browser attaches on
|
|
6
|
+
// its own can authorize a request, so another site can't make one on the user's behalf.
|
|
7
|
+
/** A Chromium extension id: 32 letters a to p. */
|
|
8
|
+
export const EXTENSION_ID = /^[a-p]{32}$/;
|
|
9
|
+
const own = (port) => [`http://127.0.0.1:${port}`, `http://localhost:${port}`];
|
|
10
|
+
/** Host and Origin, checked on every request, static files included. */
|
|
11
|
+
export function checkOrigin(req, rules) {
|
|
12
|
+
const host = req.headers.host;
|
|
13
|
+
if (host !== `127.0.0.1:${rules.port}` && host !== `localhost:${rules.port}`)
|
|
14
|
+
return { ok: false, status: 403, message: "this server answers only to 127.0.0.1 and localhost on its own port" };
|
|
15
|
+
const origin = req.headers.origin;
|
|
16
|
+
if (origin === undefined || own(rules.port).includes(origin))
|
|
17
|
+
return { ok: true };
|
|
18
|
+
const ext = /^chrome-extension:\/\/([a-p]{32})$/.exec(origin)?.[1];
|
|
19
|
+
if (ext && rules.extensions.has(ext))
|
|
20
|
+
return { ok: true, cors: origin };
|
|
21
|
+
return { ok: false, status: 403, message: "requests from this origin are not allowed; an extension must be allowed with --allow-extension" };
|
|
22
|
+
}
|
|
23
|
+
/** The bearer token, checked on every /mcp request after checkOrigin. */
|
|
24
|
+
export function checkToken(req, rules) {
|
|
25
|
+
const m = /^Bearer (\S+)$/.exec(req.headers.authorization ?? "");
|
|
26
|
+
return tokenMatches(rules.token, m?.[1]) ? { ok: true } : { ok: false, status: 401, message: "a valid bearer token is required" };
|
|
27
|
+
}
|
|
28
|
+
/** CORS headers for an allowed extension origin, on preflights and on responses. */
|
|
29
|
+
export function corsHeaders(origin) {
|
|
30
|
+
return {
|
|
31
|
+
"access-control-allow-origin": origin,
|
|
32
|
+
"access-control-allow-methods": "GET, POST, DELETE",
|
|
33
|
+
"access-control-allow-headers": "authorization, content-type, accept, mcp-session-id, mcp-protocol-version, last-event-id",
|
|
34
|
+
"access-control-expose-headers": "mcp-session-id",
|
|
35
|
+
"access-control-max-age": "600",
|
|
36
|
+
vary: "Origin",
|
|
37
|
+
};
|
|
38
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { type Server } from "node:http";
|
|
2
|
+
import { SessionTable, type SessionOptions } from "./sessions.js";
|
|
3
|
+
export declare const DEFAULT_PORT = 4747;
|
|
4
|
+
/** Sent on every response, the static app's included. */
|
|
5
|
+
export declare const SECURITY_HEADERS: Readonly<Record<string, string>>;
|
|
6
|
+
export interface ServeOptions extends SessionOptions {
|
|
7
|
+
port: number;
|
|
8
|
+
token: string;
|
|
9
|
+
extensions: ReadonlySet<string>;
|
|
10
|
+
/** The built web app (dist/web), or null to serve /mcp only. */
|
|
11
|
+
webRoot: string | null;
|
|
12
|
+
}
|
|
13
|
+
export interface Serving {
|
|
14
|
+
server: Server;
|
|
15
|
+
sessions: SessionTable;
|
|
16
|
+
port: number;
|
|
17
|
+
/** The URL to open, with the token in the fragment (never sent to the server). */
|
|
18
|
+
url: string;
|
|
19
|
+
close(): Promise<void>;
|
|
20
|
+
}
|
|
21
|
+
export declare function startServer(opts: ServeOptions): Promise<Serving>;
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { createReadStream, statSync } from "node:fs";
|
|
2
|
+
import { createServer } from "node:http";
|
|
3
|
+
import { extname, join, resolve, sep } from "node:path";
|
|
4
|
+
import { checkOrigin, checkToken, corsHeaders } from "./auth.js";
|
|
5
|
+
import { SessionTable } from "./sessions.js";
|
|
6
|
+
// `docsworth serve`: the knowledge repo over MCP (Streamable HTTP) at /mcp, for the web app
|
|
7
|
+
// and the browser extension, plus the web app's static files at /. Bound to 127.0.0.1
|
|
8
|
+
// only. Every response carries the security headers; /mcp also needs the bearer token.
|
|
9
|
+
// The static files hold no data: everything the app shows comes through /mcp, under the
|
|
10
|
+
// same ACL, gate, policy, taint check and audit log as an agent's stdio session.
|
|
11
|
+
export const DEFAULT_PORT = 4747;
|
|
12
|
+
/** Sent on every response, the static app's included. */
|
|
13
|
+
export const SECURITY_HEADERS = {
|
|
14
|
+
"content-security-policy": "default-src 'self'; img-src 'self' data:; connect-src 'self'; base-uri 'none'; object-src 'none'; form-action 'none'; frame-ancestors 'none'",
|
|
15
|
+
"referrer-policy": "no-referrer",
|
|
16
|
+
"x-content-type-options": "nosniff",
|
|
17
|
+
};
|
|
18
|
+
const TYPES = {
|
|
19
|
+
".html": "text/html; charset=utf-8",
|
|
20
|
+
".js": "text/javascript; charset=utf-8",
|
|
21
|
+
".css": "text/css; charset=utf-8",
|
|
22
|
+
".svg": "image/svg+xml",
|
|
23
|
+
".png": "image/png",
|
|
24
|
+
".ico": "image/x-icon",
|
|
25
|
+
".json": "application/json",
|
|
26
|
+
".woff2": "font/woff2",
|
|
27
|
+
};
|
|
28
|
+
function refuse(res, v, extra = {}) {
|
|
29
|
+
const auth = v.status === 401 ? { "www-authenticate": 'Bearer realm="docsworth"' } : {};
|
|
30
|
+
res.writeHead(v.status, { ...SECURITY_HEADERS, ...extra, ...auth, "content-type": "application/json" });
|
|
31
|
+
res.end(JSON.stringify({ error: { code: v.status === 401 ? "unauthorized" : "forbidden", message: v.message } }));
|
|
32
|
+
}
|
|
33
|
+
/** A file under `webRoot` for a URL path, or the app's index for a client-side route. */
|
|
34
|
+
function staticFile(webRoot, pathname) {
|
|
35
|
+
let rel;
|
|
36
|
+
try {
|
|
37
|
+
rel = decodeURIComponent(pathname);
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
if (rel.includes("\0"))
|
|
43
|
+
return null;
|
|
44
|
+
const root = resolve(webRoot);
|
|
45
|
+
const file = resolve(root, `.${rel.endsWith("/") ? `${rel}index.html` : rel}`);
|
|
46
|
+
if (file !== root && !file.startsWith(root + sep))
|
|
47
|
+
return null; // never outside the app
|
|
48
|
+
try {
|
|
49
|
+
if (statSync(file).isFile())
|
|
50
|
+
return file;
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
/* not a file: maybe a client-side route */
|
|
54
|
+
}
|
|
55
|
+
// A path without an extension is one of the app's routes (/p/<id>, /graph): its index.
|
|
56
|
+
return extname(rel) === "" ? join(root, "index.html") : null;
|
|
57
|
+
}
|
|
58
|
+
function serveStatic(webRoot, req, res, pathname) {
|
|
59
|
+
if (req.method !== "GET" && req.method !== "HEAD") {
|
|
60
|
+
res.writeHead(405, { ...SECURITY_HEADERS, allow: "GET, HEAD" }).end();
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
const file = webRoot ? staticFile(webRoot, pathname) : null;
|
|
64
|
+
let size = 0;
|
|
65
|
+
try {
|
|
66
|
+
size = file ? statSync(file).size : 0;
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
/* index.html missing: the app isn't built */
|
|
70
|
+
}
|
|
71
|
+
if (!file || (!size && file.endsWith("index.html"))) {
|
|
72
|
+
const body = webRoot ? "Not found.\n" : "The web app isn't built into this install; /mcp is served.\n";
|
|
73
|
+
res.writeHead(404, { ...SECURITY_HEADERS, "content-type": "text/plain; charset=utf-8" }).end(body);
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
res.writeHead(200, {
|
|
77
|
+
...SECURITY_HEADERS,
|
|
78
|
+
"content-type": TYPES[extname(file)] ?? "application/octet-stream",
|
|
79
|
+
"content-length": String(size),
|
|
80
|
+
// Hashed asset names can be cached; the index must not be, or a new build wouldn't load.
|
|
81
|
+
"cache-control": file.endsWith("index.html") ? "no-store" : "max-age=31536000, immutable",
|
|
82
|
+
});
|
|
83
|
+
if (req.method === "HEAD")
|
|
84
|
+
res.end();
|
|
85
|
+
else
|
|
86
|
+
createReadStream(file).pipe(res);
|
|
87
|
+
}
|
|
88
|
+
export async function startServer(opts) {
|
|
89
|
+
const sessions = new SessionTable(opts);
|
|
90
|
+
let rules = { port: opts.port, token: opts.token, extensions: opts.extensions };
|
|
91
|
+
const server = createServer((req, res) => {
|
|
92
|
+
const origin = checkOrigin(req, rules);
|
|
93
|
+
if (!origin.ok)
|
|
94
|
+
return refuse(res, origin);
|
|
95
|
+
const pathname = new URL(req.url ?? "/", "http://localhost").pathname;
|
|
96
|
+
if (pathname !== "/mcp")
|
|
97
|
+
return serveStatic(opts.webRoot, req, res, pathname);
|
|
98
|
+
const cors = origin.cors ? corsHeaders(origin.cors) : {};
|
|
99
|
+
// A preflight can't carry the token; Host and Origin were checked above.
|
|
100
|
+
if (req.method === "OPTIONS") {
|
|
101
|
+
res.writeHead(204, { ...SECURITY_HEADERS, ...cors }).end();
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
const auth = checkToken(req, rules);
|
|
105
|
+
if (!auth.ok)
|
|
106
|
+
return refuse(res, auth, cors);
|
|
107
|
+
// Merged into whatever the transport writes.
|
|
108
|
+
for (const [k, v] of Object.entries({ ...SECURITY_HEADERS, ...cors, vary: "Origin" }))
|
|
109
|
+
res.setHeader(k, v);
|
|
110
|
+
sessions.handle(req, res).catch((err) => {
|
|
111
|
+
opts.log?.(`request failed: ${err.message}`);
|
|
112
|
+
if (!res.headersSent)
|
|
113
|
+
res.writeHead(500, { "content-type": "application/json" });
|
|
114
|
+
res.end(JSON.stringify({ error: { code: "internal", message: "internal error" } }));
|
|
115
|
+
});
|
|
116
|
+
});
|
|
117
|
+
await new Promise((done, fail) => {
|
|
118
|
+
server.once("error", fail);
|
|
119
|
+
server.listen(opts.port, "127.0.0.1", () => done());
|
|
120
|
+
});
|
|
121
|
+
// With port 0 (tests), the Host check needs the port the OS picked.
|
|
122
|
+
const port = server.address().port;
|
|
123
|
+
rules = { ...rules, port };
|
|
124
|
+
return {
|
|
125
|
+
server,
|
|
126
|
+
sessions,
|
|
127
|
+
port,
|
|
128
|
+
url: `http://127.0.0.1:${port}/#t=${opts.token}`,
|
|
129
|
+
async close() {
|
|
130
|
+
await sessions.close();
|
|
131
|
+
// Stop listening, then drop what is still open (a tab's event stream, a call still
|
|
132
|
+
// arriving): close() alone waits for every connection to finish, so Ctrl-C would hang.
|
|
133
|
+
const closed = new Promise((done) => server.close(() => done()));
|
|
134
|
+
server.closeAllConnections();
|
|
135
|
+
await closed;
|
|
136
|
+
},
|
|
137
|
+
};
|
|
138
|
+
}
|