@docsworth/cli 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -8
- package/dist/cli.js +52 -3
- package/dist/core/acl.d.ts +9 -0
- package/dist/core/acl.js +40 -6
- package/dist/core/adopt.d.ts +68 -0
- package/dist/core/adopt.js +665 -0
- package/dist/core/config.d.ts +6 -1
- package/dist/core/config.js +46 -2
- package/dist/core/edits.d.ts +13 -0
- package/dist/core/edits.js +20 -0
- package/dist/core/frontmatter.d.ts +12 -0
- package/dist/core/frontmatter.js +16 -7
- package/dist/core/gfm.d.ts +13 -0
- package/dist/core/gfm.js +244 -0
- package/dist/core/links.d.ts +82 -0
- package/dist/core/links.js +345 -0
- package/dist/core/lint.d.ts +1 -2
- package/dist/core/lint.js +40 -6
- package/dist/core/mcp/server.js +11 -2
- package/dist/core/printable.d.ts +4 -0
- package/dist/core/printable.js +16 -0
- package/dist/core/references.d.ts +38 -3
- package/dist/core/references.js +543 -41
- package/dist/core/repo.d.ts +4 -0
- package/dist/core/repo.js +3 -1
- package/dist/core/schema.d.ts +4 -0
- package/dist/core/schema.js +8 -6
- package/dist/core/search.d.ts +8 -0
- package/dist/core/search.js +34 -0
- package/dist/core/serve/server.d.ts +1 -2
- package/dist/core/serve/sessions.d.ts +5 -4
- package/dist/core/serve/sessions.js +102 -29
- package/dist/core/service.d.ts +118 -20
- package/dist/core/service.js +249 -91
- package/dist/core/stale.js +2 -1
- package/dist/core/tool-registry.js +6 -3
- package/dist/core/verify-pages.d.ts +19 -0
- package/dist/core/verify-pages.js +110 -0
- package/dist/core/version.d.ts +2 -2
- package/dist/core/version.js +2 -2
- package/dist/web/assets/index-8J-FUbb0.js +29 -0
- package/dist/web/assets/index-BGpAZzeM.css +1 -0
- package/dist/web/index.html +2 -2
- package/docs/format-spec.md +40 -2
- package/package.json +2 -2
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/bin/docsworth-mcp.mjs +1 -1
- package/dist/web/assets/index-BglLMIyN.css +0 -1
- package/dist/web/assets/index-DugFPoRu.js +0 -25
package/dist/core/repo.d.ts
CHANGED
|
@@ -10,11 +10,15 @@ export interface Page extends Frontmatter {
|
|
|
10
10
|
links: string[];
|
|
11
11
|
sha256: string;
|
|
12
12
|
unknownFields: string[];
|
|
13
|
+
/** Its frontmatter `slug`, if a string: where a docs site serves it (`siteRoute`). Not in the format; read for redaction and lint. */
|
|
14
|
+
slug?: string;
|
|
13
15
|
}
|
|
14
16
|
export interface InvalidPage {
|
|
15
17
|
path: string;
|
|
16
18
|
/** The `id` field if present and a string, so lint can still report duplicates. */
|
|
17
19
|
id?: string;
|
|
20
|
+
/** The `slug` field if a string: redaction hides the route it gives the page too. */
|
|
21
|
+
slug?: string;
|
|
18
22
|
issues: FieldIssue[];
|
|
19
23
|
}
|
|
20
24
|
export interface Repo {
|
package/dist/core/repo.js
CHANGED
|
@@ -67,11 +67,13 @@ export function parsePage(path, text) {
|
|
|
67
67
|
if (!doc.data)
|
|
68
68
|
return { path, issues: [{ field: "(frontmatter)", message: doc.error ?? "missing frontmatter" }] };
|
|
69
69
|
const rawId = typeof doc.data.id === "string" ? doc.data.id : undefined;
|
|
70
|
+
const slug = typeof doc.data.slug === "string" ? { slug: doc.data.slug } : {};
|
|
70
71
|
const v = validateFrontmatter(doc.data);
|
|
71
72
|
if (!v.ok)
|
|
72
|
-
return { path, id: rawId, issues: v.issues };
|
|
73
|
+
return { path, id: rawId, ...slug, issues: v.issues };
|
|
73
74
|
return {
|
|
74
75
|
...v.value,
|
|
76
|
+
...slug,
|
|
75
77
|
path,
|
|
76
78
|
body: doc.body,
|
|
77
79
|
headings: extractHeadings(doc.body),
|
package/dist/core/schema.d.ts
CHANGED
|
@@ -5,6 +5,10 @@ export declare const REQUIRED_FIELDS: readonly ["title", "id", "type", "status",
|
|
|
5
5
|
export declare const OPTIONAL_FIELDS: readonly ["supersedes", "applies_to", "acl", "description"];
|
|
6
6
|
/** Default ACL: anyone with read access to the knowledge repo. */
|
|
7
7
|
export declare const ACL_INTERNAL = "internal";
|
|
8
|
+
/** A team slug, or `internal` (owner_team, acl). */
|
|
9
|
+
export declare const SLUG_RE: RegExp;
|
|
10
|
+
/** A page id: lowercase letters, digits and dashes. */
|
|
11
|
+
export declare const ID_RE: RegExp;
|
|
8
12
|
export declare const FrontmatterSchema: z.ZodObject<{
|
|
9
13
|
title: z.ZodString;
|
|
10
14
|
id: z.ZodString;
|
package/dist/core/schema.js
CHANGED
|
@@ -15,22 +15,24 @@ export const REQUIRED_FIELDS = [
|
|
|
15
15
|
export const OPTIONAL_FIELDS = ["supersedes", "applies_to", "acl", "description"];
|
|
16
16
|
/** Default ACL: anyone with read access to the knowledge repo. */
|
|
17
17
|
export const ACL_INTERNAL = "internal";
|
|
18
|
-
|
|
19
|
-
const
|
|
18
|
+
/** A team slug, or `internal` (owner_team, acl). */
|
|
19
|
+
export const SLUG_RE = /^[a-z0-9][a-z0-9_-]*$/;
|
|
20
|
+
/** A page id: lowercase letters, digits and dashes. */
|
|
21
|
+
export const ID_RE = /^[a-z0-9][a-z0-9-]*$/;
|
|
20
22
|
export const FrontmatterSchema = z.object({
|
|
21
23
|
title: z.string().trim().min(1),
|
|
22
|
-
id: z.string().regex(
|
|
24
|
+
id: z.string().regex(ID_RE, "lowercase letters, digits and dashes"),
|
|
23
25
|
type: z.enum(PAGE_TYPES),
|
|
24
26
|
status: z.enum(STATUSES),
|
|
25
27
|
// A bare team slug; the org comes from config (github.org). Never `@user`.
|
|
26
|
-
owner_team: z.string().regex(
|
|
28
|
+
owner_team: z.string().regex(SLUG_RE, "a team slug like `platform` (no @, no org)"),
|
|
27
29
|
last_verified: z
|
|
28
30
|
.string()
|
|
29
31
|
.refine((s) => parseDate(s) !== null, "a real calendar date as YYYY-MM-DD"),
|
|
30
32
|
review_every: z.string().regex(DURATION_RE, "a period like 30d, 12w, 6m or 1y"),
|
|
31
|
-
supersedes: z.array(z.string().regex(
|
|
33
|
+
supersedes: z.array(z.string().regex(ID_RE)).optional(),
|
|
32
34
|
applies_to: z.array(z.string().min(1)).optional(),
|
|
33
|
-
acl: z.string().regex(
|
|
35
|
+
acl: z.string().regex(SLUG_RE, "`internal` or a team slug").default(ACL_INTERNAL),
|
|
34
36
|
description: z.string().optional(),
|
|
35
37
|
});
|
|
36
38
|
/** Validate raw frontmatter, returning issues keyed by field for the linter. */
|
package/dist/core/search.d.ts
CHANGED
|
@@ -20,5 +20,13 @@ export declare class SearchIndex {
|
|
|
20
20
|
/** Top hits for a query; `accept` filters candidates before ranking (ACL, filters). */
|
|
21
21
|
search(query: string, limit: number, accept?: (index: number) => boolean): Hit[];
|
|
22
22
|
}
|
|
23
|
+
/**
|
|
24
|
+
* A page's markdown as a reader sees it, to cut snippets from: its words, code as written,
|
|
25
|
+
* an image as its label, a table's cells (`a | b`); no markup, link targets or raw HTML.
|
|
26
|
+
* It reads the tree the web app renders from (`parseGfm`), so a snippet cut from it holds
|
|
27
|
+
* exactly the words the page shows, and never ends inside markup (search's
|
|
28
|
+
* `snippets: "text"`).
|
|
29
|
+
*/
|
|
30
|
+
export declare function plainText(markdown: string): string;
|
|
23
31
|
/** A ~maxLen window of `body` around the densest run of query terms. */
|
|
24
32
|
export declare function snippet(body: string, query: string, maxLen?: number): string;
|
package/dist/core/search.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// In-memory BM25 (BM25F-lite: per-field boosts folded into term frequency).
|
|
2
2
|
// The plan says "BM25 first, embeddings later"; this keeps search dependency-free and
|
|
3
3
|
// fast enough for 5,000 pages (scripts/bench-search.ts holds it to p95 < 500 ms).
|
|
4
|
+
import { isTableCell, parseGfm, textIn } from "./gfm.js";
|
|
4
5
|
const STOPWORDS = new Set("a an and are as at be by can do does for from has have how i if in into is it its of on or our should that the their then there this to was we what when where which who why will with you your".split(" "));
|
|
5
6
|
const BOOST = { title: 3, keywords: 3, headings: 2, description: 2, body: 1 };
|
|
6
7
|
const K1 = 1.2;
|
|
@@ -74,6 +75,39 @@ export class SearchIndex {
|
|
|
74
75
|
.slice(0, limit);
|
|
75
76
|
}
|
|
76
77
|
}
|
|
78
|
+
// Blocks whose text a reader sees apart from what's next to them.
|
|
79
|
+
const BLOCKS = new Set(["paragraph", "heading", "item", "block_quote", "code_block", "custom_block", "thematic_break"]);
|
|
80
|
+
/**
|
|
81
|
+
* A page's markdown as a reader sees it, to cut snippets from: its words, code as written,
|
|
82
|
+
* an image as its label, a table's cells (`a | b`); no markup, link targets or raw HTML.
|
|
83
|
+
* It reads the tree the web app renders from (`parseGfm`), so a snippet cut from it holds
|
|
84
|
+
* exactly the words the page shows, and never ends inside markup (search's
|
|
85
|
+
* `snippets: "text"`).
|
|
86
|
+
*/
|
|
87
|
+
export function plainText(markdown) {
|
|
88
|
+
let out = "";
|
|
89
|
+
const walker = parseGfm(markdown).walker();
|
|
90
|
+
for (let e = walker.next(); e; e = walker.next()) {
|
|
91
|
+
const n = e.node;
|
|
92
|
+
if (!e.entering)
|
|
93
|
+
continue;
|
|
94
|
+
if (n.type === "text" || n.type === "code")
|
|
95
|
+
out += n.literal ?? "";
|
|
96
|
+
else if (n.type === "code_block")
|
|
97
|
+
out += ` ${n.literal ?? ""} `;
|
|
98
|
+
else if (isTableCell(n))
|
|
99
|
+
out += n.prev ? " | " : " ";
|
|
100
|
+
// As the web app labels an image (packages/ui/src/markdown.ts): `image: ` and its words,
|
|
101
|
+
// or its address when it has none, once; what's inside it (an image too) isn't shown.
|
|
102
|
+
else if (n.type === "image") {
|
|
103
|
+
out += `image: ${textIn(n) || n.destination || ""}`;
|
|
104
|
+
walker.resumeAt(n, false);
|
|
105
|
+
}
|
|
106
|
+
else if (n.type === "softbreak" || n.type === "linebreak" || BLOCKS.has(n.type))
|
|
107
|
+
out += " ";
|
|
108
|
+
}
|
|
109
|
+
return out.replace(/\s+/g, " ").trim();
|
|
110
|
+
}
|
|
77
111
|
/** A ~maxLen window of `body` around the densest run of query terms. */
|
|
78
112
|
export function snippet(body, query, maxLen = 240) {
|
|
79
113
|
const text = body.replace(/\s+/g, " ").trim();
|
|
@@ -3,8 +3,7 @@ import { SessionTable, type SessionOptions } from "./sessions.js";
|
|
|
3
3
|
export declare const DEFAULT_PORT = 4747;
|
|
4
4
|
/** Sent on every response, the static app's included. */
|
|
5
5
|
export declare const SECURITY_HEADERS: Readonly<Record<string, string>>;
|
|
6
|
-
export interface ServeOptions extends
|
|
7
|
-
knowledgeRoot: string;
|
|
6
|
+
export interface ServeOptions extends SessionOptions {
|
|
8
7
|
port: number;
|
|
9
8
|
token: string;
|
|
10
9
|
extensions: ReadonlySet<string>;
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import type { IncomingMessage, ServerResponse } from "node:http";
|
|
2
2
|
import type { Identity } from "../acl.js";
|
|
3
3
|
import { type AuditLog } from "../audit-log.js";
|
|
4
|
-
import { type IdentitySource, KnowledgeService } from "../service.js";
|
|
4
|
+
import { type IdentitySource, KnowledgeService, RepoLoader } from "../service.js";
|
|
5
5
|
export interface SessionOptions {
|
|
6
6
|
root: string;
|
|
7
|
-
knowledgeRoot: string;
|
|
8
7
|
identity: IdentitySource;
|
|
9
8
|
/** The identity proven at server start, to attribute the first event of each session. */
|
|
10
9
|
who: Identity;
|
|
@@ -22,7 +21,8 @@ export declare class SessionTable {
|
|
|
22
21
|
private readonly sessions;
|
|
23
22
|
/** Sessions being set up: counted against the cap before they have an id. */
|
|
24
23
|
private starting;
|
|
25
|
-
|
|
24
|
+
/** One load of the repo for every session; its watcher marks it stale on any change. */
|
|
25
|
+
readonly loader: RepoLoader;
|
|
26
26
|
private readonly sweeper;
|
|
27
27
|
private readonly max;
|
|
28
28
|
private readonly idleMs;
|
|
@@ -30,9 +30,10 @@ export declare class SessionTable {
|
|
|
30
30
|
get size(): number;
|
|
31
31
|
/** The services of live sessions (tests). */
|
|
32
32
|
services(): KnowledgeService[];
|
|
33
|
-
private invalidate;
|
|
34
33
|
handle(req: IncomingMessage, res: ServerResponse): Promise<void>;
|
|
35
34
|
private open;
|
|
35
|
+
/** The session that has had no request in flight for longest, if any. */
|
|
36
|
+
private idlest;
|
|
36
37
|
/** Close sessions idle longer than the limit, never one with a request in flight. */
|
|
37
38
|
sweep(now?: number): Promise<void>;
|
|
38
39
|
close(): Promise<void>;
|
|
@@ -1,22 +1,54 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { watch } from "node:fs";
|
|
3
|
-
import { join } from "node:path";
|
|
4
2
|
import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
|
|
3
|
+
import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js";
|
|
5
4
|
import { actorOf } from "../audit-log.js";
|
|
6
5
|
import { auditTransport } from "../mcp/audit-transport.js";
|
|
7
6
|
import { createMcpServer } from "../mcp/server.js";
|
|
8
|
-
import { KnowledgeService,
|
|
7
|
+
import { KnowledgeService, RepoLoader } from "../service.js";
|
|
9
8
|
export const MAX_SESSIONS = 16;
|
|
9
|
+
/** An initialize request is a few hundred bytes; a first request bigger than this isn't one. */
|
|
10
|
+
const MAX_START_BYTES = 64 * 1024;
|
|
10
11
|
export const SESSION_IDLE_MS = 30 * 60_000;
|
|
11
12
|
function json(res, status, body) {
|
|
12
13
|
res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(body));
|
|
13
14
|
}
|
|
15
|
+
const NOT_A_START = { status: 400, error: { code: "invalid", message: "start a session with an initialize request" } };
|
|
16
|
+
/**
|
|
17
|
+
* A first request's body, if it can start a session: the headers the transport requires,
|
|
18
|
+
* and exactly one initialize request. Checked before a slot is counted or reclaimed, so a
|
|
19
|
+
* request the transport would refuse can't close someone's idle session for nothing.
|
|
20
|
+
*/
|
|
21
|
+
async function startBody(req) {
|
|
22
|
+
const accept = req.headers.accept ?? "";
|
|
23
|
+
if (!accept.includes("application/json") || !accept.includes("text/event-stream"))
|
|
24
|
+
return { status: 406, error: { code: "invalid", message: "the client must accept both application/json and text/event-stream" } };
|
|
25
|
+
if (!(req.headers["content-type"] ?? "").includes("application/json"))
|
|
26
|
+
return { status: 415, error: { code: "invalid", message: "the body must be application/json" } };
|
|
27
|
+
const chunks = [];
|
|
28
|
+
let size = 0;
|
|
29
|
+
for await (const chunk of req) {
|
|
30
|
+
size += chunk.length;
|
|
31
|
+
if (size > MAX_START_BYTES)
|
|
32
|
+
return NOT_A_START;
|
|
33
|
+
chunks.push(chunk);
|
|
34
|
+
}
|
|
35
|
+
let body;
|
|
36
|
+
try {
|
|
37
|
+
body = JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return NOT_A_START;
|
|
41
|
+
}
|
|
42
|
+
const only = Array.isArray(body) ? (body.length === 1 ? body[0] : undefined) : body;
|
|
43
|
+
return isInitializeRequest(only) ? { body } : NOT_A_START;
|
|
44
|
+
}
|
|
14
45
|
export class SessionTable {
|
|
15
46
|
opts;
|
|
16
47
|
sessions = new Map();
|
|
17
48
|
/** Sessions being set up: counted against the cap before they have an id. */
|
|
18
49
|
starting = 0;
|
|
19
|
-
watcher
|
|
50
|
+
/** One load of the repo for every session; its watcher marks it stale on any change. */
|
|
51
|
+
loader;
|
|
20
52
|
sweeper;
|
|
21
53
|
max;
|
|
22
54
|
idleMs;
|
|
@@ -24,15 +56,22 @@ export class SessionTable {
|
|
|
24
56
|
this.opts = opts;
|
|
25
57
|
this.max = opts.maxSessions ?? MAX_SESSIONS;
|
|
26
58
|
this.idleMs = opts.idleMs ?? SESSION_IDLE_MS;
|
|
27
|
-
// One watcher for every session: a change marks
|
|
28
|
-
//
|
|
29
|
-
// It watches pages only. The config and CODEOWNERS (the policy, admins,
|
|
30
|
-
// outside the knowledge root, and every call already stats them
|
|
31
|
-
//
|
|
32
|
-
//
|
|
33
|
-
this.
|
|
34
|
-
|
|
35
|
-
|
|
59
|
+
// One load and one watcher for every session: a change marks the load stale, and the
|
|
60
|
+
// next call from any session builds a new one, never in the middle of a write that is
|
|
61
|
+
// still landing. It watches pages only. The config and CODEOWNERS (the policy, admins,
|
|
62
|
+
// owners) sit outside the knowledge root, and every call already stats them, so a
|
|
63
|
+
// tightened policy applies to an open session's next call (test/serve.test.ts,
|
|
64
|
+
// "applies a tightened policy").
|
|
65
|
+
this.loader = new RepoLoader(opts.root);
|
|
66
|
+
// A new load is built inside one session's call; every session with nothing in flight
|
|
67
|
+
// moves onto it now, and a busy one when its last request ends (handle). So however
|
|
68
|
+
// the repo changes, the sessions hold one load between them, not one each.
|
|
69
|
+
this.loader.onLoad = () => {
|
|
70
|
+
for (const s of this.sessions.values())
|
|
71
|
+
if (s.active === 0)
|
|
72
|
+
s.service.release();
|
|
73
|
+
};
|
|
74
|
+
this.loader.watch();
|
|
36
75
|
this.sweeper = setInterval(() => void this.sweep(), Math.min(60_000, this.idleMs));
|
|
37
76
|
this.sweeper.unref();
|
|
38
77
|
}
|
|
@@ -43,10 +82,6 @@ export class SessionTable {
|
|
|
43
82
|
services() {
|
|
44
83
|
return [...this.sessions.values()].map((s) => s.service);
|
|
45
84
|
}
|
|
46
|
-
invalidate() {
|
|
47
|
-
for (const s of this.sessions.values())
|
|
48
|
-
s.service.invalidate();
|
|
49
|
-
}
|
|
50
85
|
async handle(req, res) {
|
|
51
86
|
const sid = req.headers["mcp-session-id"];
|
|
52
87
|
if (sid !== undefined) {
|
|
@@ -60,40 +95,68 @@ export class SessionTable {
|
|
|
60
95
|
finally {
|
|
61
96
|
session.active--;
|
|
62
97
|
session.lastUsed = Date.now();
|
|
98
|
+
if (session.active === 0)
|
|
99
|
+
session.service.release();
|
|
63
100
|
}
|
|
64
101
|
}
|
|
65
102
|
if (req.method !== "POST")
|
|
66
|
-
return json(res, 400, { error:
|
|
67
|
-
|
|
68
|
-
if (
|
|
69
|
-
return json(res,
|
|
103
|
+
return json(res, 400, { error: NOT_A_START.error });
|
|
104
|
+
const first = await startBody(req);
|
|
105
|
+
if (!("body" in first))
|
|
106
|
+
return json(res, first.status, { error: first.error });
|
|
107
|
+
// Checked and counted with no await in between, so concurrent starts can't pass the cap.
|
|
108
|
+
// A full server makes room by closing the session idle longest: a closed tab never
|
|
109
|
+
// tells the server, so refusing would lock the user out until it expired. Only when
|
|
110
|
+
// every slot is busy (answering, or still starting) is a new one refused.
|
|
111
|
+
let reclaimed;
|
|
112
|
+
if (this.sessions.size + this.starting >= this.max) {
|
|
113
|
+
reclaimed = this.idlest();
|
|
114
|
+
if (!reclaimed)
|
|
115
|
+
return json(res, 503, { error: { code: "unavailable", message: `this server is answering ${this.max} sessions at once; try again in a moment` } });
|
|
116
|
+
this.sessions.delete(reclaimed.http.sessionId); // out of the table before any await
|
|
117
|
+
}
|
|
118
|
+
// This start's slot: counted in `starting` until its session joins the table, then
|
|
119
|
+
// there instead (never in both, or a start beside a slow first answer finds no room).
|
|
70
120
|
this.starting++;
|
|
121
|
+
let reserved = true;
|
|
122
|
+
const joined = () => {
|
|
123
|
+
if (reserved)
|
|
124
|
+
this.starting--;
|
|
125
|
+
reserved = false;
|
|
126
|
+
};
|
|
71
127
|
try {
|
|
72
|
-
|
|
128
|
+
if (reclaimed)
|
|
129
|
+
await reclaimed.server.close().catch((err) => this.opts.log?.(`closing an idle session failed: ${err.message}`));
|
|
130
|
+
const session = await this.open(joined);
|
|
73
131
|
try {
|
|
74
|
-
await session.http.handleRequest(req, res);
|
|
132
|
+
await session.http.handleRequest(req, res, first.body); // the body is read: hand it over
|
|
75
133
|
}
|
|
76
134
|
finally {
|
|
77
135
|
// Busy from birth: the session joins the table while this request is still
|
|
78
136
|
// being answered, so a sweep in between must not take it.
|
|
79
137
|
session.active--;
|
|
80
138
|
session.lastUsed = Date.now();
|
|
139
|
+
if (session.active === 0)
|
|
140
|
+
session.service.release(); // as for any request (above)
|
|
81
141
|
}
|
|
82
142
|
// A first request that wasn't an initialize gets the SDK's error and no session.
|
|
83
143
|
if (!session.http.sessionId)
|
|
84
144
|
await session.server.close();
|
|
85
145
|
}
|
|
86
146
|
finally {
|
|
87
|
-
|
|
147
|
+
joined();
|
|
88
148
|
}
|
|
89
149
|
}
|
|
90
|
-
async open() {
|
|
150
|
+
async open(joined) {
|
|
91
151
|
const o = this.opts;
|
|
92
|
-
const service = new KnowledgeService(o.root, o.identity, o.now);
|
|
152
|
+
const service = new KnowledgeService(o.root, o.identity, o.now, this.loader);
|
|
93
153
|
const http = new StreamableHTTPServerTransport({
|
|
94
154
|
sessionIdGenerator: randomUUID,
|
|
95
155
|
enableJsonResponse: true,
|
|
96
|
-
onsessioninitialized: (id) =>
|
|
156
|
+
onsessioninitialized: (id) => {
|
|
157
|
+
this.sessions.set(id, session);
|
|
158
|
+
joined();
|
|
159
|
+
},
|
|
97
160
|
});
|
|
98
161
|
const transport = o.audit
|
|
99
162
|
? auditTransport(http, o.audit, () => o.identity.get(), {
|
|
@@ -116,15 +179,25 @@ export class SessionTable {
|
|
|
116
179
|
await server.connect(transport);
|
|
117
180
|
return session;
|
|
118
181
|
}
|
|
182
|
+
/** The session that has had no request in flight for longest, if any. */
|
|
183
|
+
idlest() {
|
|
184
|
+
let found;
|
|
185
|
+
for (const s of this.sessions.values())
|
|
186
|
+
if (s.active === 0 && (!found || s.lastUsed < found.lastUsed))
|
|
187
|
+
found = s;
|
|
188
|
+
return found;
|
|
189
|
+
}
|
|
119
190
|
/** Close sessions idle longer than the limit, never one with a request in flight. */
|
|
120
191
|
async sweep(now = Date.now()) {
|
|
121
192
|
const idle = [...this.sessions.values()].filter((s) => s.active === 0 && now - s.lastUsed > this.idleMs);
|
|
193
|
+
// Out of the table first, so a start can't pick one of these to reclaim as well.
|
|
194
|
+
for (const s of idle)
|
|
195
|
+
this.sessions.delete(s.http.sessionId);
|
|
122
196
|
await Promise.all(idle.map((s) => s.server.close()));
|
|
123
197
|
}
|
|
124
198
|
async close() {
|
|
125
199
|
clearInterval(this.sweeper);
|
|
126
|
-
this.
|
|
127
|
-
this.watcher = null;
|
|
200
|
+
this.loader.close();
|
|
128
201
|
await Promise.all([...this.sessions.values()].map((s) => s.server.close()));
|
|
129
202
|
}
|
|
130
203
|
}
|
package/dist/core/service.d.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { type Identity } from "./acl.js";
|
|
2
|
+
import { type MirrorPage } from "./mirror-view.js";
|
|
2
3
|
import { type NewPageType } from "./new-page.js";
|
|
3
4
|
import { type Policy } from "./policy.js";
|
|
4
5
|
import type { GitHubClient } from "./github.js";
|
|
5
|
-
import { type Page } from "./repo.js";
|
|
6
|
+
import { type Page, type Repo } from "./repo.js";
|
|
6
7
|
import { type PageType } from "./schema.js";
|
|
8
|
+
import { SearchIndex } from "./search.js";
|
|
7
9
|
import { type StaleEntry } from "./stale.js";
|
|
8
10
|
import { type Edit } from "./writes.js";
|
|
9
11
|
export type ToolErrorCode = "not_found" | "forbidden" | "invalid" | "unavailable";
|
|
@@ -67,30 +69,105 @@ export declare const DRAFT_MAX_CHARS: number;
|
|
|
67
69
|
export declare const DRAFT_NAME_ATTEMPTS = 3;
|
|
68
70
|
/** At most this many mirrored pages per search: context, not a second result list. */
|
|
69
71
|
export declare const MIRROR_LIMIT = 3;
|
|
72
|
+
type MirrorView = {
|
|
73
|
+
pages: MirrorPage[];
|
|
74
|
+
origin: MirrorPage[];
|
|
75
|
+
hidden: MirrorPage[];
|
|
76
|
+
index: SearchIndex;
|
|
77
|
+
};
|
|
78
|
+
type Snapshot = {
|
|
79
|
+
commit: string | null;
|
|
80
|
+
dirty: boolean | null;
|
|
81
|
+
content_sha256: string;
|
|
82
|
+
};
|
|
83
|
+
/**
|
|
84
|
+
* One load of the repo: the pages, their search index and the served mirror. Never changed
|
|
85
|
+
* after it is built, except two caches filled on first use: the mirror's views (one per
|
|
86
|
+
* audience) and the snapshot. A reload builds a new one, so a call that took this one keeps
|
|
87
|
+
* a consistent view.
|
|
88
|
+
*/
|
|
89
|
+
interface Loaded {
|
|
90
|
+
readonly repo: Repo;
|
|
91
|
+
readonly index: SearchIndex;
|
|
92
|
+
readonly mirror: MirrorPage[];
|
|
93
|
+
/** Keyed by audience (repo access and teams), so services with any identity can share it. */
|
|
94
|
+
readonly mirrorViews: Map<string, MirrorView>;
|
|
95
|
+
/** mtimes of the non-page inputs (config, CODEOWNERS) it was built from. */
|
|
96
|
+
readonly inputs: string;
|
|
97
|
+
snap: Snapshot | null;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Loads the repo and keeps it until something changes, for one service or, in `docsworth
|
|
101
|
+
* serve`, for every session's service: one copy of the pages and their index however many
|
|
102
|
+
* tabs are open (a 3,000-page repo took about 50 MB per copy). It owns the one file watcher.
|
|
103
|
+
* What belongs to a session (its taint, its cached default branch) stays on the service.
|
|
104
|
+
*/
|
|
105
|
+
export declare class RepoLoader {
|
|
106
|
+
private readonly root;
|
|
107
|
+
private loaded;
|
|
108
|
+
private dirty;
|
|
109
|
+
/** watch() was asked for, and close() hasn't stopped it. */
|
|
110
|
+
private watching;
|
|
111
|
+
private watcher;
|
|
112
|
+
private settle;
|
|
113
|
+
/** The directory being watched, or to be once it exists (tests read it). */
|
|
114
|
+
watchedDir: string | null;
|
|
115
|
+
/** How many times the repo was loaded (tests). */
|
|
116
|
+
loads: number;
|
|
117
|
+
/** Called after each load: serve moves its idle sessions onto it, so no old load is kept. */
|
|
118
|
+
onLoad: (() => void) | null;
|
|
119
|
+
constructor(root: string);
|
|
120
|
+
/** The current load: the same object until a change is seen, then a new one, built once. */
|
|
121
|
+
current(): Loaded;
|
|
122
|
+
/** The newest load, without looking for changes. */
|
|
123
|
+
latest(): Loaded | null;
|
|
124
|
+
/** Has a change been seen that the next call will load (tests). */
|
|
125
|
+
get pending(): boolean;
|
|
126
|
+
private load;
|
|
127
|
+
/** Config and CODEOWNERS live outside the watched knowledge root; stat them per call. */
|
|
128
|
+
private inputsStamp;
|
|
129
|
+
/** Load again on the next call. */
|
|
130
|
+
invalidate(): void;
|
|
131
|
+
/** Mark the load stale on any change under the knowledge root. */
|
|
132
|
+
watch(): void;
|
|
133
|
+
/** Watch `repo`'s knowledge root, in place of whatever was watched before. */
|
|
134
|
+
private follow;
|
|
135
|
+
close(): void;
|
|
136
|
+
private stop;
|
|
137
|
+
}
|
|
70
138
|
export declare class KnowledgeService {
|
|
71
139
|
private readonly root;
|
|
72
140
|
private readonly identity;
|
|
73
141
|
private readonly now;
|
|
142
|
+
private readonly loader;
|
|
143
|
+
/** A loader this service made is its own to close; a shared one belongs to its owner. */
|
|
144
|
+
private readonly ownsLoader;
|
|
145
|
+
private loaded;
|
|
74
146
|
private repo;
|
|
75
147
|
private index;
|
|
76
|
-
private dirty;
|
|
77
|
-
/** mtimes of the non-page inputs (config, CODEOWNERS) the loaded repo was built from. */
|
|
78
|
-
private inputs;
|
|
79
|
-
private watcher;
|
|
80
|
-
private watchedDir;
|
|
81
|
-
private settle;
|
|
82
148
|
private defaultBranch;
|
|
83
149
|
/** Mirrored pages, when mirror.serve is on; one search index per audience (views). */
|
|
84
150
|
private mirror;
|
|
85
|
-
private
|
|
151
|
+
private mirrorViews;
|
|
86
152
|
/** Team ACLs whose content this session has seen (write-path taint check). */
|
|
87
153
|
private readonly tainted;
|
|
88
154
|
/** The random end of a drafted page's id and file name (replaced in tests). */
|
|
89
155
|
private draftSuffix;
|
|
90
|
-
|
|
156
|
+
/** `loader`: a load shared with other services (`docsworth serve`'s sessions); else its own. */
|
|
157
|
+
constructor(root: string, identity: IdentitySource, now?: () => Date, loader?: RepoLoader);
|
|
158
|
+
/** Load the repo again now. */
|
|
91
159
|
reload(): void;
|
|
92
|
-
/**
|
|
93
|
-
|
|
160
|
+
/**
|
|
161
|
+
* Point at the loader's newest load, letting go of an older one. Only while no call of
|
|
162
|
+
* this service is running: `docsworth serve` calls it for a session with no request in
|
|
163
|
+
* flight, so an idle tab doesn't keep a whole old copy of the repo alive.
|
|
164
|
+
*/
|
|
165
|
+
release(): void;
|
|
166
|
+
/** Take a new load, and drop what this service cached from the last one. */
|
|
167
|
+
private adopt;
|
|
168
|
+
/** The watched directory and a pending reload live on the loader (tests read them here). */
|
|
169
|
+
private get watchedDir();
|
|
170
|
+
private get dirty();
|
|
94
171
|
/**
|
|
95
172
|
* The mirrored pages `who` may see, each with links to the ones they can't see
|
|
96
173
|
* redacted (and to repo pages they can't see: a wiki page can link to one by its URL),
|
|
@@ -98,23 +175,19 @@ export declare class KnowledgeService {
|
|
|
98
175
|
* a page outside it. One per audience (the caller's teams), rebuilt on reload.
|
|
99
176
|
*/
|
|
100
177
|
private mirrorView;
|
|
101
|
-
/** Reload on the next call.
|
|
178
|
+
/** Reload on the next call. */
|
|
102
179
|
invalidate(): void;
|
|
103
180
|
/** Mark the index dirty on any change under the knowledge root. */
|
|
104
181
|
watch(): void;
|
|
182
|
+
/** Stops this service's own watcher; a shared loader's is its owner's to stop. */
|
|
105
183
|
close(): void;
|
|
106
184
|
private current;
|
|
107
|
-
private snap;
|
|
108
185
|
/**
|
|
109
186
|
* The knowledge actually being served: the checkout's HEAD and whether the served files
|
|
110
187
|
* differ from it, plus a hash of the loaded config, ownership and pages. Refreshed after
|
|
111
188
|
* every reload, so an audit event names the snapshot its answer came from.
|
|
112
189
|
*/
|
|
113
|
-
snapshot():
|
|
114
|
-
commit: string | null;
|
|
115
|
-
dirty: boolean | null;
|
|
116
|
-
content_sha256: string;
|
|
117
|
-
};
|
|
190
|
+
snapshot(): Snapshot;
|
|
118
191
|
/** Every page id, team and applies_to tag the repo uses: what an audited argument may record. */
|
|
119
192
|
knownValues(): {
|
|
120
193
|
ids: Set<string>;
|
|
@@ -141,11 +214,14 @@ export declare class KnowledgeService {
|
|
|
141
214
|
applies_to?: string;
|
|
142
215
|
include_stale?: boolean;
|
|
143
216
|
limit?: number;
|
|
217
|
+
/** "text": snippets cut from the page as a reader sees it (the web app); else markdown. */
|
|
218
|
+
snippets?: "markdown" | "text";
|
|
144
219
|
}): Promise<{
|
|
145
220
|
results: SearchResult[];
|
|
146
221
|
excluded: Excluded[];
|
|
147
222
|
mirrored?: MirroredHit[];
|
|
148
223
|
note?: string;
|
|
224
|
+
snippets?: "text";
|
|
149
225
|
}>;
|
|
150
226
|
read(args: {
|
|
151
227
|
id: string;
|
|
@@ -228,8 +304,15 @@ export declare class KnowledgeService {
|
|
|
228
304
|
writes_off?: string | undefined;
|
|
229
305
|
pro: string[];
|
|
230
306
|
}>;
|
|
231
|
-
/**
|
|
232
|
-
|
|
307
|
+
/**
|
|
308
|
+
* Counts over the caller's visible pages, by team, with what is due soon and what is stale.
|
|
309
|
+
* `list_limit` keeps a big repo's overview short: the first of each list, the most pressing
|
|
310
|
+
* first, with `due_soon_total` and `stale_total`. Only the pages returned are named, so only
|
|
311
|
+
* those taint the session; the counts above the lists name no page.
|
|
312
|
+
*/
|
|
313
|
+
uiOverview(args?: {
|
|
314
|
+
list_limit?: number;
|
|
315
|
+
}): Promise<{
|
|
233
316
|
as_of: string;
|
|
234
317
|
totals: {
|
|
235
318
|
pages: number;
|
|
@@ -248,6 +331,9 @@ export declare class KnowledgeService {
|
|
|
248
331
|
}[];
|
|
249
332
|
due_soon: PageSummary[];
|
|
250
333
|
stale: StaleEntry[];
|
|
334
|
+
due_soon_total: number;
|
|
335
|
+
stale_total: number;
|
|
336
|
+
skipped_files: number;
|
|
251
337
|
}>;
|
|
252
338
|
/**
|
|
253
339
|
* A page as `read` shows it, with `links`: the repo path and id of each visible page it
|
|
@@ -295,9 +381,15 @@ export declare class KnowledgeService {
|
|
|
295
381
|
* Visible pages and the links between them. A link to a page the caller can't see is no
|
|
296
382
|
* edge; `has_restricted_links` says a page has one, exactly as its `read` text shows a
|
|
297
383
|
* placeholder for it, so the graph reveals nothing `read` doesn't.
|
|
384
|
+
*
|
|
385
|
+
* `max_nodes` keeps a big repo drawable: past it, the most-linked pages (links between
|
|
386
|
+
* visible pages only; ties by id) and the links among them. `total` says how many it
|
|
387
|
+
* chose from, and `teams` how many visible pages each team has, a count that names no
|
|
388
|
+
* page. Only the returned pages are named, so only those taint the session.
|
|
298
389
|
*/
|
|
299
390
|
uiGraph(args?: {
|
|
300
391
|
owner_team?: string;
|
|
392
|
+
max_nodes?: number;
|
|
301
393
|
}): Promise<{
|
|
302
394
|
nodes: {
|
|
303
395
|
id: string;
|
|
@@ -316,6 +408,11 @@ export declare class KnowledgeService {
|
|
|
316
408
|
from: string;
|
|
317
409
|
to: string;
|
|
318
410
|
}[];
|
|
411
|
+
total: number;
|
|
412
|
+
teams: {
|
|
413
|
+
team: string;
|
|
414
|
+
pages: number;
|
|
415
|
+
}[];
|
|
319
416
|
}>;
|
|
320
417
|
/** What every write needs first: the policy, a live identity, a GitHub repo. */
|
|
321
418
|
private writeIdentity;
|
|
@@ -373,3 +470,4 @@ export declare class KnowledgeService {
|
|
|
373
470
|
last_verified: string;
|
|
374
471
|
}>;
|
|
375
472
|
}
|
|
473
|
+
export {};
|