@staix/agent-hub 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +70 -3
- package/README.md +2 -2
- package/docs/agent-notes/README.md +11 -0
- package/docs/agent-notes/adapters.md +10 -5
- package/docs/agent-notes/benchmarks.md +27 -0
- package/docs/agent-notes/budget.md +2 -1
- package/docs/agent-notes/bus.md +3 -3
- package/docs/agent-notes/daemon.md +20 -7
- package/docs/agent-notes/local-worker.md +5 -0
- package/docs/agent-notes/models.md +2 -0
- package/docs/agent-notes/tasks.md +20 -10
- package/docs/agent-notes/tests.md +13 -5
- package/docs/bench.md +1 -1
- package/docs/data-policy.md +2 -0
- package/docs/dogfood.md +41 -5
- package/docs/events.md +2 -1
- package/docs/memory.md +9 -3
- package/docs/operations.md +598 -63
- package/docs/profiles.md +13 -1
- package/docs/quickstart.md +18 -9
- package/docs/research.md +7 -4
- package/docs/sdlc.md +66 -10
- package/docs/security.md +397 -35
- package/docs/smoke.md +127 -0
- package/docs/specs/2026-09-19-agent-hub-design.md +1149 -19
- package/docs/verification/2026-10-27-0.19.0.md +5 -5
- package/docs/verification/2026-10-27-0.20.0.md +75 -0
- package/docs/verification/2026-10-28-claude-2.1.311-2.1.312-bash-permission.json +635 -0
- package/docs/verification/2026-10-28-claude-2.1.311-2.1.312-permission-transport.json +403 -0
- package/docs/verification/2026-10-28-claude-implementer-permission-triage.md +123 -0
- package/docs/verification/2026-10-28-jcode-0.99.1-usage.md +41 -0
- package/docs/verification/2026-10-28-realpath-name-audit.md +222 -0
- package/docs/verification/2026-10-28-routing-f0bb8ad0.md +1000 -0
- package/docs/verification/2026-10-29-0.21.0.md +61 -0
- package/docs/verification/2026-10-29-model-effort-capabilities.md +123 -0
- package/docs/verification/2026-10-29-modelrouting-b152ddb9.md +953 -0
- package/docs/verification/claude-implementer-permission-triage.ts +160 -0
- package/docs/verification/claude-permission-transport-native.py +71 -14
- package/docs/verification/profiles/YYYY-MM-DD-research.md +1 -1
- package/docs/verified.json +26 -18
- package/package.json +2 -2
- package/plugins/agent-hub/.claude-plugin/plugin.json +1 -1
- package/plugins/agent-hub/server.js +1116 -284
- package/src/adapters/acp.ts +87 -3
- package/src/adapters/channel-peer.ts +11 -5
- package/src/adapters/claude-channel.ts +13 -6
- package/src/adapters/claude-implementer.ts +61 -7
- package/src/adapters/codex-appserver.ts +88 -5
- package/src/adapters/conformance/enrollment.ts +4 -4
- package/src/adapters/conformance/factories.ts +2 -1
- package/src/adapters/conformance/fakes/jcode-agent.ts +6 -2
- package/src/adapters/conformance/kit.ts +10 -6
- package/src/adapters/definitions/builtin-metadata.ts +3 -114
- package/src/adapters/definitions/native-config-defaults.ts +1 -1
- package/src/adapters/definitions/native-config-policy.ts +3 -2
- package/src/adapters/definitions/native-context.ts +12 -0
- package/src/adapters/external-metadata.ts +48 -0
- package/src/adapters/external-modules.ts +63 -3
- package/src/adapters/jcode.ts +6 -2
- package/src/adapters/local-worker.ts +42 -25
- package/src/adapters/native-proof-ports.ts +10 -0
- package/src/adapters/pi.ts +7 -0
- package/src/adapters/registry-types.ts +4 -0
- package/src/adapters/reviewer-print.ts +1 -0
- package/src/broker/protocol.ts +203 -0
- package/src/broker/server.ts +314 -0
- package/src/broker/store.ts +548 -0
- package/src/cli/bench.ts +3 -2
- package/src/cli/broker.ts +58 -0
- package/src/cli/conductor-recovery.ts +142 -0
- package/src/cli/configured-acp-entry.ts +26 -6
- package/src/cli/console-state.ts +59 -17
- package/src/cli/console.ts +17 -2
- package/src/cli/facts-hook.ts +3 -1
- package/src/cli/help.ts +28 -16
- package/src/cli/identity.ts +23 -10
- package/src/cli/init.ts +34 -7
- package/src/cli/intake-output.ts +33 -0
- package/src/cli/main.ts +314 -71
- package/src/cli/output.ts +41 -1
- package/src/cli/recovery-package.ts +18 -1
- package/src/cli/role-holders.ts +35 -0
- package/src/cli/room.ts +304 -0
- package/src/cli/say-input.ts +16 -0
- package/src/cli/schedule-output.ts +49 -0
- package/src/cli/sdlc-output.ts +44 -6
- package/src/cli/smoke-pty-bridge.ts +33 -0
- package/src/cli/smoke-pty.ts +145 -0
- package/src/cli/smoke-run.ts +139 -0
- package/src/cli/smoke-target.ts +130 -0
- package/src/cli/source-output.ts +29 -0
- package/src/cli/task-output.ts +3 -2
- package/src/cli/upgrade-runtime.ts +98 -16
- package/src/cli/upgrade.ts +66 -7
- package/src/hub/ask.ts +76 -12
- package/src/hub/board.ts +132 -22
- package/src/hub/bus.ts +399 -56
- package/src/hub/child-process.ts +2 -0
- package/src/hub/ci-watch.ts +162 -0
- package/src/hub/claude-failure-stage.ts +85 -0
- package/src/hub/claude-launch-proof.ts +67 -2
- package/src/hub/claude-shell-wrapper.ts +11 -5
- package/src/hub/conductor.ts +17 -4
- package/src/hub/config-apply.ts +32 -5
- package/src/hub/config-reload.ts +7 -5
- package/src/hub/config-trust.ts +8 -1
- package/src/hub/context-window.ts +33 -3
- package/src/hub/control-client.ts +14 -2
- package/src/hub/corpus-config.ts +34 -0
- package/src/hub/corpus-evidence.ts +184 -0
- package/src/hub/corpus.ts +373 -0
- package/src/hub/cost-governance.ts +26 -0
- package/src/hub/daemon.ts +1228 -242
- package/src/hub/deliverable-files.ts +8 -2
- package/src/hub/delivery-journal.ts +123 -17
- package/src/hub/dogfood-profile.ts +32 -5
- package/src/hub/dogfood-read.ts +6 -2
- package/src/hub/envelope.ts +11 -0
- package/src/hub/eval-gate.ts +131 -0
- package/src/hub/events.ts +1 -1
- package/src/hub/external-instance.ts +69 -0
- package/src/hub/facts.ts +15 -5
- package/src/hub/folder-trigger.ts +223 -0
- package/src/hub/hub-tools.ts +2 -1
- package/src/hub/inference.ts +20 -5
- package/src/hub/instance-config.ts +69 -3
- package/src/hub/instance-execution.ts +6 -2
- package/src/hub/instances.ts +32 -4
- package/src/hub/intake.ts +125 -0
- package/src/hub/learning.ts +85 -0
- package/src/hub/lifecycle.ts +7 -3
- package/src/hub/limits.ts +12 -5
- package/src/hub/native-permission-request.ts +77 -0
- package/src/hub/operational-authority.ts +63 -5
- package/src/hub/operational-settings.ts +76 -0
- package/src/hub/operational-view.ts +1 -1
- package/src/hub/packs.ts +132 -0
- package/src/hub/path-policy.ts +7 -0
- package/src/hub/peer-doing.ts +130 -0
- package/src/hub/peers.ts +46 -0
- package/src/hub/queue-backpressure.ts +58 -0
- package/src/hub/read-tools.ts +2 -0
- package/src/hub/recovery-authority.ts +151 -0
- package/src/hub/recovery-policy.ts +220 -0
- package/src/hub/research.ts +24 -14
- package/src/hub/restart.ts +7 -0
- package/src/hub/routing.ts +54 -11
- package/src/hub/schedules.ts +266 -0
- package/src/hub/sdlc-merge.ts +38 -14
- package/src/hub/sdlc-metrics.ts +32 -10
- package/src/hub/sdlc-queue.ts +7 -1
- package/src/hub/sdlc-release.ts +1 -1
- package/src/hub/sdlc.ts +37 -7
- package/src/hub/security-baseline.ts +38 -0
- package/src/hub/settings.ts +7 -1
- package/src/hub/shipped-packs.ts +8 -0
- package/src/hub/snapshots.ts +72 -6
- package/src/hub/source-read.ts +160 -0
- package/src/hub/sources.ts +311 -0
- package/src/hub/task-shell-default.ts +3 -2
- package/src/hub/tasks.ts +1047 -115
- package/src/hub/workflow.ts +1 -1
- package/src/local/deny.ts +21 -0
- package/src/local/doc-read.ts +69 -0
- package/src/local/proxy.ts +4 -13
- package/src/local/sandbox.ts +3 -1
- package/src/local/tools.ts +37 -3
- package/src/local/web-fetch-config.ts +43 -0
- package/src/local/web-fetch.ts +183 -0
- package/src/memory/brief.ts +85 -13
- package/src/memory/capture.ts +1 -1
- package/src/rooms/application.ts +114 -0
- package/src/rooms/boundary.ts +103 -0
- package/src/rooms/commands.ts +294 -0
- package/src/rooms/crypto.ts +371 -0
- package/src/rooms/link.ts +584 -0
- package/src/rooms/policy.ts +82 -0
- package/src/rooms/remote-peer.ts +80 -0
- package/src/rooms/service.ts +426 -0
- package/src/rooms/status.ts +146 -0
- package/src/rooms/store.ts +1210 -0
- package/src/rooms/wire.ts +465 -0
- package/src/ui/index.html +91 -4
- package/templates/packs/example.pack.json +13 -0
|
@@ -0,0 +1,373 @@
|
|
|
1
|
+
import { closeSync, constants, fstatSync, lstatSync, openSync, readSync, readdirSync, type Dirent, type Stats } from "node:fs";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
4
|
+
import { isDenied } from "../local/deny.ts";
|
|
5
|
+
import { foldSegment } from "./permission-mode.ts";
|
|
6
|
+
import { matchesPolicy, matchesPolicyLexically, policyRealPath } from "./path-policy.ts";
|
|
7
|
+
import { realPath } from "./project.ts";
|
|
8
|
+
|
|
9
|
+
// agent-hub #510 (hub task #331): the read-only plain-notes index. Spec: "Knowledge corpus (#510), owner decision 2026-10-22".
|
|
10
|
+
// The hub reads plain notes folders. A folder that is, lies in or contains an Obsidian vault is refused: a vault is reached only through the Obsidian MCP tools.
|
|
11
|
+
|
|
12
|
+
export const MAX_NOTES = 5000;
|
|
13
|
+
export const MAX_DIRS = 4096;
|
|
14
|
+
export const MAX_ENTRIES = 100_000;
|
|
15
|
+
export const MAX_DEPTH = 24;
|
|
16
|
+
export const MAX_NOTE_BYTES = 256 * 1024;
|
|
17
|
+
export const MAX_HEADINGS = 256;
|
|
18
|
+
export const MAX_LINKS = 512;
|
|
19
|
+
export const MAX_TAGS = 64;
|
|
20
|
+
export const MAX_FRONTMATTER_BYTES = 8 * 1024;
|
|
21
|
+
/** A single evidence request may open at most this many freshly checked notes. */
|
|
22
|
+
export const MAX_CORPUS_READS = 32;
|
|
23
|
+
const NOTE_EXT = /\.(md|markdown)$/i;
|
|
24
|
+
/** The one source of the sentence that names where a vault is read, so the refusal and its tests cannot drift. */
|
|
25
|
+
export const VAULT_REASON = "is, lies in or contains an Obsidian vault (a .obsidian/ folder); the hub never reads a vault from disk. Reach it through the Obsidian MCP tools (mcp__obsidian__*) instead";
|
|
26
|
+
|
|
27
|
+
export interface CorpusContext {
|
|
28
|
+
cwd: string;
|
|
29
|
+
paths: readonly string[];
|
|
30
|
+
/** `pii.paths`: project-relative exact paths, subtrees or globs. A match is never read. */
|
|
31
|
+
piiPaths?: readonly string[];
|
|
32
|
+
/** `local.deny` substrings, on top of the built-in secrets denylist. */
|
|
33
|
+
deny?: readonly string[];
|
|
34
|
+
/** `signals.pii_patterns`: a note whose text matches (or whose pattern cannot be evaluated) is not indexed. */
|
|
35
|
+
piiPatterns?: readonly string[];
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export type CorpusRootCheck = { ok: true; real: string } | { ok: false; reason: string };
|
|
39
|
+
|
|
40
|
+
const errno = (error: unknown): string | undefined => (error as NodeJS.ErrnoException | undefined)?.code;
|
|
41
|
+
const clean = (text: string): string => text.replace(/[\x00-\x1f\x7f]/g, " ").replace(/\s+/g, " ").trim();
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Whether an entry named `.obsidian` exists directly in `dir`, in any folded spelling (the walk below a root folds names the same way, so a root and its ancestors are
|
|
45
|
+
* judged alike on a case-sensitive disk, where `.OBSIDIAN` is a different name than `.obsidian`). A literal `lstat` answers first (it also covers a case-insensitive disk
|
|
46
|
+
* without a listing); otherwise the folder is listed, bounded. A thrown error for anything but absence: unreadable evidence is never "no vault".
|
|
47
|
+
*/
|
|
48
|
+
function hasObsidian(dir: string): boolean {
|
|
49
|
+
try { lstatSync(join(dir, ".obsidian")); return true; }
|
|
50
|
+
catch (error) { if (errno(error) !== "ENOENT" && errno(error) !== "ENOTDIR") throw error; }
|
|
51
|
+
let names: string[];
|
|
52
|
+
try { names = readdirSync(dir); }
|
|
53
|
+
catch (error) { if (errno(error) === "ENOTDIR") return false; throw error; }
|
|
54
|
+
if (names.length > MAX_ENTRIES) throw Object.assign(new Error("too many entries"), { code: "E2BIG" });
|
|
55
|
+
return names.some(name => foldSegment(name) === ".obsidian");
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Whether a configured corpus root may be read. The check runs on the `realPath` of the root (a symlink into a vault is caught), looks for `.obsidian/` at the
|
|
60
|
+
* root and in every ancestor of the real path (a root inside a vault), and refuses on evidence it cannot read. `pii.paths` and the deny rules are checked here too:
|
|
61
|
+
* a root under either is never a corpus, and the walk refuses a root that holds a protected folder, because that folder is never listed and cannot be shown free of a vault. The walk below the root (refusing any `.obsidian` it meets) is `CorpusIndex.refresh`.
|
|
62
|
+
*/
|
|
63
|
+
export function checkCorpusRoot(configured: string, ctx: CorpusContext): CorpusRootCheck {
|
|
64
|
+
let real: string;
|
|
65
|
+
try {
|
|
66
|
+
const cwd = realPath(ctx.cwd);
|
|
67
|
+
real = policyRealPath(resolve(cwd, configured));
|
|
68
|
+
if (!lstatSync(real).isDirectory()) return { ok: false, reason: `${configured} is not a folder` };
|
|
69
|
+
real = realPath(real);
|
|
70
|
+
} catch (error) { return { ok: false, reason: `${configured} cannot be resolved (${errno(error) ?? "unreadable"}); a path that cannot be checked is never read` }; }
|
|
71
|
+
try {
|
|
72
|
+
for (let dir = real; ; dir = dirname(dir)) {
|
|
73
|
+
if (hasObsidian(dir)) return { ok: false, reason: `${configured} ${VAULT_REASON}${dir === real ? "" : ` (the vault folder is ${dir})`}` };
|
|
74
|
+
if (dirname(dir) === dir) break;
|
|
75
|
+
}
|
|
76
|
+
} catch (error) { return { ok: false, reason: `${configured}: a folder above it cannot be read (${errno(error) ?? "unreadable"}), so it cannot be shown to be free of a vault` }; }
|
|
77
|
+
if (isDenied(real, [...(ctx.deny ?? [])])) return { ok: false, reason: `${configured} is on the secrets denylist or local.deny` };
|
|
78
|
+
if (ctx.piiPaths?.length && matchesPolicy(ctx.cwd, real, [...ctx.piiPaths])) return { ok: false, reason: `${configured} is, or lies in, a path under pii.paths` };
|
|
79
|
+
return { ok: true, real };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export interface NoteLink { target: string; heading?: string; embed: boolean }
|
|
83
|
+
export interface ParsedNote {
|
|
84
|
+
title?: string;
|
|
85
|
+
frontmatter: Record<string, string | string[]>;
|
|
86
|
+
tags: string[];
|
|
87
|
+
aliases: string[];
|
|
88
|
+
headings: { level: number; text: string }[];
|
|
89
|
+
links: NoteLink[];
|
|
90
|
+
/** A cap cut headings, links or frontmatter short. */
|
|
91
|
+
truncated: boolean;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const scalar = (raw: string): string => {
|
|
95
|
+
const text = raw.trim();
|
|
96
|
+
if ((text.startsWith('"') && text.endsWith('"') && text.length >= 2) || (text.startsWith("'") && text.endsWith("'") && text.length >= 2)) return clean(text.slice(1, -1));
|
|
97
|
+
return clean(text);
|
|
98
|
+
};
|
|
99
|
+
const listOf = (raw: string): string[] => raw.trim().replace(/^\[/, "").replace(/\]$/, "").split(",").map(scalar).filter(Boolean);
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* A bounded frontmatter reader for the shapes notes use: `key: value`, `key: [a, b]` and a block list under `key:`. It is not a YAML parser (zero runtime
|
|
103
|
+
* dependencies): nested maps, anchors and multi-line scalars are kept as their first line or ignored, never executed or evaluated.
|
|
104
|
+
* ponytail: ceiling is flat keys; upgrade path is a vetted YAML subset parser if notes need nested metadata.
|
|
105
|
+
*/
|
|
106
|
+
function frontmatterOf(text: string): { values: Record<string, string | string[]>; body: string; truncated: boolean } {
|
|
107
|
+
const lines = text.split(/\r?\n/);
|
|
108
|
+
if (lines[0]?.trimEnd() !== "---") return { values: {}, body: text, truncated: false };
|
|
109
|
+
let end = -1, bytes = 0;
|
|
110
|
+
for (let i = 1; i < lines.length; i++) {
|
|
111
|
+
bytes += lines[i]!.length + 1;
|
|
112
|
+
if (bytes > MAX_FRONTMATTER_BYTES) return { values: {}, body: lines.slice(i).join("\n"), truncated: true };
|
|
113
|
+
if (/^(---|\.\.\.)\s*$/.test(lines[i]!)) { end = i; break; }
|
|
114
|
+
}
|
|
115
|
+
if (end < 0) return { values: {}, body: text, truncated: false };
|
|
116
|
+
const values: Record<string, string | string[]> = {};
|
|
117
|
+
let key: string | undefined;
|
|
118
|
+
for (const line of lines.slice(1, end)) {
|
|
119
|
+
const item = /^\s+-\s+(.*)$/.exec(line);
|
|
120
|
+
if (item && key) { const current = values[key]; values[key] = [...(Array.isArray(current) ? current : current ? [current] : []), scalar(item[1]!)].filter(Boolean).slice(0, MAX_TAGS); continue; }
|
|
121
|
+
const pair = /^([A-Za-z0-9_][A-Za-z0-9_ -]{0,63}):\s*(.*)$/.exec(line);
|
|
122
|
+
if (!pair || /^\s/.test(line)) { if (!/^\s/.test(line)) key = undefined; continue; }
|
|
123
|
+
key = pair[1]!.trim().toLowerCase();
|
|
124
|
+
if (Object.hasOwn(values, key) || ["__proto__", "constructor", "prototype"].includes(key)) { key = undefined; continue; }
|
|
125
|
+
const value = pair[2]!;
|
|
126
|
+
if (value.startsWith("[")) values[key] = listOf(value).slice(0, MAX_TAGS);
|
|
127
|
+
else if (value.trim() === "" ) values[key] = [];
|
|
128
|
+
else values[key] = scalar(value).slice(0, 500);
|
|
129
|
+
}
|
|
130
|
+
return { values, body: lines.slice(end + 1).join("\n"), truncated: false };
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** The body without fenced and inline code, so a `[[link]]` or a `# heading` inside code is not read as one. */
|
|
134
|
+
function prose(body: string): { text: string; headings: { level: number; text: string }[]; truncated: boolean } {
|
|
135
|
+
const kept: string[] = [], headings: { level: number; text: string }[] = [];
|
|
136
|
+
let fence: string | undefined, truncated = false;
|
|
137
|
+
for (const line of body.split(/\r?\n/)) {
|
|
138
|
+
const marker = /^\s{0,3}(`{3,}|~{3,})/.exec(line);
|
|
139
|
+
if (marker) { const mark = marker[1]![0]!; if (!fence) fence = mark; else if (fence === mark) fence = undefined; continue; }
|
|
140
|
+
if (fence) continue;
|
|
141
|
+
const heading = /^ {0,3}(#{1,6})\s+(.+?)(?:\s+#+)?\s*$/.exec(line); // a closing run of # is a closer only after whitespace: `# C#` is "C#"
|
|
142
|
+
if (heading) { if (headings.length < MAX_HEADINGS) headings.push({ level: heading[1]!.length, text: clean(heading[2]!).slice(0, 200) }); else truncated = true; }
|
|
143
|
+
kept.push(line.replace(/`[^`\n]*`/g, " "));
|
|
144
|
+
}
|
|
145
|
+
return { text: kept.join("\n"), headings: headings.filter(item => item.text), truncated };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Frontmatter, headings and wikilinks of one note. Never evaluates anything and never throws on malformed input. */
|
|
149
|
+
export function parseNote(text: string): ParsedNote {
|
|
150
|
+
const { values, body, truncated: frontTruncated } = frontmatterOf(text);
|
|
151
|
+
const { text: scan, headings, truncated: headTruncated } = prose(body);
|
|
152
|
+
const links: NoteLink[] = [];
|
|
153
|
+
let linkTruncated = false;
|
|
154
|
+
for (const match of scan.matchAll(/(!?)\[\[([^\[\]\n|#]{1,200})(?:#([^\[\]\n|]{1,200}))?(?:\|[^\[\]\n]{0,200})?\]\]/g)) {
|
|
155
|
+
const target = clean(match[2]!);
|
|
156
|
+
if (!target) continue;
|
|
157
|
+
if (links.length >= MAX_LINKS) { linkTruncated = true; break; }
|
|
158
|
+
links.push({ target, ...(match[3] ? { heading: clean(match[3]) } : {}), embed: match[1] === "!" });
|
|
159
|
+
}
|
|
160
|
+
const list = (key: string): string[] => { const value = values[key]; return (Array.isArray(value) ? value : value ? value.split(",").map(scalar) : []).map(item => item.replace(/^#/, "")).filter(Boolean).slice(0, MAX_TAGS); };
|
|
161
|
+
const title = typeof values.title === "string" && values.title ? values.title : headings.find(item => item.level === 1)?.text;
|
|
162
|
+
return { ...(title ? { title } : {}), frontmatter: values, tags: [...new Set([...list("tags"), ...list("tag")])], aliases: [...new Set([...list("aliases"), ...list("alias")])], headings, links, truncated: frontTruncated || headTruncated || linkTruncated };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export interface NoteRecord extends ParsedNote {
|
|
166
|
+
/** `note:<root label>/<path>`; a heading is cited as `<id>#<heading>`. */
|
|
167
|
+
id: string;
|
|
168
|
+
root: string;
|
|
169
|
+
path: string;
|
|
170
|
+
bytes: number;
|
|
171
|
+
mtimeMs: number;
|
|
172
|
+
sha256: string;
|
|
173
|
+
}
|
|
174
|
+
export interface CorpusRootReport {
|
|
175
|
+
configured: string;
|
|
176
|
+
label?: string;
|
|
177
|
+
real?: string;
|
|
178
|
+
refused?: string;
|
|
179
|
+
notes: number;
|
|
180
|
+
/** Counts of what was left out, by reason: never silently dropped. */
|
|
181
|
+
skipped: { protected: number; links: number; large: number; unreadable: number; pii: number; overCap: number };
|
|
182
|
+
}
|
|
183
|
+
export interface CorpusRefresh { roots: CorpusRootReport[]; notes: number; parsed: number; reused: number; removed: number }
|
|
184
|
+
export interface CorpusRead { record: NoteRecord; text: string; path: string }
|
|
185
|
+
|
|
186
|
+
const sizeKey = (st: { size: number; mtimeMs: number; ctimeMs: number; ino: number; dev: number }) => `${st.dev}:${st.ino}:${st.size}:${st.mtimeMs}:${st.ctimeMs}`;
|
|
187
|
+
|
|
188
|
+
function readNote(path: string, expected: { dev: number; ino: number }): Buffer {
|
|
189
|
+
const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
190
|
+
try {
|
|
191
|
+
const st = fstatSync(fd);
|
|
192
|
+
if (!st.isFile() || st.nlink !== 1 || st.dev !== expected.dev || st.ino !== expected.ino || st.size > MAX_NOTE_BYTES) throw new Error("not a bounded regular note");
|
|
193
|
+
const bytes = Buffer.alloc(st.size); let got = 0;
|
|
194
|
+
while (got < bytes.length) { const n = readSync(fd, bytes, got, bytes.length - got, got); if (!n) break; got += n; }
|
|
195
|
+
if (got !== st.size) throw new Error("note changed while read");
|
|
196
|
+
return bytes;
|
|
197
|
+
} finally { closeSync(fd); }
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* The read-only index of the configured plain-notes folders. `refresh` walks each root (bounded in folders, entries and notes), skips what `pii.paths`, the deny
|
|
202
|
+
* rules and symbolic links protect, refuses a root that holds an Obsidian vault anywhere, and re-reads only the notes whose size, times or identity changed.
|
|
203
|
+
* It stores structure (title, tags, headings, links, a hash), never the note's text, and it writes nothing. The index is in memory: a restart rebuilds it.
|
|
204
|
+
* ponytail: the cache is per process; persist it in hub.db if a rebuild becomes slow for big folders.
|
|
205
|
+
*/
|
|
206
|
+
export class CorpusIndex {
|
|
207
|
+
private cache = new Map<string, { key: string; record: NoteRecord }>();
|
|
208
|
+
private byId = new Map<string, NoteRecord>();
|
|
209
|
+
constructor(private readonly ctx: CorpusContext) {}
|
|
210
|
+
|
|
211
|
+
notes(): NoteRecord[] { return [...this.byId.values()].sort((a, b) => a.id.localeCompare(b.id)); }
|
|
212
|
+
get(id: string): NoteRecord | undefined { return this.byId.get(id); }
|
|
213
|
+
|
|
214
|
+
/** Read one indexed note from disk after rebuilding the bounded index and rechecking current policy. Text is never cached. */
|
|
215
|
+
read(id: string): CorpusRead | undefined { return this.readMany([id])[0]; }
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* Freshly read a bounded batch. One full refresh revalidates every configured root and its vault-safe walk; each selected file then gets current lexical policy,
|
|
219
|
+
* path, inode, size, content-hash and PII checks immediately before any text is returned. A cache hit is metadata only and never supplies old text.
|
|
220
|
+
*/
|
|
221
|
+
readMany(ids: readonly string[]): CorpusRead[] {
|
|
222
|
+
if (ids.length === 0) return [];
|
|
223
|
+
this.refresh();
|
|
224
|
+
const selected = [...new Set(ids.slice(0, MAX_CORPUS_READS))];
|
|
225
|
+
const roots = new Map(this.refreshReports().filter((item): item is CorpusRootReport & { label: string; real: string } => !!item.label && !!item.real).map(item => [item.label, item]));
|
|
226
|
+
const checkedRoots = new Map<string, string>();
|
|
227
|
+
for (const root of new Set(selected.map(id => this.byId.get(id)?.root).filter((label): label is string => !!label))) {
|
|
228
|
+
const report = roots.get(root);
|
|
229
|
+
if (!report) continue;
|
|
230
|
+
const checked = checkCorpusRoot(report.configured, this.ctx);
|
|
231
|
+
if (checked.ok && checked.real === report.real) checkedRoots.set(root, checked.real);
|
|
232
|
+
}
|
|
233
|
+
const output: CorpusRead[] = [];
|
|
234
|
+
for (const id of selected) {
|
|
235
|
+
const record = this.byId.get(id);
|
|
236
|
+
if (!record) continue;
|
|
237
|
+
const root = roots.get(record.root);
|
|
238
|
+
const checkedReal = checkedRoots.get(record.root);
|
|
239
|
+
if (!root || !checkedReal) continue;
|
|
240
|
+
const abs = resolve(checkedReal, record.path);
|
|
241
|
+
const rel = relative(checkedReal, abs);
|
|
242
|
+
if (!rel || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel) || !NOTE_EXT.test(abs)) continue;
|
|
243
|
+
const pii = [...(this.ctx.piiPaths ?? [])], deny = [...(this.ctx.deny ?? [])];
|
|
244
|
+
if (isDenied(abs, deny) || (pii.length > 0 && (matchesPolicyLexically(this.ctx.cwd, abs, pii) || matchesPolicy(this.ctx.cwd, abs, pii)))) continue;
|
|
245
|
+
try {
|
|
246
|
+
let parent = checkedReal;
|
|
247
|
+
const segments = record.path.split("/");
|
|
248
|
+
if (!segments.length || segments.some(part => !part || part === "." || part === "..")) continue;
|
|
249
|
+
for (const segment of segments.slice(0, -1)) {
|
|
250
|
+
parent = join(parent, segment);
|
|
251
|
+
const parentStat = lstatSync(parent);
|
|
252
|
+
if (parentStat.isSymbolicLink() || !parentStat.isDirectory()) { parent = ""; break; }
|
|
253
|
+
}
|
|
254
|
+
if (!parent) continue;
|
|
255
|
+
const st = lstatSync(abs);
|
|
256
|
+
if (!st.isFile() || st.nlink !== 1 || st.size > MAX_NOTE_BYTES || st.size !== record.bytes) continue;
|
|
257
|
+
// Each root-relative parent is checked with lstat; an in-root symlink is not an approved notes path either.
|
|
258
|
+
const bytes = readNote(abs, st);
|
|
259
|
+
if (bytes.includes(0)) continue;
|
|
260
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
261
|
+
if (createHash("sha256").update(bytes).digest("hex") !== record.sha256 || this.piiFlagged(text)) continue;
|
|
262
|
+
output.push({ record, text, path: abs });
|
|
263
|
+
} catch { /* A disappeared, replaced, unreadable or invalid note is not evidence. */ }
|
|
264
|
+
}
|
|
265
|
+
return output;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
private refreshReports(): CorpusRootReport[] { return this.lastReports; }
|
|
269
|
+
|
|
270
|
+
private lastReports: CorpusRootReport[] = [];
|
|
271
|
+
|
|
272
|
+
/** The notes a wikilink target names, in the same root as `fromId`: by path (with or without the extension) or by file name, case and spelling folded. */
|
|
273
|
+
resolveLink(fromId: string, target: string): NoteRecord[] {
|
|
274
|
+
const from = this.byId.get(fromId);
|
|
275
|
+
if (!from) return [];
|
|
276
|
+
const want = foldSegment(target.replace(/\\/g, "/").replace(NOTE_EXT, "").replace(/^\/+/, ""));
|
|
277
|
+
if (!want) return [];
|
|
278
|
+
return this.notes().filter(note => {
|
|
279
|
+
if (note.root !== from.root) return false;
|
|
280
|
+
const path = foldSegment(note.path.replace(NOTE_EXT, ""));
|
|
281
|
+
return want.includes("/") ? path === want || path.endsWith(`/${want}`) : path === want || path.endsWith(`/${want}`);
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
refresh(): CorpusRefresh {
|
|
286
|
+
const seen = new Set<string>(), labels = new Set<string>(), reports: CorpusRootReport[] = [];
|
|
287
|
+
let parsed = 0, reused = 0;
|
|
288
|
+
const nextById = new Map<string, NoteRecord>();
|
|
289
|
+
for (const configured of this.ctx.paths) {
|
|
290
|
+
const report: CorpusRootReport = { configured, notes: 0, skipped: { protected: 0, links: 0, large: 0, unreadable: 0, pii: 0, overCap: 0 } };
|
|
291
|
+
reports.push(report);
|
|
292
|
+
const checked = checkCorpusRoot(configured, this.ctx);
|
|
293
|
+
if (!checked.ok) { report.refused = checked.reason; continue; }
|
|
294
|
+
report.real = checked.real;
|
|
295
|
+
let label = clean(basename(checked.real)).replace(/[^\w.-]+/g, "-").replace(/^-+|-+$/g, "") || "notes", n = 1;
|
|
296
|
+
while (labels.has(label)) label = `${clean(basename(checked.real)).replace(/[^\w.-]+/g, "-") || "notes"}-${++n}`;
|
|
297
|
+
labels.add(label); report.label = label;
|
|
298
|
+
const found = this.walk(checked.real, report);
|
|
299
|
+
if (typeof found === "string") { report.refused = `${configured} ${found}`; continue; }
|
|
300
|
+
for (const file of found) {
|
|
301
|
+
const abs = join(checked.real, file.path), key = sizeKey(file.st), id = `note:${label}/${file.path}`;
|
|
302
|
+
const cached = this.cache.get(abs);
|
|
303
|
+
let record: NoteRecord;
|
|
304
|
+
if (cached && cached.key === key && cached.record.id === id) { record = cached.record; reused++; }
|
|
305
|
+
else {
|
|
306
|
+
try {
|
|
307
|
+
const bytes = readNote(abs, file.st), text = new TextDecoder("utf-8", { fatal: false }).decode(bytes);
|
|
308
|
+
if (bytes.includes(0) || text.includes("\uFFFD")) { report.skipped.unreadable++; continue; }
|
|
309
|
+
if (this.piiFlagged(text)) { report.skipped.pii++; continue; }
|
|
310
|
+
record = { ...parseNote(text), id, root: label, path: file.path, bytes: bytes.length, mtimeMs: file.st.mtimeMs, sha256: createHash("sha256").update(bytes).digest("hex") };
|
|
311
|
+
parsed++;
|
|
312
|
+
} catch { report.skipped.unreadable++; continue; }
|
|
313
|
+
}
|
|
314
|
+
this.cache.set(abs, { key, record }); seen.add(abs);
|
|
315
|
+
nextById.set(record.id, record); report.notes++;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
let removed = 0;
|
|
319
|
+
for (const path of [...this.cache.keys()]) if (!seen.has(path)) { this.cache.delete(path); removed++; }
|
|
320
|
+
this.byId = nextById;
|
|
321
|
+
this.lastReports = reports;
|
|
322
|
+
return { roots: reports, notes: nextById.size, parsed, reused, removed };
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
private piiFlagged(text: string): boolean {
|
|
326
|
+
if (!this.ctx.piiPatterns?.length) return false;
|
|
327
|
+
try { return this.ctx.piiPatterns.some(pattern => new RegExp(pattern).test(text)); } catch { return true; } // a pattern that cannot be evaluated is not a clear verdict
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/** The notes under `root`, or a refusal string (a vault below it, or evidence the walk could not read). Directories are visited in name order so a cap cuts deterministically. */
|
|
331
|
+
private walk(root: string, report: CorpusRootReport): { path: string; st: Stats }[] | string {
|
|
332
|
+
const out: { path: string; st: Stats }[] = [];
|
|
333
|
+
let dirs = 0, entries = 0;
|
|
334
|
+
const pii = [...(this.ctx.piiPaths ?? [])], deny = [...(this.ctx.deny ?? [])];
|
|
335
|
+
const stack: { dir: string; depth: number }[] = [{ dir: root, depth: 0 }];
|
|
336
|
+
while (stack.length) {
|
|
337
|
+
const { dir, depth } = stack.pop()!;
|
|
338
|
+
if (++dirs > MAX_DIRS || depth > MAX_DEPTH) return `has more than ${MAX_DIRS} folders or nests deeper than ${MAX_DEPTH}, so it cannot be shown to be free of a vault; point corpus.paths at a smaller folder`;
|
|
339
|
+
let list: Dirent[];
|
|
340
|
+
try { list = readdirSync(dir, { withFileTypes: true }); }
|
|
341
|
+
catch (error) { return `has a folder that cannot be read (${errno(error) ?? "unreadable"}), so it cannot be shown to be free of a vault`; }
|
|
342
|
+
list.sort((a, b) => a.name.localeCompare(b.name));
|
|
343
|
+
const subdirs: string[] = [];
|
|
344
|
+
for (const entry of list) {
|
|
345
|
+
if (++entries > MAX_ENTRIES) return `has more than ${MAX_ENTRIES} entries, so it cannot be shown to be free of a vault; point corpus.paths at a smaller folder`;
|
|
346
|
+
if (foldSegment(entry.name) === ".obsidian") return `${VAULT_REASON} (found ${join(dir, entry.name)})`;
|
|
347
|
+
const abs = join(dir, entry.name);
|
|
348
|
+
if (entry.isSymbolicLink()) { report.skipped.links++; continue; }
|
|
349
|
+
const rel = relative(root, abs).split(sep).join("/");
|
|
350
|
+
const protectedPath = () => isDenied(abs, deny) || (pii.length > 0 && (matchesPolicyLexically(this.ctx.cwd, abs, pii) || matchesPolicy(this.ctx.cwd, abs, pii)));
|
|
351
|
+
if (entry.isDirectory()) {
|
|
352
|
+
// A protected folder is never listed or read, so a vault anywhere below it cannot be ruled out: the root cannot be shown free of a vault, and it is refused
|
|
353
|
+
// (the owner decision: evidence that cannot be checked refuses). Point corpus.paths at a folder that holds no protected folder. A protected FILE is only skipped.
|
|
354
|
+
if (protectedPath()) return `holds the protected folder ${abs} (pii.paths, the deny rules or local.deny), which is never listed, so the root cannot be shown to be free of an Obsidian vault; point corpus.paths at a folder with no protected folder in it`;
|
|
355
|
+
subdirs.push(abs);
|
|
356
|
+
} else if (entry.isFile() && NOTE_EXT.test(entry.name)) {
|
|
357
|
+
if (protectedPath()) { report.skipped.protected++; continue; }
|
|
358
|
+
if (out.length >= MAX_NOTES) { report.skipped.overCap++; continue; }
|
|
359
|
+
let st: Stats;
|
|
360
|
+
try { st = lstatSync(abs); } catch { report.skipped.unreadable++; continue; }
|
|
361
|
+
if (!st.isFile() || st.nlink !== 1) { report.skipped.links++; continue; }
|
|
362
|
+
if (st.size > MAX_NOTE_BYTES) { report.skipped.large++; continue; }
|
|
363
|
+
out.push({ path: rel, st });
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
for (const sub of subdirs.reverse()) stack.push({ dir: sub, depth: depth + 1 });
|
|
367
|
+
}
|
|
368
|
+
return out;
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/** The paths, spelled for a person: configured text as written. Exported for the status line and tests. */
|
|
373
|
+
export const corpusRootLabel = (configured: string): string => isAbsolute(configured) ? configured : `./${configured}`;
|
|
@@ -23,6 +23,32 @@ export function cleanCostPlan(input: unknown): CostPlan {
|
|
|
23
23
|
}
|
|
24
24
|
return result;
|
|
25
25
|
}
|
|
26
|
+
/**
|
|
27
|
+
* #481 S7: what the guarded merge records about a merged PR, from the PR read once at merge. Every field is `null` when the read could not establish it, never 0.
|
|
28
|
+
* CI figures need the complete read (runs linked by every head of the PR); a rerun is an attempt after the first of one run id, i.e. the same head run again.
|
|
29
|
+
*/
|
|
30
|
+
export interface MergeCost { mergedAt: number | null; ciRuns: number | null; ciReruns: number | null; reviews: number | null }
|
|
31
|
+
export function ciRunCounts(runs: readonly { id: number; attempt: number }[] | undefined): { runs: number; reruns: number } | null {
|
|
32
|
+
if (!runs || runs.length > 100) return null;
|
|
33
|
+
const attempts = new Map<number, number>();
|
|
34
|
+
for (const run of runs) {
|
|
35
|
+
if (!Number.isSafeInteger(run?.id) || run.id < 1 || !Number.isSafeInteger(run.attempt) || run.attempt < 1 || run.attempt > 10_000) return null;
|
|
36
|
+
attempts.set(run.id, Math.max(attempts.get(run.id) ?? 0, run.attempt));
|
|
37
|
+
}
|
|
38
|
+
return { runs: attempts.size, reruns: [...attempts.values()].reduce((sum, n) => sum + n - 1, 0) };
|
|
39
|
+
}
|
|
40
|
+
export function mergeCostFrom(pr: CostPr | undefined): MergeCost | undefined {
|
|
41
|
+
if (!pr || pr.state !== "MERGED") return undefined;
|
|
42
|
+
const counts = pr.complete ? ciRunCounts(pr.ciRuns) : null;
|
|
43
|
+
return { mergedAt: Number.isSafeInteger(pr.mergedAt) && pr.mergedAt! >= 0 ? pr.mergedAt! : null, ciRuns: counts?.runs ?? null, ciReruns: counts?.reruns ?? null,
|
|
44
|
+
reviews: Number.isSafeInteger(pr.reviews) && pr.reviews! >= 0 ? pr.reviews! : null };
|
|
45
|
+
}
|
|
46
|
+
/** Bounded, finite or absent: a stored record is data, never trusted shape. */
|
|
47
|
+
export function cleanMergeCost(value: unknown): MergeCost | undefined {
|
|
48
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
49
|
+
const raw = value as Record<string, unknown>, field = (key: string) => Number.isSafeInteger(raw[key]) && (raw[key] as number) >= 0 ? raw[key] as number : null;
|
|
50
|
+
return { mergedAt: field("mergedAt"), ciRuns: field("ciRuns"), ciReruns: field("ciReruns"), reviews: field("reviews") };
|
|
51
|
+
}
|
|
26
52
|
const validRepo = (repo: string) => /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo) && !repo.startsWith("-");
|
|
27
53
|
const key = (repo: string, pr: number) => `${repo.toLowerCase()}#${pr}`;
|
|
28
54
|
/** One round is a board review handoff, not one reviewer in a multi-reviewer round. CI attempts share a run id. */
|