@gcunharodrigues/wrxn 0.6.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1190 @@
1
+ #!/usr/bin/env node
2
+ 'use strict';
3
+
4
+ // WRXN harvest adapter — the install-local, REPORT-ONLY curation-debt detector (harvest-02 / H2).
5
+ // Sibling to wiki.cjs / dream.cjs / sync.cjs. Self-contained: this ships INTO an install and MUST NOT
6
+ // import the kernel lib or recon — node stdlib ONLY (fs / http / path), so it is install-portable.
7
+ //
8
+ // What it does: `harvest.cjs check [--root]` scans the 4 knowledge tiers and writes a durable structured
9
+ // report under `.wrxn/harvest/<ts>.jsonl` — one JSON record per finding, classified:
10
+ // · near_dup clusters of pages over a MEASURED semantic-similarity threshold, via recon's
11
+ // hybrid similarity over the warm serve door (the recall-surface.cjs contract:
12
+ // discoverEndpoint → POST recon_find → read parsed.hits[].semanticScore). For each
13
+ // prose page we query the door with its body; other harvest-tier pages whose dense
14
+ // cosine clears the threshold are near-dup neighbours. Pairwise edges are collapsed
15
+ // into connected-component CLUSTERS (so an A↔B match is reported once, not twice).
16
+ // · decay_candidate an orphaned page (its `derived_from:` source FILE is gone) that is NOT yet annotated.
17
+ // A page already carrying `stale:` or `superseded_by:` is RESOLVED (the curated end
18
+ // state) and excluded, so a fully-curated tree reads clean. A LOCAL scan — no door.
19
+ // · malformed bad frontmatter — the existing wiki-lint signal (kernel-11), over the 4 tiers.
20
+ //
21
+ // This is LAYER 1 of harvest: detection only. The report is the SOLE input to the destructive curation
22
+ // layer (H3 merge / H4 decay) and the debt-gate (H5). It is strictly REPORT-ONLY: `check` writes ONLY
23
+ // under `.wrxn/harvest/`; it NEVER edits, deletes, or annotates a knowledge page, and re-running writes a
24
+ // FRESH timestamped report (it never mutates a prior one). FAIL-SOFT: a cold/unreachable door degrades
25
+ // near-dup to status "unavailable" in the report while the local malformed + orphaned/superseded scans
26
+ // still run — `check` never throws, never blocks (exit 0).
27
+ //
28
+ // Scope = the 4 tiers concepts/decisions/gotchas/_rules ONLY. The retired `sessions` tier is NEVER
29
+ // scanned (it is absent from HARVEST_TIERS), and a near-dup neighbour OUTSIDE these tiers (a sessions
30
+ // page, a .scratch draft, code) is never clustered — curation acts only on the curated knowledge set.
31
+ //
32
+ // Flag: --root <dir> (override the install-root walk-up; mainly for tests).
33
+
34
+ const fs = require('fs');
35
+ const http = require('http');
36
+ const path = require('path');
37
+ const crypto = require('crypto');
38
+ const { execFileSync } = require('child_process');
39
+
40
+ // The harvest curation scope (8-decision grill: scope = the 4 knowledge tiers). NOTE this DIFFERS from
41
+ // wiki-lint's TIERS: harvest scans `_rules` (the dream-written tier) and never scans the retired
42
+ // `sessions` tier — the handoff baton + dream consolidation are the close-out now (harvest-01).
43
+ const HARVEST_TIERS = ['concepts', 'decisions', 'gotchas', '_rules'];
44
+ const REQUIRED_KEYS = ['name', 'description', 'tier']; // wiki-lint's malformed-frontmatter contract.
45
+
46
+ const HARVEST_DIR = ['.wrxn', 'harvest'];
47
+ // merge (harvest-03) staging trail — mirrors sync's .wrxn/sync/. NON-.md so recon's prose ingestion (which
48
+ // walks all of .wrxn and reads *.md) never recalls a staged-but-unconfirmed merge. Coexists with check's
49
+ // timestamped <ts>.jsonl reports in the same dir (distinct fixed names, no collision).
50
+ const STAGED_FILE = 'staged.jsonl'; // the proposed-but-unconfirmed merges (survivor body + absorbed, by-reference).
51
+ const AUDIT_FILE = 'audit.jsonl'; // append-only outcome log (stage + commit + decay events).
52
+ const DECAY_STAGED_FILE = 'decay-staged.jsonl'; // harvest-04: proposed-but-unconfirmed decay annotations (by-reference). Distinct fixed name from merge's staged.jsonl; both non-.md so recon never recalls a staged-but-unconfirmed op.
53
+ const BODY_MAX = 32000; // survivor body cap (chars) — a durable merged page, not a dump (dream/sync parity).
54
+ const WIKI_REL = ['.wrxn', 'wiki']; // all merge targets confine under <root>/.wrxn/wiki/<knowledge-tier>/.
55
+ const WIKI_PREFIX = '.wrxn/wiki/'; // stripped to form the reinforce.json wiki-rel join key (recall-surface parity).
56
+ const REINFORCE_REL = path.join('.wrxn', 'reinforce.json'); // the coalesced access-recency sidecar (harvest-08 / D2) — STATE.
57
+ // The reinforced-exclusion window (AC4). 30 days is the project's established staleness boundary (the
58
+ // ai-memory briefing buckets activity at 7d/30d and treats >30d as stale): a page surfaced in Recall
59
+ // within the last month is demonstrably LIVE knowledge and must never be flagged stale/superseded, while a
60
+ // page un-surfaced for over a month is fair game for a decay annotation. Day-granular (the sidecar's grain).
61
+ const REINFORCE_WINDOW_DAYS = 30;
62
+ const VALUE_MAX = 256; // cap on a decay annotation VALUE (a path) — a frontmatter write channel, not a body.
63
+ const ENDPOINT_REL = path.join('.recon-wrxn', 'serve-endpoint.json');
64
+ const FIND_PATH = '/api/tools/recon_find'; // the recon serve door recall-surface/brain query.
65
+ const PROSE_TYPES = new Set(['Page', 'Section']); // prose scope — code symbols are never near-dup targets.
66
+
67
+ // ── the MEASURED near-dup threshold ──────────────────────────────────────────────
68
+ // NEAR_DUP_THRESHOLD gates the door's dense cosine (`semanticScore`), NOT the fused RRF `score` (RRF is
69
+ // rank-based, not a relevance magnitude — ADR 0002 / recall-surface). BASIS (measured on the live
70
+ // WRXN-OS prose corpus via the recon HTTP door, 384-d hybrid embedder, 2026-06-17):
71
+ // · a page queried with its own body scores ~0.85–0.99 against itself (≈1.0 ceiling for identical text);
72
+ // · merely-RELATED but distinct pages top out at ~0.66–0.70 (e.g. a PRD vs a sibling design doc);
73
+ // · the repo's established RELEVANCE floor is 0.40 (recall-surface SEMANTIC_FLOOR, validated on real prose).
74
+ // 0.85 sits in the clear gap above the ~0.70 "related" ceiling (a ~0.15 margin → merely-related pages do
75
+ // NOT trip near-dup) and below the ~1.0 identical ceiling (so genuine heavy-overlap duplicates DO). It is
76
+ // well above the 0.40 relevance floor because near-DUPLICATE is a strictly stronger relation than relevance.
77
+ const NEAR_DUP_THRESHOLD = 0.85;
78
+ // Operator-invoked (an explicit `wrxn harvest`), not the per-prompt hot path — so a generous query budget
79
+ // (recall-surface caps at 512 for its 150ms ceiling; truncating the near-dup query that hard suppresses
80
+ // the signal — the measurement showed a 512-char self-query drops to ~0.78). We send more of the body.
81
+ const NEAR_DUP_QUERY_CHARS = 2000;
82
+ const FETCH_LIMIT = 15; // ask the door wide; we post-filter to harvest-tier prose then threshold.
83
+ const TIMEOUT_MS = 5000; // bounded so a wedged door can't hang the command (sync.cjs parity).
84
+ const MAX_RESPONSE_BYTES = 4 * 1024 * 1024; // hard cap on an accumulated door response body (anti-flood).
85
+
86
+ // ── install-root resolution (mirrors wiki.cjs / dream.cjs / sync.cjs) ─────────────
87
+ function findInstallRoot(start) {
88
+ let dir = start || process.env.CLAUDE_PROJECT_DIR || process.cwd();
89
+ for (let i = 0; i < 12; i++) {
90
+ if (fs.existsSync(path.join(dir, 'wrxn.install.json'))) return dir;
91
+ const up = path.dirname(dir);
92
+ if (up === dir) break;
93
+ dir = up;
94
+ }
95
+ return null;
96
+ }
97
+
98
+ function flag(name) {
99
+ const i = process.argv.indexOf(`--${name}`);
100
+ return i > -1 ? process.argv[i + 1] : undefined;
101
+ }
102
+
103
+ function installRoot() {
104
+ const root = flag('root') || findInstallRoot();
105
+ if (!root) {
106
+ fail('cannot resolve the install root — run inside a wrxn install (no wrxn.install.json found walking up) or pass --root <dir>');
107
+ }
108
+ return root;
109
+ }
110
+
111
+ function fail(msg) {
112
+ process.stderr.write(`harvest: ${msg}\n`);
113
+ process.exit(2);
114
+ }
115
+
116
+ function print(obj) {
117
+ process.stdout.write(JSON.stringify(obj, null, 2) + '\n');
118
+ }
119
+
120
+ // ── frontmatter helpers (mirror drift-detect.cjs) ────────────────────────────────
121
+ function frontmatter(content) {
122
+ const m = /^---\r?\n([\s\S]*?)\r?\n---/.exec(String(content));
123
+ return m ? m[1] : '';
124
+ }
125
+
126
+ function unquote(s) {
127
+ return String(s).trim().replace(/^["']|["']$/g, '').trim();
128
+ }
129
+
130
+ // Normalize a derived_from path-ish value to an install-root-relative POSIX path: drop a `#symbol`
131
+ // anchor, resolve relative/absolute forms. Returns '' on empty. (drift-detect.cjs relTo.)
132
+ function relTo(root, p) {
133
+ const s = String(p == null ? '' : p).split('#')[0].trim();
134
+ if (!s) return '';
135
+ const abs = path.isAbsolute(s) ? s : path.resolve(root, s);
136
+ return path.relative(root, abs).split(path.sep).join('/');
137
+ }
138
+
139
+ // Parse the `derived_from:` declaration(s) from a page's frontmatter — scalar, inline list, or block
140
+ // list. Returns the raw value strings (anchors intact). (drift-detect.cjs parseDerivedFrom.)
141
+ function parseDerivedFrom(content) {
142
+ const fm = frontmatter(content);
143
+ if (!fm) return [];
144
+ const lines = fm.split(/\r?\n/);
145
+ const out = [];
146
+ for (let i = 0; i < lines.length; i++) {
147
+ const m = /^derived_from:\s*(.*)$/.exec(lines[i]);
148
+ if (!m) continue;
149
+ const val = m[1].trim();
150
+ if (val.startsWith('[')) {
151
+ for (const part of val.replace(/^\[|\]$/g, '').split(',')) {
152
+ const v = unquote(part);
153
+ if (v) out.push(v);
154
+ }
155
+ } else if (val) {
156
+ out.push(unquote(val));
157
+ } else {
158
+ for (let j = i + 1; j < lines.length; j++) {
159
+ const li = /^\s*-\s+(.*)$/.exec(lines[j]);
160
+ if (!li) break;
161
+ const v = unquote(li[1]);
162
+ if (v) out.push(v);
163
+ }
164
+ }
165
+ }
166
+ return out;
167
+ }
168
+
169
+ // The `superseded_by:` forward-link, or null — the supersession convention H4 writes. Exported pure helper;
170
+ // NOT consulted by scanLocal, which treats a `superseded_by:` page as the RESOLVED end state, not debt.
171
+ function parseSupersededBy(content) {
172
+ const fm = frontmatter(content);
173
+ if (!fm) return null;
174
+ const m = /^superseded_by:\s*(.+)$/m.exec(fm);
175
+ const v = m ? unquote(m[1]) : null;
176
+ return v || null;
177
+ }
178
+
179
+ // The malformed signal (wiki-lint.cjs lintPage): the reason a page is malformed, or null when well-formed.
180
+ function lintPage(text) {
181
+ const src = String(text || '');
182
+ if (!src.startsWith('---')) return 'no frontmatter';
183
+ const end = src.indexOf('\n---', 3);
184
+ if (end < 0) return 'unterminated frontmatter';
185
+ const fm = src.slice(3, end);
186
+ const missing = REQUIRED_KEYS.filter((k) => !new RegExp(`^${k}\\s*:`, 'm').test(fm));
187
+ if (missing.length) return `missing ${missing.join('/')}`;
188
+ return null;
189
+ }
190
+
191
+ // ── tier/slug derivation from a wiki relpath ─────────────────────────────────────
192
+ // `.wrxn/wiki/<tier>/<slug>.md` → tier (or null if not a tier-scoped wiki page).
193
+ function tierOfPath(file) {
194
+ const m = /^\.wrxn\/wiki\/([^/]+)\/[^/]+\.md$/.exec(String(file || ''));
195
+ return m ? m[1] : null;
196
+ }
197
+
198
+ function isHarvestPath(file) {
199
+ return HARVEST_TIERS.includes(tierOfPath(file));
200
+ }
201
+
202
+ function slugOfPath(file) {
203
+ return path.basename(String(file || ''), '.md');
204
+ }
205
+
206
+ // List every .md page under the 4 harvest tiers as { file (relpath), slug, tier, query }. A missing tier
207
+ // dir is skipped. The query is the page BODY (frontmatter stripped) trimmed to the near-dup budget — the
208
+ // most discriminating near-dup signal — falling back to the slug for an empty body.
209
+ function listProsePages(root) {
210
+ const out = [];
211
+ for (const tier of HARVEST_TIERS) {
212
+ const dir = path.join(root, '.wrxn', 'wiki', tier);
213
+ let names;
214
+ try {
215
+ names = fs.readdirSync(dir).filter((n) => n.endsWith('.md'));
216
+ } catch {
217
+ continue; // tier dir absent → no pages
218
+ }
219
+ for (const name of names) {
220
+ const file = path.join(dir, name);
221
+ let text;
222
+ try {
223
+ text = fs.readFileSync(file, 'utf8');
224
+ } catch {
225
+ continue; // unreadable page → skip (fail-soft per file)
226
+ }
227
+ const rel = path.relative(root, file).split(path.sep).join('/');
228
+ const slug = name.replace(/\.md$/, '');
229
+ const body = String(text).replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/, '').trim();
230
+ out.push({ file: rel, slug, tier, query: (body || slug).slice(0, NEAR_DUP_QUERY_CHARS) });
231
+ }
232
+ }
233
+ return out;
234
+ }
235
+
236
+ // ── the local scan (PURE over fs) — malformed + decay (orphaned/superseded) ──────
237
+ function scanLocal(root) {
238
+ const malformed = [];
239
+ const decay = [];
240
+ for (const tier of HARVEST_TIERS) {
241
+ const dir = path.join(root, '.wrxn', 'wiki', tier);
242
+ let names;
243
+ try {
244
+ names = fs.readdirSync(dir).filter((n) => n.endsWith('.md'));
245
+ } catch {
246
+ continue;
247
+ }
248
+ for (const name of names) {
249
+ const file = path.join(dir, name);
250
+ let text;
251
+ try {
252
+ text = fs.readFileSync(file, 'utf8');
253
+ } catch {
254
+ continue;
255
+ }
256
+ const rel = path.relative(root, file).split(path.sep).join('/');
257
+ const slug = name.replace(/\.md$/, '');
258
+
259
+ const reason = lintPage(text);
260
+ if (reason) malformed.push({ type: 'malformed', slug, path: rel, tier, reason });
261
+
262
+ // An already-annotated page (carries stale: or superseded_by:) is RESOLVED — its decay was already
263
+ // actioned: a superseded_by: forward-link is the desired end state (PRD US3, "forward-linked, not
264
+ // deleted"), and a stale: stamp is the resolution harvest offers for an orphan. Re-emitting it as a
265
+ // decay_candidate would re-arm the handoff debt nudge over a fully-curated tree forever (harvest-05
266
+ // AC2/AC8: clean set → silent). Skip the decay scan for it (the malformed lint above still applies).
267
+ if (hasFrontmatterKey(text, 'stale') || hasFrontmatterKey(text, 'superseded_by')) continue;
268
+
269
+ // orphaned — a declared derived_from source FILE no longer exists on disk (a moved SYMBOL in an
270
+ // existing file is drift, sync's job, not orphaning; we strip the #anchor and test the FILE).
271
+ for (const raw of parseDerivedFrom(text)) {
272
+ const cleaned = String(raw).split('#')[0].trim();
273
+ if (!cleaned) continue;
274
+ const abs = path.isAbsolute(cleaned) ? cleaned : path.resolve(root, cleaned);
275
+ if (!fs.existsSync(abs)) {
276
+ decay.push({ type: 'decay_candidate', subtype: 'orphaned', slug, path: rel, tier, reason: `derived_from source missing: ${relTo(root, raw)}`, missing_source: relTo(root, raw) });
277
+ }
278
+ }
279
+ }
280
+ }
281
+ return { malformed, decay };
282
+ }
283
+
284
+ // ── near-dup gate + clusterer (PURE) ─────────────────────────────────────────────
285
+ function isProse(hit) {
286
+ return !!hit && PROSE_TYPES.has(hit.type);
287
+ }
288
+
289
+ // A near-dup edge requires the dense cosine to clear the MEASURED threshold AND the dense arm to actually
290
+ // be present in `sources` (not just a stray semanticScore) — mirrors recall-surface's producer-drift
291
+ // defense. The fused RRF `score` is never consulted.
292
+ function nearDupQualifies(hit) {
293
+ const sem = Number(hit && hit.semanticScore);
294
+ const s = hit && hit.sources;
295
+ const hasSemantic = Array.isArray(s) && s.includes('semantic');
296
+ return Number.isFinite(sem) && hasSemantic && sem >= NEAR_DUP_THRESHOLD;
297
+ }
298
+
299
+ // Collapse pairwise near-dup edges into connected-component CLUSTERS (union-find), so a symmetric A↔B
300
+ // match is reported once and a transitive A-B-C chain is one cluster of three. Each cluster carries its
301
+ // sorted members + the STRONGEST edge similarity (the clearest dup signal). Singletons are not clusters.
302
+ function clusterNearDups(edges) {
303
+ const parent = new Map();
304
+ const find = (x) => {
305
+ while (parent.get(x) !== x) {
306
+ parent.set(x, parent.get(parent.get(x)));
307
+ x = parent.get(x);
308
+ }
309
+ return x;
310
+ };
311
+ const add = (x) => { if (!parent.has(x)) parent.set(x, x); };
312
+ for (const e of edges) {
313
+ add(e.a);
314
+ add(e.b);
315
+ const ra = find(e.a);
316
+ const rb = find(e.b);
317
+ if (ra !== rb) parent.set(ra, rb);
318
+ }
319
+ const groups = new Map();
320
+ for (const node of parent.keys()) {
321
+ const r = find(node);
322
+ if (!groups.has(r)) groups.set(r, new Set());
323
+ groups.get(r).add(node);
324
+ }
325
+ const groupScore = new Map();
326
+ for (const e of edges) {
327
+ const r = find(e.a);
328
+ const cur = groupScore.has(r) ? groupScore.get(r) : -Infinity;
329
+ if (e.score > cur) groupScore.set(r, e.score);
330
+ }
331
+ const clusters = [];
332
+ for (const [r, members] of groups) {
333
+ if (members.size < 2) continue;
334
+ clusters.push({ members: [...members].sort(), score: Math.round((groupScore.get(r) || 0) * 1e4) / 1e4 });
335
+ }
336
+ return clusters.sort((a, b) => (a.members[0] < b.members[0] ? -1 : 1));
337
+ }
338
+
339
+ // ── the door (IO shell, injectable transport) — the recall-surface.cjs contract ──
340
+ function pidAlive(pid) {
341
+ try {
342
+ process.kill(pid, 0);
343
+ return true;
344
+ } catch (e) {
345
+ return !!e && e.code === 'EPERM';
346
+ }
347
+ }
348
+
349
+ // Refuse a discovery file another user could have planted, or that is group/world-writable. lstat so a
350
+ // symlink's OWN ownership/mode is judged. Any fault → not trusted (treated as not-warm).
351
+ function endpointTrusted(file) {
352
+ let st;
353
+ try {
354
+ st = fs.lstatSync(file);
355
+ } catch {
356
+ return false;
357
+ }
358
+ if (typeof process.getuid === 'function' && st.uid !== process.getuid()) return false;
359
+ if ((st.mode & 0o022) !== 0) return false;
360
+ return true;
361
+ }
362
+
363
+ // Discover the warm serve door from <root>/.recon-wrxn/serve-endpoint.json = {pid,port}. Returns
364
+ // {pid,port} only when the file is well-owned, present, well-formed, and the pid is alive — else null.
365
+ function discoverEndpoint(root) {
366
+ const file = path.join(root, ENDPOINT_REL);
367
+ if (!endpointTrusted(file)) return null;
368
+ let raw;
369
+ try {
370
+ raw = fs.readFileSync(file, 'utf8');
371
+ } catch {
372
+ return null;
373
+ }
374
+ let obj;
375
+ try {
376
+ obj = JSON.parse(raw);
377
+ } catch {
378
+ return null;
379
+ }
380
+ const pid = Number(obj && obj.pid);
381
+ const port = Number(obj && obj.port);
382
+ if (!Number.isInteger(pid) || pid <= 0) return null;
383
+ if (!Number.isInteger(port) || port <= 0) return null;
384
+ if (!pidAlive(pid)) return null;
385
+ return { pid, port };
386
+ }
387
+
388
+ // Default transport: a real POST over http with a hard timeout. Resolves {statusCode, body}; rejects on
389
+ // socket error or timeout. Injectable so unit tests never touch the network (mirrors sync.cjs).
390
+ function httpTransport({ port, path: reqPath, body, timeoutMs }) {
391
+ return new Promise((resolve, reject) => {
392
+ const payload = Buffer.from(JSON.stringify(body));
393
+ const deadline = timeoutMs || TIMEOUT_MS;
394
+ let settled = false;
395
+ let wall = null;
396
+ const done = (fn, arg) => {
397
+ if (settled) return;
398
+ settled = true;
399
+ if (wall) clearTimeout(wall);
400
+ fn(arg);
401
+ };
402
+ const req = http.request(
403
+ {
404
+ host: '127.0.0.1',
405
+ port,
406
+ path: reqPath,
407
+ method: 'POST',
408
+ headers: { 'Content-Type': 'application/json', 'Content-Length': payload.length },
409
+ },
410
+ (res) => {
411
+ const chunks = [];
412
+ let total = 0;
413
+ res.on('data', (c) => {
414
+ total += c.length;
415
+ if (total > MAX_RESPONSE_BYTES) { req.destroy(new Error('harvest door response too large')); return; }
416
+ chunks.push(c);
417
+ });
418
+ res.on('end', () => done(resolve, { statusCode: res.statusCode, body: Buffer.concat(chunks).toString('utf8') }));
419
+ res.on('error', (e) => done(reject, e));
420
+ }
421
+ );
422
+ req.on('error', (e) => done(reject, e));
423
+ req.setTimeout(deadline, () => req.destroy(new Error('harvest door timeout')));
424
+ wall = setTimeout(() => req.destroy(new Error('harvest door wall-clock timeout')), deadline);
425
+ req.write(payload);
426
+ req.end();
427
+ });
428
+ }
429
+
430
+ // IO shell: discover the door; if cold → { status:'unavailable' } (AC4). Otherwise query recon_find once
431
+ // per harvest-tier page with the page body, keep prose hits IN the 4 tiers that clear the threshold
432
+ // (excluding self), build pairwise edges, and cluster them. A per-page query that throws / returns
433
+ // non-200 / a malformed body contributes no edges (fail-soft per page) — the scan still completes.
434
+ async function nearDupFromDoor(root, { transport, timeoutMs } = {}) {
435
+ const door = discoverEndpoint(root);
436
+ if (!door) return { status: 'unavailable', clusters: [] }; // not warm → near-dup unavailable
437
+ const pages = listProsePages(root);
438
+ const edges = [];
439
+ for (const p of pages) {
440
+ let resp;
441
+ try {
442
+ resp = await (transport || httpTransport)({
443
+ port: door.port,
444
+ path: FIND_PATH,
445
+ body: { query: p.query, limit: FETCH_LIMIT },
446
+ timeoutMs: timeoutMs || TIMEOUT_MS,
447
+ });
448
+ } catch {
449
+ continue; // timeout / connection refused → no edges from this page
450
+ }
451
+ if (!resp || resp.statusCode !== 200) continue;
452
+ let parsed;
453
+ try {
454
+ parsed = JSON.parse(resp.body);
455
+ } catch {
456
+ continue; // malformed body → no edges from this page
457
+ }
458
+ const hits = Array.isArray(parsed.hits) ? parsed.hits : [];
459
+ for (const h of hits) {
460
+ if (!isProse(h) || !isHarvestPath(h.file) || h.file === p.file) continue;
461
+ if (!nearDupQualifies(h)) continue;
462
+ edges.push({ a: p.file, b: h.file, score: Number(h.semanticScore) });
463
+ }
464
+ }
465
+ return { status: 'ok', clusters: clusterNearDups(edges) };
466
+ }
467
+
468
+ // ── record assembly (PURE) ───────────────────────────────────────────────────────
469
+ // One JSON record per finding, each carrying enough to seed a downstream proposal (AC5). A cold door
470
+ // emits a single near_dup "unavailable" marker (AC4); malformed + decay always pass through.
471
+ function assembleRecords(ts, near, local) {
472
+ const records = [];
473
+ if (!near || near.status === 'unavailable') {
474
+ records.push({ ts, type: 'near_dup', status: 'unavailable', reason: 'recon serve door not warm — near-dup detection skipped (malformed + decay scans still ran)' });
475
+ } else {
476
+ for (const c of near.clusters) {
477
+ records.push({
478
+ ts,
479
+ type: 'near_dup',
480
+ members: c.members.map((f) => ({ slug: slugOfPath(f), path: f, tier: tierOfPath(f) })),
481
+ score: c.score,
482
+ threshold: NEAR_DUP_THRESHOLD,
483
+ reason: `semantic similarity >= ${NEAR_DUP_THRESHOLD} (measured near-dup band; dense cosine over the recon door)`,
484
+ });
485
+ }
486
+ }
487
+ for (const d of local.decay) records.push(Object.assign({ ts }, d));
488
+ for (const m of local.malformed) records.push(Object.assign({ ts }, m));
489
+ return records;
490
+ }
491
+
492
+ function harvestDir(root) {
493
+ const dir = path.join(root, ...HARVEST_DIR);
494
+ fs.mkdirSync(dir, { recursive: true });
495
+ return dir;
496
+ }
497
+
498
+ // A fresh, never-mutated timestamped report path (AC2). A same-millisecond collision bumps a counter.
499
+ function reportPath(dir, ts) {
500
+ const base = ts.replace(/[:.]/g, '-');
501
+ let file = path.join(dir, `${base}.jsonl`);
502
+ let n = 1;
503
+ while (fs.existsSync(file)) file = path.join(dir, `${base}-${n++}.jsonl`);
504
+ return file;
505
+ }
506
+
507
+ // ── check: the IO orchestrator ───────────────────────────────────────────────────
508
+ // scanLocal (always) + nearDupFromDoor (degrades to unavailable when cold) → assemble → write the jsonl.
509
+ // REPORT-ONLY: the ONLY write is the report under .wrxn/harvest/. `transport` is injected in tests.
510
+ async function check(root, { transport, timeoutMs } = {}) {
511
+ const local = scanLocal(root);
512
+ let near;
513
+ try {
514
+ near = await nearDupFromDoor(root, { transport, timeoutMs });
515
+ } catch {
516
+ near = { status: 'unavailable', clusters: [] }; // belt-and-suspenders: never throws
517
+ }
518
+ const ts = new Date().toISOString();
519
+ const records = assembleRecords(ts, near, local);
520
+ const dir = harvestDir(root);
521
+ const file = reportPath(dir, ts);
522
+ fs.writeFileSync(file, records.length ? records.map((r) => JSON.stringify(r)).join('\n') + '\n' : '');
523
+ const nearDupCount = near.status === 'unavailable' ? 0 : near.clusters.length;
524
+ return {
525
+ report: path.relative(root, file),
526
+ records,
527
+ summary: {
528
+ nearDupStatus: near.status,
529
+ findings: { near_dup: nearDupCount, decay_candidate: local.decay.length, malformed: local.malformed.length },
530
+ },
531
+ };
532
+ }
533
+
534
+ async function runCheck() {
535
+ const root = installRoot();
536
+ const res = await check(root, {});
537
+ print({ report: res.report, summary: res.summary });
538
+ process.exit(0);
539
+ }
540
+
541
+ // ════════════════════════════════════════════════════════════════════════════════
542
+ // MERGE (harvest-03 / H3) — the ONE sanctioned hard-delete: fold N near-dups into one survivor, then
543
+ // delete the absorbed. The SKILL (LLM) drafts the survivor (union of facts + union of evidence); THIS
544
+ // adapter GATES + writes. Reuses sync's propose→confirm by-reference spine (secret-scan, sha256 integrity,
545
+ // path-confinement) and dream's wiki-bridge indirection (the absorbed delete goes through wiki.cjs).
546
+ // ════════════════════════════════════════════════════════════════════════════════
547
+
548
+ // ── shared CLI/IO helpers (mirror sync.cjs / dream.cjs) ──────────────────────────
549
+ // The first positional after the subcommand (the JSON file path), up to the first --flag.
550
+ function positionalFile() {
551
+ for (let i = 3; i < process.argv.length; i++) {
552
+ if (process.argv[i].startsWith('--')) break;
553
+ return process.argv[i];
554
+ }
555
+ fail('missing <file.json> argument');
556
+ return undefined;
557
+ }
558
+
559
+ // The first positional at-or-after `start` (up to the first --flag), or undefined when none. Used by the
560
+ // two-word `decay propose|confirm` grammar (the sub-verb is argv[3], so its file argument starts at argv[4]).
561
+ function positionalAfter(start) {
562
+ for (let i = start; i < process.argv.length; i++) {
563
+ if (process.argv[i].startsWith('--')) break;
564
+ return process.argv[i];
565
+ }
566
+ return undefined;
567
+ }
568
+
569
+ function readJson(file) {
570
+ try {
571
+ return JSON.parse(fs.readFileSync(file, 'utf8'));
572
+ } catch (err) {
573
+ fail(`cannot read JSON from "${file}": ${err.message}`);
574
+ return undefined;
575
+ }
576
+ }
577
+
578
+ function appendLine(file, obj) {
579
+ fs.appendFileSync(file, JSON.stringify(obj) + '\n');
580
+ }
581
+
582
+ function isObj(x) {
583
+ return !!x && typeof x === 'object' && !Array.isArray(x);
584
+ }
585
+
586
+ // A wiki slug — the same kebab contract wiki.cjs enforces. A non-kebab absorbed slug would (a) inject a
587
+ // newline/colon into the survivor's `merged_from:` frontmatter and (b) fail wiki.cjs delete-page — reject it.
588
+ function isKebab(s) {
589
+ return typeof s === 'string' && /^[a-z0-9][a-z0-9-]*$/.test(s);
590
+ }
591
+
592
+ // ── credential / secret scan (reused from dream.cjs / sync.cjs, security M2) ─────
593
+ // A merged survivor must never harden a session secret into recalled prose. Same patterns + case-sensitive
594
+ // scope as dream/sync — replicated because each install-only adapter is self-contained (node stdlib only).
595
+ const SECRET_PATTERNS = [
596
+ /AKIA[0-9A-Z]{16}/, // AWS access key id
597
+ /gh[pousr]_[A-Za-z0-9]{36}/, // GitHub token (ghp_/gho_/ghu_/ghs_/ghr_)
598
+ /npm_[A-Za-z0-9]{36}/, // npm automation token
599
+ /sk-[A-Za-z0-9]{20,}/, // OpenAI-style secret key
600
+ /-----BEGIN [A-Z ]*PRIVATE KEY-----/, // PEM private-key header
601
+ ];
602
+
603
+ function secretScan(text) {
604
+ const s = String(text || ''); // NOT lowercased — the token shapes are case-sensitive.
605
+ for (const re of SECRET_PATTERNS) if (re.test(s)) return 'contains_secret';
606
+ return null;
607
+ }
608
+
609
+ // ── path safety: a merge target may address ONLY a .md page under a KNOWLEDGE tier ─
610
+ // The survivor + every absorbed path is LLM/operator-controlled, so it is the trust boundary. This is
611
+ // sync's resolveSafeDoc TIGHTENED to the curation scope: the path must resolve to exactly
612
+ // .wrxn/wiki/<tier>/<slug>.md where tier ∈ HARVEST_TIERS (concepts/decisions/gotchas/_rules) — not the
613
+ // retired sessions tier, not the _slots focus slot, not anything outside the wiki subtree. Returns the
614
+ // abs path or null. Lexical (path.resolve never follows a symlink) — a planted symlink's STRING is judged.
615
+ function resolveSafeHarvestDoc(root, doc) {
616
+ if (typeof doc !== 'string' || !doc.trim()) return null;
617
+ const wikiRoot = path.join(root, ...WIKI_REL);
618
+ const abs = path.resolve(root, doc);
619
+ const rel = path.relative(wikiRoot, abs);
620
+ if (!rel || rel.startsWith('..') || path.isAbsolute(rel)) return null; // escapes .wrxn/wiki/
621
+ if (!abs.endsWith('.md')) return null; // prose pages only
622
+ const parts = rel.split(path.sep);
623
+ if (parts.length !== 2) return null; // must be exactly <tier>/<slug>.md — no nesting
624
+ if (!HARVEST_TIERS.includes(parts[0])) return null; // not a knowledge tier (sessions / _slots / other)
625
+ // EXPLICIT tier-resolves invariant (harvest-review). path.resolve above COLLAPSES `..`, so a NON-CANONICAL
626
+ // form like `.wrxn/wiki/concepts/../concepts/x.md` survives the confinement check yet tierOfPath() returns
627
+ // null on that raw string — and the merge/decay callers stamp tier/slug from this SAME string (tierOfPath/
628
+ // slugOfPath), yielding a malformed `tier: null` survivor page on the ONE destructive (delete) path. Require
629
+ // the path the callers stamp from to tier-resolve cleanly, so the page written and the recorded tier always
630
+ // agree. tierOfPath only matches a clean `.wrxn/wiki/<tier>/<slug>.md` → a non-canonical path is refused.
631
+ if (!tierOfPath(doc)) return null; // non-canonical / not tier-resolvable → fail-closed (no tier:null page, no delete)
632
+ return abs;
633
+ }
634
+
635
+ // The integrity fingerprint captured at stage over the fields that DETERMINE the write+deletes (survivor
636
+ // target, its description+body, the absorbed set). Recomputed at the write boundary and compared to the
637
+ // staged value → a record whose body/target/absorbed-list was altered after staging cannot write or delete
638
+ // (AC2 tamper-refusal). absorbed is sorted so the hash is order-independent (mirrors sync's proposalHash).
639
+ function mergeHash(p) {
640
+ const absorbed = (Array.isArray(p.absorbed) ? p.absorbed.map(String) : []).slice().sort();
641
+ const canon = JSON.stringify({ survivor: String(p.survivor || ''), description: String(p.description || ''), body: String(p.body || ''), absorbed });
642
+ return crypto.createHash('sha256').update(canon).digest('hex');
643
+ }
644
+
645
+ // ── the survivor frontmatter write-channel sanitiser (PURE) ──────────────────────
646
+ // composeSurvivor interpolates the LLM/operator-drafted `description` VERBATIM as `description: <value>`
647
+ // on one frontmatter line. A newline injects an arbitrary extra frontmatter key (e.g. a poisoned
648
+ // `importance:` that hijacks decay-weighted recall — harvest never stamps importance: itself); a colon
649
+ // creates YAML mapping ambiguity. Reject both — the same write-channel discipline decay's
650
+ // annotationValueProblem applies, here WITHOUT its path-length cap + non-empty check (a description is
651
+ // optional free prose). name/tier/merged_from in the same fence are kebab/allowlist-validated already, so
652
+ // `description` is the lone free-text frontmatter field. Returns a problem code or null.
653
+ function descriptionProblem(value) {
654
+ if (/[\r\n:]/.test(String(value == null ? '' : value))) return 'malformed_description';
655
+ return null;
656
+ }
657
+
658
+ // ── the survivor page (PURE) — the net-new write transform ───────────────────────
659
+ // Compose a well-formed knowledge page (passes lintPage: name/description/tier) stamped with
660
+ // `merged_from: [<absorbed slugs>]` — the provenance lands on the SURVIVING page, NEVER on the deleted
661
+ // ones (you cannot stamp a deleted page). `superseded_by:` is H4's non-destructive op, not merge's. The
662
+ // absorbed slugs are kebab-validated before they reach here, so the inline list cannot inject frontmatter.
663
+ function composeSurvivor({ tier, slug, description, body, mergedFrom }) {
664
+ return [
665
+ '---',
666
+ `name: ${slug}`,
667
+ `description: ${description || ''}`,
668
+ `tier: ${tier}`,
669
+ 'source: harvest-merge',
670
+ `merged_from: [${mergedFrom.join(', ')}]`,
671
+ '---',
672
+ '',
673
+ body,
674
+ '',
675
+ ].join('\n');
676
+ }
677
+
678
+ // ── the wiki delete bridge (dream.cjs indirection contract) ──────────────────────
679
+ function wikiAdapter() {
680
+ return path.join(__dirname, 'wiki.cjs'); // sibling in the same install .wrxn/ dir
681
+ }
682
+
683
+ // Guard the harvest→wiki bridge against argv flag-injection (dream's security M3): a `--`-leading value
684
+ // would be parsed by wiki.cjs's flag scan as a flag. tier/slug are allowlisted/kebab so this is the
685
+ // defense-in-depth backstop at the exec boundary.
686
+ function guardArgv(values) {
687
+ for (const v of values) {
688
+ if (typeof v === 'string' && v.startsWith('--')) {
689
+ throw new Error(`flag-injection guard: refusing a --leading value at the wiki bridge: ${JSON.stringify(v.slice(0, 32))}`);
690
+ }
691
+ }
692
+ }
693
+
694
+ // Delete an absorbed page VIA the wiki adapter's delete-by-reference path (the indirection contract — we
695
+ // never unlink a .md directly). wiki.cjs confines the delete to the wiki subtree (tier allowlist + kebab
696
+ // slug); harvest has ALREADY confined to the knowledge tiers via resolveSafeHarvestDoc — defense in depth.
697
+ function wikiDeletePage(root, tier, slug) {
698
+ guardArgv([tier, slug]);
699
+ const args = [wikiAdapter(), 'delete-page', tier, slug, '--root', root];
700
+ return JSON.parse(execFileSync('node', args, { encoding: 'utf8' }));
701
+ }
702
+
703
+ // ── stage / commit by-reference (mirror sync's propose/confirm) ──────────────────
704
+ // Read .wrxn/harvest/staged.jsonl into a survivor → staged-record map (last staged wins). Malformed lines
705
+ // skip. A record needs a survivor key, a string body, and an absorbed array to be a usable merge.
706
+ function readStaged(root) {
707
+ const map = new Map();
708
+ let txt;
709
+ try {
710
+ txt = fs.readFileSync(path.join(root, ...HARVEST_DIR, STAGED_FILE), 'utf8');
711
+ } catch {
712
+ return map; // no staging trail yet → nothing to confirm by reference
713
+ }
714
+ for (const line of txt.split('\n')) {
715
+ const s = line.trim();
716
+ if (!s) continue;
717
+ try {
718
+ const rec = JSON.parse(s);
719
+ if (rec && rec.survivor && typeof rec.body === 'string' && Array.isArray(rec.absorbed)) map.set(rec.survivor, rec);
720
+ } catch {
721
+ /* skip a malformed staging line */
722
+ }
723
+ }
724
+ return map;
725
+ }
726
+
727
+ // Normalize commit input into the operator-approved SURVIVOR list (["survivor-path"…] or { approved:[…] }).
728
+ // An EMPTY list is the DECLINE — commit writes nothing, deletes nothing (AC3).
729
+ function approvedSurvivors(input) {
730
+ if (Array.isArray(input)) return input.map(String);
731
+ if (input && typeof input === 'object' && Array.isArray(input.approved)) return input.approved.map(String);
732
+ return [];
733
+ }
734
+
735
+ // stage (PROPOSE): validate the drafted survivor + the absorbed cluster, secret-scan, then record the
736
+ // proposal by-reference under .wrxn/harvest/staged.jsonl with an integrity fingerprint. NEVER touches a
737
+ // knowledge page (the survivor is not written, the absorbed are not deleted) — mirrors sync's propose.
738
+ function runStage() {
739
+ const input = readJson(positionalFile());
740
+ const root = installRoot();
741
+ const p = isObj(input) ? input : {};
742
+
743
+ const survAbs = resolveSafeHarvestDoc(root, p.survivor);
744
+ if (!survAbs) fail('stage needs a "survivor" path under .wrxn/wiki/<knowledge-tier>/ ending in .md');
745
+ if (!isKebab(slugOfPath(p.survivor))) fail('the survivor slug must be kebab-case ([a-z0-9-])');
746
+ if (typeof p.body !== 'string' || !p.body.trim()) fail('stage needs a non-empty "body" — the synthesised survivor the skill drafted');
747
+ if (p.body.length > BODY_MAX) fail(`stage rejected — body exceeds the ${BODY_MAX}-char cap (body_too_large); a durable merged page, not a dump`);
748
+
749
+ const absorbed = Array.isArray(p.absorbed) ? p.absorbed.map(String) : [];
750
+ if (absorbed.length === 0) fail('stage needs a non-empty "absorbed" list — the near-dup slugs folded into the survivor');
751
+ for (const a of absorbed) {
752
+ if (!resolveSafeHarvestDoc(root, a)) fail(`absorbed target escapes the knowledge tiers (must be .wrxn/wiki/<knowledge-tier>/<slug>.md): ${JSON.stringify(a)}`);
753
+ if (!isKebab(slugOfPath(a))) fail(`absorbed slug must be kebab-case: ${JSON.stringify(a)}`);
754
+ if (a === p.survivor) fail('the survivor cannot also be an absorbed (delete) target');
755
+ }
756
+
757
+ const description = p.description ? String(p.description) : '';
758
+ const sec = secretScan(`${description}\n${p.body}`); // AC1: secret-scan BEFORE staging
759
+ if (sec) fail(`stage rejected — the drafted survivor contains a credential (${sec}); never fold a session secret into knowledge`);
760
+ if (descriptionProblem(description)) fail('stage rejected — the survivor description must not contain a newline or colon (frontmatter-injection guard)');
761
+
762
+ const dir = harvestDir(root);
763
+ const ts = new Date().toISOString();
764
+ const record = {
765
+ ts, op: 'stage',
766
+ survivor: p.survivor, tier: tierOfPath(p.survivor), slug: slugOfPath(p.survivor),
767
+ description, body: p.body, absorbed,
768
+ hash: mergeHash({ survivor: p.survivor, description, body: p.body, absorbed }),
769
+ };
770
+ appendLine(path.join(dir, STAGED_FILE), record);
771
+ appendLine(path.join(dir, AUDIT_FILE), { ts, op: 'stage', survivor: p.survivor, absorbed });
772
+ return print({ staged: 1, survivor: p.survivor, absorbed, stagedFile: path.relative(root, path.join(dir, STAGED_FILE)) });
773
+ }
774
+
775
+ // commitOne: the write-boundary re-gate for ONE approved survivor. RE-VALIDATE (secret-scan → integrity →
776
+ // path-confine survivor AND every absorbed → survivor is new) BEFORE any mutation, so a tampered/altered
777
+ // proposal cannot write or delete (AC2). Then — and only then — WRITE THE SURVIVOR FIRST (knowledge
778
+ // preserved), THEN delete each absorbed (AC4 survivor-before-delete). Atomic on validation: if ANY target
779
+ // is unsafe the whole merge is refused (no partial delete). Returns { ok, ... } | { ok:false, reason }.
780
+ function commitOne(root, rec) {
781
+ const description = rec.description ? String(rec.description) : '';
782
+ const sec = secretScan(`${description}\n${rec.body}`); // re-scan at the write boundary
783
+ if (sec) return { ok: false, reason: sec };
784
+ if (descriptionProblem(description)) return { ok: false, reason: 'malformed_description' }; // re-check the frontmatter write channel — a tampered staged record (valid hash) can't smuggle an injected key through
785
+ if (mergeHash(rec) !== rec.hash) return { ok: false, reason: 'integrity_mismatch' }; // tamper → refuse
786
+
787
+ const survAbs = resolveSafeHarvestDoc(root, rec.survivor);
788
+ if (!survAbs) return { ok: false, reason: 'unsafe_survivor' };
789
+ if (!isKebab(slugOfPath(rec.survivor))) return { ok: false, reason: 'unsafe_survivor' };
790
+ // additive / refuse-overwrite: the survivor is a NEW page. lstat (not existsSync) so a planted symlink or
791
+ // any pre-existing entry is caught — never clobber a curated page. This check runs BEFORE any delete, so a
792
+ // refused survivor write means nothing is deleted (the structural proof of survivor-before-delete).
793
+ try {
794
+ fs.lstatSync(survAbs);
795
+ return { ok: false, reason: 'survivor_exists' };
796
+ } catch {
797
+ /* nothing there → safe to create */
798
+ }
799
+
800
+ // confine + validate EVERY absorbed target up front (atomic) — one unsafe target refuses the whole merge.
801
+ const targets = [];
802
+ for (const a of rec.absorbed.map(String)) {
803
+ const abs = resolveSafeHarvestDoc(root, a);
804
+ if (!abs || !isKebab(slugOfPath(a))) return { ok: false, reason: 'unsafe_absorbed' };
805
+ if (a === rec.survivor) return { ok: false, reason: 'survivor_in_absorbed' }; // never delete the survivor
806
+ targets.push({ rel: a, tier: tierOfPath(a), slug: slugOfPath(a) });
807
+ }
808
+
809
+ // WRITE THE SURVIVOR FIRST — the merged knowledge exists on disk before any page is deleted (AC4).
810
+ const mergedFrom = targets.map((t) => t.slug).slice().sort();
811
+ const page = composeSurvivor({ tier: rec.tier, slug: rec.slug, description, body: rec.body, mergedFrom });
812
+ fs.mkdirSync(path.dirname(survAbs), { recursive: true });
813
+ fs.writeFileSync(survAbs, page);
814
+
815
+ // THEN delete each absorbed — ONLY the staged cluster members (no free-form delete path) (AC4).
816
+ const deleted = [];
817
+ const deleteFailed = [];
818
+ for (const t of targets) {
819
+ try {
820
+ wikiDeletePage(root, t.tier, t.slug);
821
+ deleted.push(t.rel);
822
+ } catch (e) {
823
+ // the survivor is already written (knowledge preserved); a failed delete leaves a harmless leftover,
824
+ // recorded for the operator — it never undoes the merge.
825
+ deleteFailed.push({ target: t.rel, reason: String((e && e.message) || 'delete_failed').split('\n')[0] });
826
+ }
827
+ }
828
+ return { ok: true, survivor: rec.survivor, merged_from: mergedFrom, deleted, deleteFailed };
829
+ }
830
+
831
+ // commit (CONFIRM): for each operator-approved survivor, look up its staged merge and run commitOne. An
832
+ // empty approval is the decline (nothing changes, AC3). Then append the outcome to the audit log.
833
+ function runCommit() {
834
+ const input = readJson(positionalFile());
835
+ const root = installRoot();
836
+ const approved = approvedSurvivors(input);
837
+ const staged = readStaged(root);
838
+ const merged = [];
839
+ const skipped = [];
840
+ for (const ref of approved) {
841
+ const key = String(ref);
842
+ const rec = staged.get(key);
843
+ if (!rec) { skipped.push({ survivor: key, reason: 'not_staged' }); continue; }
844
+ const res = commitOne(root, rec);
845
+ if (res.ok) merged.push({ survivor: res.survivor, merged_from: res.merged_from, deleted: res.deleted, deleteFailed: res.deleteFailed });
846
+ else skipped.push({ survivor: key, reason: res.reason });
847
+ }
848
+ appendLine(path.join(harvestDir(root), AUDIT_FILE), { ts: new Date().toISOString(), op: 'commit', merged: merged.map((m) => m.survivor), skipped });
849
+ return print({ merged, skipped });
850
+ }
851
+
852
+ // ════════════════════════════════════════════════════════════════════════════════
853
+ // DECAY / SUPERSESSION (harvest-04 / H4) — the NON-destructive curation op: ANNOTATE a superseded or
854
+ // orphaned page so Recall + the operator know its status, WITHOUT ever deleting it (provenance survives —
855
+ // Letta eviction-not-delete). Two annotation kinds, each landing as ONE forward-link key in the page
856
+ // FRONTMATTER (the body is never touched):
857
+ // · stale: <missing-source-path> — an orphaned page whose `derived_from:` source FILE is gone. Auto-
858
+ // derived from H2's scanLocal (mechanical — no judgment needed).
859
+ // · superseded_by: <path> — a page replaced by another. A skill/operator JUDGMENT drafted into
860
+ // a proposal file (auto-scan cannot invent the replacement target).
861
+ // Reuses sync-06 / merge's propose→confirm by-reference spine (secret-scan, sha256 integrity, path-
862
+ // confinement) and the sync restampDoc spirit (an in-place single-key frontmatter stamp that preserves all
863
+ // other frontmatter). There is NO delete path here — decay only annotates (delete is H3 alone).
864
+ // ════════════════════════════════════════════════════════════════════════════════
865
+
866
+ // ── the reinforced-exclusion reader (AC4) — read the coalesced recency sidecar ────
867
+ // A day-granular UTC stamp (YYYY-MM-DD) — the reinforce.json grain (recall-surface.cjs dayStamp parity).
868
+ // Injectable clock so the window is deterministic under test.
869
+ function dayStamp(now) {
870
+ const d = now instanceof Date ? now : new Date(now == null ? Date.now() : now);
871
+ return d.toISOString().slice(0, 10);
872
+ }
873
+
874
+ // The window cutoff: today minus REINFORCE_WINDOW_DAYS, as a YYYY-MM-DD string. ISO dates sort lexically,
875
+ // so a string `>=` against this cutoff is the within-window test.
876
+ function cutoffDay(now) {
877
+ const base = now instanceof Date ? new Date(now.getTime()) : new Date(now == null ? Date.now() : now);
878
+ base.setUTCDate(base.getUTCDate() - REINFORCE_WINDOW_DAYS);
879
+ return base.toISOString().slice(0, 10);
880
+ }
881
+
882
+ // The wiki-root-relative join key for a page path: tolerate a leading './', normalize separators, strip the
883
+ // '.wrxn/wiki/' prefix → e.g. 'concepts/foo.md'. IDENTICAL to recall-surface.cjs wikiRelPath (the reinforce
884
+ // sidecar is keyed this way on the writer side — a slug-vs-path mismatch would silently break the exclusion).
885
+ function wikiRelOf(file) {
886
+ const f = String(file || '').replace(/\\/g, '/').replace(/^\.\//, '');
887
+ const i = f.indexOf(WIKI_PREFIX);
888
+ if (i === -1) return null;
889
+ return f.slice(i + WIKI_PREFIX.length) || null;
890
+ }
891
+
892
+ // Read <root>/.wrxn/reinforce.json → the Set of wiki-rel paths surfaced within the window (AC4). Wholly
893
+ // graceful: an absent sidecar (the common case — D2 may not have run yet), a corrupt body, or a non-map
894
+ // shape all yield an EMPTY set (nothing treated as reinforced) and NEVER throw.
895
+ function reinforcedSet(root, now) {
896
+ const out = new Set();
897
+ let raw;
898
+ try {
899
+ raw = fs.readFileSync(path.join(root, REINFORCE_REL), 'utf8');
900
+ } catch {
901
+ return out; // absent sidecar → nothing reinforced (graceful)
902
+ }
903
+ let map;
904
+ try {
905
+ map = JSON.parse(raw);
906
+ } catch {
907
+ return out; // corrupt → nothing reinforced (never clobber, never throw)
908
+ }
909
+ if (!map || typeof map !== 'object' || Array.isArray(map)) return out; // not a map → nothing reinforced
910
+ const cutoff = cutoffDay(now);
911
+ for (const [key, val] of Object.entries(map)) {
912
+ if (typeof val !== 'string') continue;
913
+ const day = val.slice(0, 10);
914
+ if (/^\d{4}-\d{2}-\d{2}$/.test(day) && day >= cutoff) out.add(key); // within the window (inclusive)
915
+ }
916
+ return out;
917
+ }
918
+
919
+ // ── the annotation value sanitiser (write-channel safety, sync's fingerprintProblem analog) ──
920
+ // The value lands VERBATIM as `<key>: <value>` on one frontmatter line — a write channel the page-confine
921
+ // gate doesn't cover. Reject a newline (frontmatter injection), a colon (YAML mapping ambiguity — and a
922
+ // legit stale/superseded_by value is a plain POSIX path with neither), empty, oversize; then secret-scan
923
+ // (a credential must never harden into recalled frontmatter). Returns a problem code or null.
924
+ function annotationValueProblem(value) {
925
+ if (typeof value !== 'string' || !value.trim()) return 'malformed_value';
926
+ if (/[\r\n:]/.test(value)) return 'malformed_value'; // newline → injection; colon → YAML ambiguity
927
+ if (value.length > VALUE_MAX) return 'malformed_value';
928
+ return secretScan(value); // 'contains_secret' (value is a write channel) | null
929
+ }
930
+
931
+ // The integrity fingerprint over the fields that DETERMINE the write (the page target, the annotation key,
932
+ // and the value). Recomputed at the write boundary and compared to the staged value → a record whose
933
+ // page/key/value was altered after staging cannot write (AC2 tamper-refusal). The reason is operator-facing
934
+ // metadata only (never written into the page), so it is deliberately OUT of the hash.
935
+ function decayHash(p) {
936
+ const canon = JSON.stringify({ page: String(p.page || ''), key: String(p.key || ''), value: String(p.value || '') });
937
+ return crypto.createHash('sha256').update(canon).digest('hex');
938
+ }
939
+
940
+ // ── the in-place frontmatter annotation (PURE) — the sync restampDoc spirit ───────
941
+ // True when the page's frontmatter ALREADY carries `<key>:` — the idempotency probe (AC5). No fence → false.
942
+ function hasFrontmatterKey(content, key) {
943
+ const m = /^---\r?\n([\s\S]*?)\r?\n---/.exec(String(content));
944
+ if (!m) return false;
945
+ return new RegExp(`^${key}:\\s*`, 'm').test(m[1]);
946
+ }
947
+
948
+ // Append `<key>: <value>` to the page's frontmatter, preserving every other frontmatter line AND the body
949
+ // BYTE-FOR-BYTE (the body is sliced off the original and re-appended unchanged — decay never rewrites it).
950
+ // A page with no frontmatter fence cannot carry a forward-link → returns null (the caller skips it). The
951
+ // caller MUST hasFrontmatterKey-guard first (idempotency) — this function unconditionally appends.
952
+ function annotateFrontmatter(content, key, value) {
953
+ const src = String(content);
954
+ const m = /^---\r?\n([\s\S]*?)\r?\n---/.exec(src);
955
+ if (!m) return null; // no frontmatter fence → not annotatable
956
+ return `---\n${m[1]}\n${key}: ${value}\n---${src.slice(m[0].length)}`; // m[0] excludes the trailing newline → body verbatim
957
+ }
958
+
959
+ // ── propose-side candidate assembly (PURE-ish: scanLocal reads fs) ────────────────
960
+ // Auto-derive a `stale:` proposal for each ORPHANED decay candidate (H2's scanLocal — the same detection
961
+ // H2 reports, re-run fresh so a since-fixed page never lingers from a stale report). superseded is a
962
+ // JUDGMENT (which page replaced which) auto-scan cannot make → those arrive via the drafted proposal file.
963
+ function autoStaleProposals(root) {
964
+ const { decay } = scanLocal(root);
965
+ const out = [];
966
+ const seen = new Set();
967
+ for (const d of decay) {
968
+ if (d.subtype !== 'orphaned' || seen.has(d.path)) continue; // one stale annotation per page (first missing source)
969
+ seen.add(d.path);
970
+ out.push({ page: d.path, key: 'stale', value: d.missing_source, reason: d.reason });
971
+ }
972
+ return out;
973
+ }
974
+
975
+ // Normalize the OPTIONAL skill-drafted proposal file into a {page,key,value,reason}[] (a single object, an
976
+ // array, or { proposals:[…] }). The skill drafts these from H2's report — chiefly superseded_by judgments.
977
+ function normalizeDraftProposals(input) {
978
+ let arr;
979
+ if (Array.isArray(input)) arr = input;
980
+ else if (input && Array.isArray(input.proposals)) arr = input.proposals;
981
+ else if (isObj(input)) arr = [input];
982
+ else arr = [];
983
+ return arr
984
+ .filter(isObj)
985
+ .map((p) => ({ page: String(p.page || ''), key: String(p.key || ''), value: p.value == null ? '' : String(p.value), reason: p.reason ? String(p.reason) : '' }));
986
+ }
987
+
988
+ // Read .wrxn/harvest/decay-staged.jsonl into a page → staged-record map (last proposed wins). Malformed
989
+ // lines skip. A usable record needs a page, a key, and a string value.
990
+ function readStagedDecay(root) {
991
+ const map = new Map();
992
+ let txt;
993
+ try {
994
+ txt = fs.readFileSync(path.join(root, ...HARVEST_DIR, DECAY_STAGED_FILE), 'utf8');
995
+ } catch {
996
+ return map; // no decay staging trail yet → nothing to confirm by reference
997
+ }
998
+ for (const line of txt.split('\n')) {
999
+ const s = line.trim();
1000
+ if (!s) continue;
1001
+ try {
1002
+ const rec = JSON.parse(s);
1003
+ if (rec && rec.page && rec.key && typeof rec.value === 'string') map.set(rec.page, rec);
1004
+ } catch {
1005
+ /* skip a malformed staging line */
1006
+ }
1007
+ }
1008
+ return map;
1009
+ }
1010
+
1011
+ // Normalize confirm input into the operator-approved PAGE list (["page"…] or { approved:[…] }). An EMPTY
1012
+ // list is the DECLINE — confirm annotates nothing (AC3).
1013
+ function approvedPages(input) {
1014
+ if (Array.isArray(input)) return input.map(String);
1015
+ if (input && typeof input === 'object' && Array.isArray(input.approved)) return input.approved.map(String);
1016
+ return [];
1017
+ }
1018
+
1019
+ // decay propose (STAGE): assemble candidates (auto-scanned orphaned → stale + the optional skill-drafted
1020
+ // proposals), then GATE each — page path-confined to a knowledge tier + present, key allowlisted, value
1021
+ // sanitised + secret-scanned, the page NOT reinforced (AC4), the page NOT already annotated (AC5) — and
1022
+ // record the survivors by-reference with an integrity hash. NEVER touches a knowledge page (mirrors merge's
1023
+ // stage / sync's propose).
1024
+ function runDecayPropose() {
1025
+ const root = installRoot();
1026
+ const fileArg = positionalAfter(4); // optional proposal file (absent → auto-scan only)
1027
+ const drafted = fileArg ? normalizeDraftProposals(readJson(fileArg)) : [];
1028
+
1029
+ // skill-drafted entries override the auto-scanned one for the same page (the operator's judgment wins).
1030
+ const byPage = new Map();
1031
+ for (const p of autoStaleProposals(root)) byPage.set(p.page, p);
1032
+ for (const p of drafted) byPage.set(p.page, p);
1033
+
1034
+ const reinforced = reinforcedSet(root); // wiki-rel paths surfaced within the window
1035
+ const staged = [];
1036
+ const skipped = [];
1037
+ for (const p of byPage.values()) {
1038
+ if (p.key !== 'stale' && p.key !== 'superseded_by') { skipped.push({ page: p.page, reason: 'bad_key' }); continue; }
1039
+ const abs = resolveSafeHarvestDoc(root, p.page);
1040
+ if (!abs || !isKebab(slugOfPath(p.page))) { skipped.push({ page: p.page, reason: 'unsafe_page' }); continue; }
1041
+ const vp = annotationValueProblem(p.value);
1042
+ if (vp) { skipped.push({ page: p.page, reason: vp }); continue; }
1043
+ let content;
1044
+ try {
1045
+ content = fs.readFileSync(abs, 'utf8');
1046
+ } catch {
1047
+ skipped.push({ page: p.page, reason: 'missing_page' }); continue; // annotate only an existing page
1048
+ }
1049
+ const rel = wikiRelOf(p.page);
1050
+ if (rel && reinforced.has(rel)) { skipped.push({ page: p.page, reason: 'reinforced' }); continue; } // AC4: live knowledge is never flagged
1051
+ if (hasFrontmatterKey(content, p.key)) { skipped.push({ page: p.page, reason: 'already_annotated' }); continue; } // AC5
1052
+ staged.push({ page: p.page, tier: tierOfPath(p.page), slug: slugOfPath(p.page), key: p.key, value: p.value, reason: p.reason || '', hash: decayHash(p) });
1053
+ }
1054
+
1055
+ const dir = harvestDir(root);
1056
+ const ts = new Date().toISOString();
1057
+ const stagedFile = path.join(dir, DECAY_STAGED_FILE);
1058
+ for (const rec of staged) appendLine(stagedFile, Object.assign({ ts, op: 'decay-propose' }, rec));
1059
+ appendLine(path.join(dir, AUDIT_FILE), { ts, op: 'decay-propose', staged: staged.map((s) => ({ page: s.page, key: s.key, value: s.value })), skipped });
1060
+ return print({ staged, skipped, stagedFile: path.relative(root, stagedFile) });
1061
+ }
1062
+
1063
+ // decayConfirmOne: the write-boundary re-gate for ONE approved page. RE-VALIDATE (key → value sanitise +
1064
+ // secret-scan → integrity → path-confine + kebab → page present + not a symlink → not already annotated)
1065
+ // BEFORE any write, so a tampered/seeded/declined record cannot write (AC2). Then — and only then — the
1066
+ // in-place single-key frontmatter stamp (body preserved verbatim). NEVER deletes. Returns { ok, … }.
1067
+ function decayConfirmOne(root, rec) {
1068
+ if (rec.key !== 'stale' && rec.key !== 'superseded_by') return { ok: false, reason: 'bad_key' };
1069
+ const vp = annotationValueProblem(rec.value); // re-sanitise + re-secret-scan at the write boundary
1070
+ if (vp) return { ok: false, reason: vp };
1071
+ if (decayHash(rec) !== rec.hash) return { ok: false, reason: 'integrity_mismatch' }; // tamper → refuse
1072
+ const abs = resolveSafeHarvestDoc(root, rec.page);
1073
+ if (!abs || !isKebab(slugOfPath(rec.page))) return { ok: false, reason: 'unsafe_page' };
1074
+ let lst;
1075
+ try {
1076
+ lst = fs.lstatSync(abs);
1077
+ } catch {
1078
+ return { ok: false, reason: 'missing_page' }; // the page vanished since staging
1079
+ }
1080
+ if (lst.isSymbolicLink()) return { ok: false, reason: 'symlink_page' }; // refuse following a planted symlink out of the tree
1081
+ let content;
1082
+ try {
1083
+ content = fs.readFileSync(abs, 'utf8');
1084
+ } catch {
1085
+ return { ok: false, reason: 'missing_page' };
1086
+ }
1087
+ // AC4 re-check at the write boundary: a page that became reinforced (surfaced by Recall) in the
1088
+ // propose→confirm window is live knowledge and must never be flagged stale/superseded. The integrity
1089
+ // hash covers page/key/value only, so it cannot catch a reinforce-state change — re-read the sidecar,
1090
+ // mirroring the propose-side reinforcedSet check.
1091
+ const rel = wikiRelOf(rec.page);
1092
+ if (rel && reinforcedSet(root).has(rel)) return { ok: false, reason: 'reinforced' };
1093
+ if (hasFrontmatterKey(content, rec.key)) return { ok: false, reason: 'already_annotated' }; // AC5: idempotent no-op, no churn
1094
+ const next = annotateFrontmatter(content, rec.key, rec.value);
1095
+ if (next == null) return { ok: false, reason: 'no_frontmatter' };
1096
+ fs.writeFileSync(abs, next); // the in-place annotation — body byte-identical, page never deleted
1097
+ return { ok: true, page: rec.page, key: rec.key, value: rec.value };
1098
+ }
1099
+
1100
+ // decay confirm (COMMIT-by-reference): for each operator-approved page, look up its staged decay record and
1101
+ // run decayConfirmOne. An empty approval is the decline (nothing changes, AC3). Then audit the outcome.
1102
+ function runDecayConfirm() {
1103
+ const fileArg = positionalAfter(4);
1104
+ if (!fileArg) fail('decay confirm needs <approved.json> — the page path(s) the operator confirms');
1105
+ const input = readJson(fileArg);
1106
+ const root = installRoot();
1107
+ const approved = approvedPages(input);
1108
+ const staged = readStagedDecay(root);
1109
+ const annotated = [];
1110
+ const skipped = [];
1111
+ for (const ref of approved) {
1112
+ const key = String(ref);
1113
+ const rec = staged.get(key);
1114
+ if (!rec) { skipped.push({ page: key, reason: 'not_staged' }); continue; }
1115
+ const res = decayConfirmOne(root, rec);
1116
+ if (res.ok) annotated.push({ page: res.page, key: res.key, value: res.value });
1117
+ else skipped.push({ page: key, reason: res.reason });
1118
+ }
1119
+ appendLine(path.join(harvestDir(root), AUDIT_FILE), { ts: new Date().toISOString(), op: 'decay-confirm', annotated: annotated.map((a) => a.page), skipped });
1120
+ return print({ annotated, skipped });
1121
+ }
1122
+
1123
+ async function main() {
1124
+ const cmd = process.argv[2];
1125
+ switch (cmd) {
1126
+ case 'check':
1127
+ return runCheck();
1128
+ case 'stage':
1129
+ return runStage();
1130
+ case 'commit':
1131
+ return runCommit();
1132
+ case 'decay': {
1133
+ const sub = process.argv[3];
1134
+ if (sub === 'propose') return runDecayPropose();
1135
+ if (sub === 'confirm') return runDecayConfirm();
1136
+ process.stdout.write('Usage: node .wrxn/harvest.cjs decay <propose [proposal.json] | confirm <approved.json>> [--root <dir>]\n');
1137
+ return process.exit(2); // a bare/unknown sub-verb is an incomplete command, not a help request
1138
+ }
1139
+ default:
1140
+ process.stdout.write('Usage: node .wrxn/harvest.cjs <check [--root <dir>] | stage <proposal.json> | commit <approved.json> | decay <propose [proposal.json]|confirm <approved.json>>> [--root <dir>]\n');
1141
+ process.exit(cmd ? 2 : 0);
1142
+ }
1143
+ }
1144
+
1145
+ if (require.main === module) {
1146
+ main().catch((err) => fail(err && err.message ? err.message : 'unexpected error'));
1147
+ }
1148
+
1149
+ module.exports = {
1150
+ // pure
1151
+ lintPage,
1152
+ parseDerivedFrom,
1153
+ parseSupersededBy,
1154
+ scanLocal,
1155
+ isProse,
1156
+ nearDupQualifies,
1157
+ clusterNearDups,
1158
+ assembleRecords,
1159
+ tierOfPath,
1160
+ isHarvestPath,
1161
+ // io
1162
+ nearDupFromDoor,
1163
+ check,
1164
+ discoverEndpoint,
1165
+ httpTransport,
1166
+ pidAlive,
1167
+ findInstallRoot,
1168
+ // merge (harvest-03) — pure gate primitives
1169
+ secretScan,
1170
+ resolveSafeHarvestDoc,
1171
+ mergeHash,
1172
+ descriptionProblem,
1173
+ composeSurvivor,
1174
+ isKebab,
1175
+ // decay (harvest-04) — pure gate + annotation primitives
1176
+ reinforcedSet,
1177
+ dayStamp,
1178
+ cutoffDay,
1179
+ wikiRelOf,
1180
+ annotationValueProblem,
1181
+ decayHash,
1182
+ hasFrontmatterKey,
1183
+ annotateFrontmatter,
1184
+ autoStaleProposals,
1185
+ // constants
1186
+ HARVEST_TIERS,
1187
+ NEAR_DUP_THRESHOLD,
1188
+ FIND_PATH,
1189
+ REINFORCE_WINDOW_DAYS,
1190
+ };