mlx-bun 0.0.8 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -20
- package/docs/design/batching-perf-path.md +232 -0
- package/docs/design/batching-v2-plan.md +515 -0
- package/docs/design/bucketing-stage.md +196 -0
- package/docs/design/compat-cli-surface-design.md +11 -11
- package/docs/design/docs-reorg-plan.md +1 -1
- package/docs/design/dreaming-nightly-pipeline.md +291 -0
- package/docs/design/dspark-speculative-decoding.md +106 -0
- package/docs/design/generic-model-support.md +381 -0
- package/docs/design/memory-inference-path.md +422 -0
- package/docs/design/memory-synthesis.md +287 -0
- package/docs/design/minicpm5-decode-megakernel.md +484 -0
- package/docs/design/mlx-lm-tool-parity-plan.md +535 -0
- package/docs/design/omlx-adoption-map.md +90 -0
- package/docs/design/orpo-dynamic-lambda.md +41 -0
- package/docs/design/orpo-training.md +35 -5
- package/docs/design/parallel-slots.md +12 -1
- package/docs/design/parity-tier-dag.md +13 -5
- package/docs/design/segmented-backward-training.md +15 -0
- package/docs/design/spec-decode-larger-targets.md +229 -0
- package/docs/design/ssd-kv-cold-tier.md +158 -0
- package/docs/design/tauri-desktop-app.md +2 -2
- package/docs/design/the-dreaming-handoff.md +141 -0
- package/docs/design/the-dreaming-master-plan.md +446 -0
- package/docs/design/write-pipeline-entity.md +468 -0
- package/docs/investigations/curve-bisector-routing.md +1 -1
- package/docs/investigations/decode-roofline-lookagain.md +348 -0
- package/docs/investigations/dspark-handoff.md +58 -0
- package/docs/investigations/kernel-perf-review-2026-07.md +171 -0
- package/docs/planning/PRODUCT_ROADMAP.md +2 -2
- package/docs/planning/curve-sampler-research-plan.md +331 -0
- package/docs/planning/memory-docs-and-dag-plan.md +185 -0
- package/docs/planning/pi-integration-review.md +178 -0
- package/docs/planning/release-notes-v0.0.10.md +57 -0
- package/docs/planning/release-notes-v0.0.9.md +122 -0
- package/docs/planning/web-ui-pass-plan.md +213 -0
- package/docs/planning/website-readme-pass-plan.md +224 -0
- package/docs/reference/cli.md +255 -0
- package/docs/reference/embedding.md +7 -2
- package/docs/reference/library-api.md +22 -3
- package/docs/reference/memory.md +5 -5
- package/docs/reference/models.md +137 -0
- package/docs/reference/orpo-quickstart.md +6 -0
- package/docs/reference/server-api.md +187 -9
- package/docs/reference/server-config.md +43 -9
- package/docs/reference/training.md +31 -12
- package/package.json +1 -1
- package/src/anthropic.ts +22 -0
- package/src/assets/curve-designer.html +1 -1
- package/src/chat-template.ts +25 -7
- package/src/cli.ts +1077 -90
- package/src/config.ts +4 -1
- package/src/download.ts +99 -8
- package/src/embed.ts +77 -0
- package/src/eval/perplexity.ts +164 -0
- package/src/eval/runner.ts +15 -6
- package/src/eval/tasks/bfcl.ts +1 -1
- package/src/generate.ts +149 -18
- package/src/harness-pi.ts +43 -17
- package/src/index.ts +2 -0
- package/src/kv-store.ts +240 -98
- package/src/lora.ts +2 -2
- package/src/memory/article.ts +558 -0
- package/src/memory/chunk-validation.ts +150 -0
- package/src/memory/chunk.ts +305 -12
- package/src/memory/cluster.ts +344 -17
- package/src/memory/coverage.ts +134 -0
- package/src/memory/crosslink.ts +0 -0
- package/src/memory/db.ts +184 -45
- package/src/memory/entity.ts +203 -0
- package/src/memory/gate.ts +187 -0
- package/src/memory/history.ts +152 -0
- package/src/memory/ingest.ts +324 -11
- package/src/memory/model.ts +321 -0
- package/src/memory/normalize.ts +671 -0
- package/src/memory/parse.ts +22 -0
- package/src/memory/pipeline.ts +242 -30
- package/src/memory/prompts.ts +42 -0
- package/src/memory/query.ts +334 -0
- package/src/memory/reconcile.ts +734 -0
- package/src/memory/redirect.ts +108 -0
- package/src/memory/reindex.ts +325 -0
- package/src/memory/resolve.ts +307 -0
- package/src/memory/route.ts +272 -0
- package/src/memory/schedule.ts +6 -4
- package/src/memory/stages.ts +774 -0
- package/src/memory/synthesize.ts +1629 -20
- package/src/memory/tools.ts +446 -96
- package/src/memory/vault.ts +462 -35
- package/src/memory/wikify.ts +1057 -16
- package/src/mlx/array.ts +21 -0
- package/src/mlx/ffi.ts +3 -0
- package/src/mlx/ops.ts +72 -0
- package/src/model/compiled-decode.ts +43 -8
- package/src/model/factory.ts +29 -8
- package/src/model/flash-attention.ts +4 -2
- package/src/model/fused-decode-kernel.ts +14 -7
- package/src/model/fused-mlp-kernel.ts +169 -0
- package/src/model/fused-swiglu-kernel.ts +240 -0
- package/src/model/gemma4-base.ts +96 -7
- package/src/model/gemma4.ts +33 -2
- package/src/model/minicpm5.ts +106 -8
- package/src/model/qwen3-delta.ts +48 -1
- package/src/model/qwen3.ts +276 -0
- package/src/model/qwen3_5.ts +11 -2
- package/src/model/steel-linear-kernel.ts +96 -0
- package/src/model/support.ts +38 -8
- package/src/model/universal/archs.ts +332 -0
- package/src/model/universal/dense.ts +576 -0
- package/src/model/universal/modules.ts +163 -0
- package/src/model/universal/rope.ts +366 -0
- package/src/pi-provider.ts +1 -1
- package/src/pi-terminal.ts +29 -8
- package/src/pi-web.ts +19 -2
- package/src/prompt-cache.ts +55 -9
- package/src/quantize/config-writer.ts +1 -1
- package/src/registry.ts +219 -8
- package/src/responses.ts +23 -0
- package/src/sampler.ts +220 -9
- package/src/serve/batch-scheduler.ts +307 -83
- package/src/serve/generation-gateway.ts +156 -29
- package/src/server.ts +746 -73
- package/src/spec/drafter.ts +81 -49
- package/src/spec/dspark/data-dflash.ts +156 -0
- package/src/spec/dspark/data.ts +201 -0
- package/src/spec/dspark/generate-dflash.ts +193 -0
- package/src/spec/dspark/generate.ts +290 -0
- package/src/spec/dspark/loss.ts +213 -0
- package/src/spec/dspark/module-dflash.ts +361 -0
- package/src/spec/dspark/module.ts +540 -0
- package/src/spec/dspark/sample.ts +112 -0
- package/src/spec/generate.ts +40 -19
- package/src/ssd-cache.ts +213 -0
- package/src/status-page.html +1 -1
- package/src/tokenizer.ts +7 -0
- package/src/train/flash-cce.ts +81 -23
- package/src/train/fuse.ts +243 -0
- package/src/train/index.ts +3 -0
- package/src/train/job.ts +8 -0
- package/src/train/loss.ts +341 -40
- package/src/train/prefix-shared.ts +62 -11
- package/src/train/segmented.ts +215 -55
- package/src/train/trainer.ts +112 -38
- package/src/web/skills/memory/SKILL.md +40 -17
package/src/memory/synthesize.ts
CHANGED
|
@@ -1,29 +1,1638 @@
|
|
|
1
|
-
// mlx-bun memory —
|
|
1
|
+
// mlx-bun memory — SYNTHESIZE / CREATE flow (P6-T3).
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
// Mirrors lucien's synthesize-dispatch.ts / synthesize-update.ts.
|
|
3
|
+
// The write pipeline's cold-start branch: turn an entity's accumulated chunks
|
|
4
|
+
// into a fresh Wikipedia-shaped article. Entered ONLY when the P1-T5/P5-T4
|
|
5
|
+
// CREATE rule fired for the entity (≥3 routed chunks, or ≥1 substantive
|
|
6
|
+
// ownership/decision chunk). The complementary UPDATE branch (fold one chunk
|
|
7
|
+
// into an existing section) is P7's section-patch — not built here.
|
|
9
8
|
//
|
|
10
|
-
//
|
|
9
|
+
// The decomposition is the whole point (capacity-by-decomposition): we NEVER
|
|
10
|
+
// hold a whole article in the model's head. CREATE is
|
|
11
|
+
// sub-cluster → OUTLINE (a clean TOC of single-topic sections) → per-section
|
|
12
|
+
// DRAFT (each section drafted from empty, the SAME bounded op as a patch) →
|
|
13
|
+
// seed INFOBOX → assemble.
|
|
14
|
+
// Sharp section structure is the primary quality goal: the read path's
|
|
15
|
+
// TOC→section reads ride this outline.
|
|
16
|
+
//
|
|
17
|
+
// Citations are deterministic by construction (AI does judgment, CODE enforces
|
|
18
|
+
// invariants): we assign each source conversation a stable `[^N]` up front and
|
|
19
|
+
// generate the `## References` `[^N]:` definition lines ourselves in the fixed
|
|
20
|
+
// Lucien standard — `[^N]: `conv:HASH` (YYYY-MM-DD, source) — desc`, HASH = the
|
|
21
|
+
// first 8 lowercase hex of the conversation UUID with hyphens stripped. The
|
|
22
|
+
// model only PLACES the markers; it never writes a definition or invents a hash.
|
|
23
|
+
// This sidesteps the most common base-model defect (malformed/fabricated
|
|
24
|
+
// footnotes) entirely.
|
|
25
|
+
//
|
|
26
|
+
// Citation/wikilink rules are ported from lucien `scripts/synthesize.ts`
|
|
27
|
+
// (SYNTHESIS_PROMPT_BOOTSTRAP). Every CREATE output is run through the P6-T1
|
|
28
|
+
// gate (a weak pass → NO-OP, vault unchanged) and the P6-T2 NORMALIZE pass, then
|
|
29
|
+
// committed via `commitVault`; the integrated (chunk, section) edges are written
|
|
30
|
+
// to the `synthesized_chunk_sections` ledger.
|
|
31
|
+
|
|
32
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
33
|
+
import { join } from "node:path";
|
|
34
|
+
|
|
35
|
+
import {
|
|
36
|
+
articleStructure,
|
|
37
|
+
fenceLabel,
|
|
38
|
+
isFenceLine,
|
|
39
|
+
parseInfobox,
|
|
40
|
+
repairFences,
|
|
41
|
+
type EntityKind,
|
|
42
|
+
} from "./article";
|
|
43
|
+
import { LEAD_ANCHOR } from "./cluster";
|
|
44
|
+
import type { MemoryStore } from "./db";
|
|
45
|
+
import { checkFootnoteIntegrity, gateEdit, type GateVerdict } from "./gate";
|
|
46
|
+
import {
|
|
47
|
+
buildContradictionClassifierPrompt,
|
|
48
|
+
classifyContradiction,
|
|
49
|
+
type ContradictionKind,
|
|
50
|
+
} from "./history";
|
|
51
|
+
import { callLocal, MAX_OUTPUT_TOKENS } from "./model";
|
|
52
|
+
import { hasStubMarker, normalizeArticle } from "./normalize";
|
|
53
|
+
import { parseLines } from "./parse";
|
|
54
|
+
import { loadMetaPolicy } from "./prompts";
|
|
55
|
+
import {
|
|
56
|
+
articlesDir,
|
|
57
|
+
commitVault,
|
|
58
|
+
extractSection,
|
|
59
|
+
listArticles,
|
|
60
|
+
parseToc,
|
|
61
|
+
resolveWikilinkToStem,
|
|
62
|
+
vaultRoot,
|
|
63
|
+
} from "./vault";
|
|
64
|
+
|
|
65
|
+
// ---- ported citation / wikilink rules (lucien SYNTHESIS_PROMPT_BOOTSTRAP) ----
|
|
66
|
+
|
|
67
|
+
/** Use ONLY content from the provided chunk(s); never copy names/values from these instructions. */
|
|
68
|
+
const NO_COPY_GUARD = `Use ONLY content from the provided chunk(s)/source notes below; never copy any name, value, or example from these instructions into the article.`;
|
|
69
|
+
|
|
70
|
+
/** Wikilink contract: exact stem, underscores, never the spaced phantom form. */
|
|
71
|
+
const WIKILINK_RULES = `WIKILINKS — link to OTHER ARTICLES below using the exact article stem, underscores not spaces: [[Exact_Stem]] or with display text [[Exact_Stem|display text]]. The spaced form [[Exact Stem]] resolves to nothing and creates a broken orphan link; it is forbidden. Only link to a stem that appears in OTHER ARTICLES.`;
|
|
72
|
+
|
|
73
|
+
/** Citation contract: place the GIVEN markers, never write defs or hashes. */
|
|
74
|
+
const CITATION_RULES = `CITATIONS — cite a substantive claim by placing one of the GIVEN footnote markers ([^1], [^2], …) immediately after it, e.g. "<claim>.[^1]". Two sources for one sentence: adjacent markers, no space — [^1][^2]. Reuse the same marker every time you cite that source. Do NOT write any footnote DEFINITION line, do NOT invent or guess a hash, do NOT emit a ## References section — the definitions are generated for you. NEVER fabricate a citation: an uncited true statement is fine, but only ever use a marker FROM THE LIST ABOVE — never a placeholder letter or any marker not in that list. FORBIDDEN — never emit HTML <sup>, #ref-/#cite- links, escaped [\\[1\\]] markers, or [[1]] wikilink markers; [^N] is the ONLY citation syntax.`;
|
|
75
|
+
|
|
76
|
+
/** Date contract (CREATE): every source note is tagged with its date. The model
|
|
77
|
+
* must treat the LATER-DATED note as the user's CURRENT position when notes
|
|
78
|
+
* conflict — chronology is a first-class signal, not just processing order. Kept
|
|
79
|
+
* schematic (no copyable concrete value) per the small-model parroting failure. */
|
|
80
|
+
const DATE_RESOLVE_RULES = `DATES — each source note below is tagged with its date in [YYYY-MM-DD] form (the same date that appears in its [^N] footnote). When two notes CONFLICT (a changed favourite, a reversed preference, an updated fact), the LATER-DATED note is the user's CURRENT position: resolve toward it. Write a superseded value as a past stage ("originally …, now …"), never as still-true.`;
|
|
81
|
+
|
|
82
|
+
// ---- conv:HASH + footnote scaffolding --------------------------------------
|
|
83
|
+
|
|
84
|
+
/** A filesystem-safe bare article stem from a free-form entity name: collapse
|
|
85
|
+
* whitespace to `_`, map path separators (a value containing `/`) and other
|
|
86
|
+
* filename-hostile characters to `-`, and strip leading dots, so a model-minted
|
|
87
|
+
* name like `<value>/<value>` yields a bare `<value>-<value>` stem (never a
|
|
88
|
+
* `articles/…` subpath). Stems with no separators are unchanged. The original
|
|
89
|
+
* surface survives as an `aliases:` entry, so name resolution still reaches the
|
|
90
|
+
* article. */
|
|
91
|
+
export function entityStem(entity: string): string {
|
|
92
|
+
const stem = entity
|
|
93
|
+
.trim()
|
|
94
|
+
.replace(/[/\\]+/g, "-")
|
|
95
|
+
.replace(/\s+/g, "_")
|
|
96
|
+
.replace(/[<>:"|?*]+/g, "")
|
|
97
|
+
.replace(/_+/g, "_")
|
|
98
|
+
.replace(/^\.+/, "")
|
|
99
|
+
.replace(/^[-_]+|[-_]+$/g, "");
|
|
100
|
+
return stem || "entity";
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** The fixed Lucien `conv:HASH`: the first 8 lowercase hex digits of the
|
|
104
|
+
* conversation UUID with hyphens stripped (`<8hex>-<4hex>-…` → `<8hex>`). An
|
|
105
|
+
* already-8-char hex id passes through unchanged. */
|
|
106
|
+
export function convHash(conv: string): string {
|
|
107
|
+
return conv.toLowerCase().replace(/[^0-9a-f]/g, "").slice(0, 8);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** `YYYY-MM-DD` (UTC) from an epoch-millis timestamp; `undated` when missing. */
|
|
111
|
+
export function footnoteDate(ms: number | null | undefined): string {
|
|
112
|
+
if (ms == null || !Number.isFinite(ms)) return "undated";
|
|
113
|
+
return new Date(ms).toISOString().slice(0, 10);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** The fixed Lucien footnote definition line for a source conversation as
|
|
117
|
+
* `[^n]: `conv:HASH` (YYYY-MM-DD, source)`. The single source of truth both
|
|
118
|
+
* CREATE (buildFootnoteMap) and PATCH (appendReferenceDef) generate defs from,
|
|
119
|
+
* so the model never writes — and cannot malform — a definition line. */
|
|
120
|
+
export function footnoteDefLine(n: number, conv: string, title: string, dateMs: number | null): string {
|
|
121
|
+
const source = (title ?? "").trim() || "conversation";
|
|
122
|
+
return `[^${n}]: \`conv:${convHash(conv)}\` (${footnoteDate(dateMs)}, ${source})`;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** One source-conversation footnote: its stable per-article number, hash, and
|
|
126
|
+
* the generated `[^N]: `conv:HASH` (date, source)` definition line. */
|
|
127
|
+
export interface FootnoteEntry {
|
|
128
|
+
conv: string;
|
|
129
|
+
n: number;
|
|
130
|
+
hash: string;
|
|
131
|
+
source: string;
|
|
132
|
+
def: string;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Assign each distinct source conversation a stable `[^N]` (first-appearance
|
|
136
|
+
* order across the entity's chunks) and pre-generate its definition line in the
|
|
137
|
+
* fixed Lucien standard. The model places these markers; it never writes defs. */
|
|
138
|
+
export function buildFootnoteMap(chunks: SynthesisChunk[]): {
|
|
139
|
+
byConv: Map<string, FootnoteEntry>;
|
|
140
|
+
entries: FootnoteEntry[];
|
|
141
|
+
} {
|
|
142
|
+
const byConv = new Map<string, FootnoteEntry>();
|
|
143
|
+
const entries: FootnoteEntry[] = [];
|
|
144
|
+
for (const c of chunks) {
|
|
145
|
+
if (byConv.has(c.conv)) continue;
|
|
146
|
+
const n = entries.length + 1;
|
|
147
|
+
const hash = convHash(c.conv);
|
|
148
|
+
const source = (c.title ?? "").trim() || "conversation";
|
|
149
|
+
const def = footnoteDefLine(n, c.conv, c.title, c.dateMs);
|
|
150
|
+
const entry: FootnoteEntry = { conv: c.conv, n, hash, source, def };
|
|
151
|
+
byConv.set(c.conv, entry);
|
|
152
|
+
entries.push(entry);
|
|
153
|
+
}
|
|
154
|
+
return { byConv, entries };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// ---- chunk input -----------------------------------------------------------
|
|
158
|
+
|
|
159
|
+
/** One source chunk for synthesis: its id, conversation, reassembled text, and
|
|
160
|
+
* the metadata the footnote line needs. Text is NOT stored — the caller pulls
|
|
161
|
+
* it from `store.chunkText(id)`. */
|
|
162
|
+
export interface SynthesisChunk {
|
|
163
|
+
id: string;
|
|
164
|
+
conv: string;
|
|
165
|
+
label: string | null;
|
|
166
|
+
text: string;
|
|
167
|
+
title: string;
|
|
168
|
+
dateMs: number | null;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// ---- deterministic tokenizer (sub-clustering + see-also) -------------------
|
|
172
|
+
|
|
173
|
+
const TOK_STOP = new Set([
|
|
174
|
+
"the", "a", "an", "and", "or", "of", "to", "for", "in", "on", "with", "is",
|
|
175
|
+
"vs", "and", "its", "my", "your", "our", "their",
|
|
176
|
+
]);
|
|
177
|
+
|
|
178
|
+
function tokens(s: string): Set<string> {
|
|
179
|
+
const out = new Set<string>();
|
|
180
|
+
for (const t of s.toLowerCase().split(/[^a-z0-9]+/)) {
|
|
181
|
+
if (t.length > 1 && !TOK_STOP.has(t)) out.add(t);
|
|
182
|
+
}
|
|
183
|
+
return out;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// ---- OUTLINE ---------------------------------------------------------------
|
|
187
|
+
|
|
188
|
+
const GENERIC_SECTION = new Set(["overview", "introduction", "summary", "details", "misc", "other"]);
|
|
189
|
+
|
|
190
|
+
/** Parse a model OUTLINE into clean single-topic section titles: strip bullet /
|
|
191
|
+
* heading / numbering markup, drop empties + bare generics, dedupe by anchor,
|
|
192
|
+
* cap at `max`. */
|
|
193
|
+
export function parseOutline(raw: string, max = 6): string[] {
|
|
194
|
+
const seen = new Set<string>();
|
|
195
|
+
const out: string[] = [];
|
|
196
|
+
for (const line of parseLines(raw)) {
|
|
197
|
+
const title = line
|
|
198
|
+
.replace(/^#{1,6}\s+/, "")
|
|
199
|
+
.replace(/^[-*+]\s+/, "")
|
|
200
|
+
.replace(/^\d+[.)]\s+/, "")
|
|
201
|
+
.replace(/[`*_]/g, "")
|
|
202
|
+
.trim();
|
|
203
|
+
if (!title) continue;
|
|
204
|
+
const key = title.toLowerCase();
|
|
205
|
+
if (seen.has(key)) continue;
|
|
206
|
+
seen.add(key);
|
|
207
|
+
out.push(title);
|
|
208
|
+
if (out.length >= max) break;
|
|
209
|
+
}
|
|
210
|
+
// An outline that is ONLY a generic single section is fine (→ stub); an empty
|
|
211
|
+
// outline falls back to one "Overview" so we always draft something.
|
|
212
|
+
if (out.length === 0) return ["Overview"];
|
|
213
|
+
return out;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** Anchor (heading slug) for a section title — must match vault.ts slugifyHeading
|
|
217
|
+
* so the read path's TOC→section anchors line up. */
|
|
218
|
+
export function sectionAnchor(title: string): string {
|
|
219
|
+
return title
|
|
220
|
+
.trim()
|
|
221
|
+
.toLowerCase()
|
|
222
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
223
|
+
.replace(/^-+|-+$/g, "");
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Assign each chunk to the single best-matching section by label/text token
|
|
228
|
+
* overlap (deterministic, GPU-free), so each section drafts only from its own
|
|
229
|
+
* sub-topic chunks and the (chunk → section) edge is recorded. A chunk with no
|
|
230
|
+
* overlap lands in the first section; sections that win no chunk are dropped by
|
|
231
|
+
* the caller so we never draft an empty section.
|
|
232
|
+
*/
|
|
233
|
+
export function subClusterChunks(
|
|
234
|
+
chunks: SynthesisChunk[],
|
|
235
|
+
sections: string[],
|
|
236
|
+
): Map<string, SynthesisChunk[]> {
|
|
237
|
+
const byAnchor = new Map<string, SynthesisChunk[]>();
|
|
238
|
+
const sectionToks = sections.map((t) => ({ title: t, anchor: sectionAnchor(t), toks: tokens(t) }));
|
|
239
|
+
for (const c of chunks) {
|
|
240
|
+
const ctoks = tokens(`${c.label ?? ""} ${c.text.slice(0, 400)}`);
|
|
241
|
+
let best = sectionToks[0]!;
|
|
242
|
+
let bestScore = -1;
|
|
243
|
+
for (const s of sectionToks) {
|
|
244
|
+
let score = 0;
|
|
245
|
+
for (const t of s.toks) if (ctoks.has(t)) score++;
|
|
246
|
+
if (score > bestScore) {
|
|
247
|
+
bestScore = score;
|
|
248
|
+
best = s;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
const list = byAnchor.get(best.anchor) ?? [];
|
|
252
|
+
list.push(c);
|
|
253
|
+
byAnchor.set(best.anchor, list);
|
|
254
|
+
}
|
|
255
|
+
return byAnchor;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// ---- prompt builders -------------------------------------------------------
|
|
259
|
+
|
|
260
|
+
/** A `getOtherArticles`-style stem list (lucien synthesize.ts) so the model's
|
|
261
|
+
* [[wikilinks]] resolve to real articles. Capped to keep the prefill bounded. */
|
|
262
|
+
function otherArticlesBlock(stems: Iterable<string>, selfStem: string, cap = 200): string {
|
|
263
|
+
const list = [...stems].filter((s) => s !== selfStem).sort((a, b) => a.localeCompare(b)).slice(0, cap);
|
|
264
|
+
return list.length ? list.map((s) => `- ${s}`).join("\n") : "(none yet)";
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** Compact source block for the OUTLINE/INFOBOX/LEAD calls: each chunk DATE-TAGGED
|
|
268
|
+
* (so the model knows which note is most recent), its label + a short text excerpt
|
|
269
|
+
* (the model reads the gist, not the whole transcript). The `[YYYY-MM-DD]` tag is
|
|
270
|
+
* the same date carried in the chunk's `[^N]` footnote. */
|
|
271
|
+
function sourceDigest(chunks: SynthesisChunk[], perChunk = 600): string {
|
|
272
|
+
return chunks
|
|
273
|
+
.map(
|
|
274
|
+
(c, i) =>
|
|
275
|
+
`[${i + 1}] [${footnoteDate(c.dateMs)}] ${c.label ?? "(chunk)"}\n${c.text.slice(0, perChunk).trim()}`,
|
|
276
|
+
)
|
|
277
|
+
.join("\n\n");
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
function buildOutlinePrompt(entity: string, chunks: SynthesisChunk[], policy: string): string {
|
|
281
|
+
return (
|
|
282
|
+
`${policy}\n\n${NO_COPY_GUARD}\n\n` +
|
|
283
|
+
`You are outlining a personal-wiki article about "${entity}". Below are the source notes (chunk label + excerpt). ` +
|
|
284
|
+
`Propose a clean table of contents: 2 to 6 section headings, each a single sub-topic, in natural reading order. ` +
|
|
285
|
+
`Each heading is 2–5 words, Title Case, specific (not "Overview"/"Details"). Output ONE heading per line, nothing else, NONE if there is only one sub-topic.\n\n` +
|
|
286
|
+
`SOURCE NOTES:\n${sourceDigest(chunks)}\n`
|
|
287
|
+
);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
function buildLeadPrompt(
|
|
291
|
+
entity: string,
|
|
292
|
+
chunks: SynthesisChunk[],
|
|
293
|
+
footnotes: Map<string, FootnoteEntry>,
|
|
294
|
+
otherArticles: string,
|
|
295
|
+
policy: string,
|
|
296
|
+
): string {
|
|
297
|
+
return (
|
|
298
|
+
`${policy}\n\n${NO_COPY_GUARD}\n\n${WIKILINK_RULES}\n\n${CITATION_RULES}\n\n${DATE_RESOLVE_RULES}\n\n` +
|
|
299
|
+
`${footnoteList(chunks, footnotes)}\n\n` +
|
|
300
|
+
`OTHER ARTICLES (link to these stems only):\n${otherArticles}\n\n` +
|
|
301
|
+
`Write the LEAD for a personal-wiki article about "${entity}": 2–4 sentences, present tense, naming **${entity}** in bold in the first sentence, describing what it is AND how the user relates to it (uses / owns / chose / considered). Write in the THIRD PERSON, Wikipedia-style — NEVER the first person ("I", "my"); refer to the user in the third person ("the user"). When the notes disagree, frame the user's CURRENT position from the LATEST-DATED note. Output ONLY the lead prose — no heading, no list, no References.\n\n` +
|
|
302
|
+
`SOURCE NOTES:\n${sourceDigest(chunks)}\n`
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function buildInfoboxPrompt(
|
|
307
|
+
entity: string,
|
|
308
|
+
kind: EntityKind,
|
|
309
|
+
chunks: SynthesisChunk[],
|
|
310
|
+
otherArticles: string,
|
|
311
|
+
policy: string,
|
|
312
|
+
): string {
|
|
313
|
+
return (
|
|
314
|
+
`${policy}\n\n${NO_COPY_GUARD}\n\n${WIKILINK_RULES}\n\n` +
|
|
315
|
+
`Produce the INFOBOX facts for a personal-wiki article about "${entity}". Output ONLY \`key: value\` lines, one per line, no fence, no prose. Rules:\n` +
|
|
316
|
+
`- snake_case keys.\n` +
|
|
317
|
+
`- A value that names another entity is a [[wikilink]] to a stem in OTHER ARTICLES (e.g. \`<entity_key>: [[Linked Entity]]\`).\n` +
|
|
318
|
+
`- Include \`type:\` (the emergent label — a short free-form noun phrase) when clear.\n` +
|
|
319
|
+
`- Include world-facts only if the notes support them; include relationship-facts (owned / acquired / used_for / chosen_over / opinion) the notes establish.\n` +
|
|
320
|
+
`- Do NOT emit \`kind:\` or \`aliases:\` — those are added for you.\n` +
|
|
321
|
+
`Emit nothing for a fact the notes do not support — never guess a spec.\n\n` +
|
|
322
|
+
`OTHER ARTICLES (link targets):\n${otherArticles}\n\n` +
|
|
323
|
+
`SOURCE NOTES:\n${sourceDigest(chunks)}\n`
|
|
324
|
+
);
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** The "Cite source X as [^n]" lines for the chunks in one draft call. */
|
|
328
|
+
function footnoteList(chunks: SynthesisChunk[], footnotes: Map<string, FootnoteEntry>): string {
|
|
329
|
+
const seen = new Set<number>();
|
|
330
|
+
const lines: string[] = [];
|
|
331
|
+
for (const c of chunks) {
|
|
332
|
+
const e = footnotes.get(c.conv);
|
|
333
|
+
if (!e || seen.has(e.n)) continue;
|
|
334
|
+
seen.add(e.n);
|
|
335
|
+
lines.push(`- [^${e.n}] = "${e.source}"`);
|
|
336
|
+
}
|
|
337
|
+
return lines.length
|
|
338
|
+
? `Cite sources with these footnote markers (use ONLY these):\n${lines.join("\n")}`
|
|
339
|
+
: `(no citable sources for this section — leave it uncited)`;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
function buildSectionPrompt(
|
|
343
|
+
entity: string,
|
|
344
|
+
sectionTitle: string,
|
|
345
|
+
chunks: SynthesisChunk[],
|
|
346
|
+
footnotes: Map<string, FootnoteEntry>,
|
|
347
|
+
otherArticles: string,
|
|
348
|
+
policy: string,
|
|
349
|
+
): string {
|
|
350
|
+
const sources = chunks
|
|
351
|
+
.map((c) => {
|
|
352
|
+
const e = footnotes.get(c.conv);
|
|
353
|
+
const tag = e ? `[^${e.n}]` : "(uncited)";
|
|
354
|
+
return `SOURCE ${tag} [${footnoteDate(c.dateMs)}] — ${c.label ?? "chunk"}:\n${c.text.slice(0, 1400).trim()}`;
|
|
355
|
+
})
|
|
356
|
+
.join("\n\n---\n\n");
|
|
357
|
+
return (
|
|
358
|
+
`${policy}\n\n${NO_COPY_GUARD}\n\n${WIKILINK_RULES}\n\n${CITATION_RULES}\n\n${DATE_RESOLVE_RULES}\n\n` +
|
|
359
|
+
`${footnoteList(chunks, footnotes)}\n\n` +
|
|
360
|
+
`OTHER ARTICLES (link to these stems only):\n${otherArticles}\n\n` +
|
|
361
|
+
`Draft ONLY the body of the "${sectionTitle}" section of the article about "${entity}". Factual prose in the THIRD PERSON, Wikipedia-style — NEVER the first person ("I", "my"); refer to the user in the third person ("the user"). These articles document what the user thinks, so they are NOT neutral: capture the user's positions and opinions accurately. Preserve specifics, and when two sources conflict fold the correction toward the user's LATEST-DATED position. Output ONLY this section's prose — do NOT repeat the "## ${sectionTitle}" heading, do NOT write other sections, do NOT write a References section.\n\n` +
|
|
362
|
+
`SOURCE MATERIAL:\n${sources}\n`
|
|
363
|
+
);
|
|
364
|
+
}
|
|
11
365
|
|
|
12
|
-
|
|
366
|
+
// ---- model-output sanitizers -----------------------------------------------
|
|
367
|
+
|
|
368
|
+
/**
|
|
369
|
+
* Meta / planning / refusal phrasings — the base model narrating ABOUT the task
|
|
370
|
+
* instead of writing the article body. A hit on any one rejects the draft (the
|
|
371
|
+
* real defect: a section body that literally read "The user is asking to draft
|
|
372
|
+
* the body prose for the <section name> section. The source material
|
|
373
|
+
* discusses: 1. …"). Tuned to NOT fire on legitimate personal-wiki prose, which
|
|
374
|
+
* IS about the user ("the user owned…", "the user weighed…").
|
|
375
|
+
*/
|
|
376
|
+
const LEAK_PATTERNS: RegExp[] = [
|
|
377
|
+
/\bthe user (is asking|is requesting|wants me|asked me|requests|would like me|needs me|wanted me)\b/i,
|
|
378
|
+
/\bthe source material\b/i,
|
|
379
|
+
/\bthe source notes?\b/i,
|
|
380
|
+
/\bsource material discusses\b/i,
|
|
381
|
+
/\bbased on the (source|provided|notes|material)\b/i,
|
|
382
|
+
/\bdraft the body\b/i,
|
|
383
|
+
/\bbody prose for\b/i,
|
|
384
|
+
/\bfor the [^.\n]{0,40} section\b/i,
|
|
385
|
+
/\bi need to\b/i,
|
|
386
|
+
/\bi (will|shall) (draft|write|now|create|provide|outline|summarize)\b/i,
|
|
387
|
+
/\bi'?ll (draft|write|provide|outline|summarize)\b/i,
|
|
388
|
+
/\blet (me|us|'?s) (draft|write|outline|provide|start|begin|summarize)\b/i,
|
|
389
|
+
/\bhere ?(is|'?s) (the|a|my) (article|section|lead|prose|draft|body)\b/i,
|
|
390
|
+
/\bas an ai\b/i,
|
|
391
|
+
/\bi (cannot|can'?t|can not)\b/i,
|
|
392
|
+
/\bi'?m sorry\b/i,
|
|
393
|
+
/\bi am sorry\b/i,
|
|
394
|
+
/\bi apologize\b/i,
|
|
395
|
+
/\bi (do not|don'?t) have (enough|sufficient|any)\b/i,
|
|
396
|
+
/\bi'?m (unable|not able)\b/i,
|
|
397
|
+
/\bi am (unable|not able)\b/i,
|
|
398
|
+
/\bsorry,?\s+(i|but)\b/i,
|
|
399
|
+
];
|
|
400
|
+
|
|
401
|
+
/**
|
|
402
|
+
* True when a drafted body is meta-commentary, a plan, or a refusal rather than
|
|
403
|
+
* article prose. Heuristic: any {@link LEAK_PATTERNS} hit, OR a leading
|
|
404
|
+
* numbered/bulleted PLAN that restates the task. Pure; empty text is not leaky.
|
|
405
|
+
*/
|
|
406
|
+
export function isLeakyDraft(text: string): boolean {
|
|
407
|
+
const s = text.trim();
|
|
408
|
+
if (!s) return false;
|
|
409
|
+
for (const re of LEAK_PATTERNS) if (re.test(s)) return true;
|
|
410
|
+
// A leading numbered/bulleted PLAN that restates the task ("1. discuss the
|
|
411
|
+
// <subject>, 2. the <subject>") — a list item FIRST, plus task-restating vocabulary.
|
|
412
|
+
const firstLine = s.split("\n").map((l) => l.trim()).find((l) => l.length > 0) ?? "";
|
|
413
|
+
if (
|
|
414
|
+
/^(\d+[.)]|[-*+])\s+/.test(firstLine) &&
|
|
415
|
+
/\b(draft|discuss|summariz|describe the|outline|restate|the user|source material)\b/i.test(s)
|
|
416
|
+
) {
|
|
417
|
+
return true;
|
|
418
|
+
}
|
|
419
|
+
return false;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
/** Remove any model-emitted fenced ```info block from drafted text so the only
|
|
423
|
+
* infobox in the final article is the ONE code-constructed block. A fenced info
|
|
424
|
+
* block is a ``` fence whose info string is exactly `info`. */
|
|
425
|
+
export function stripFencedInfoBlocks(text: string): string {
|
|
426
|
+
const lines = text.split("\n");
|
|
427
|
+
const out: string[] = [];
|
|
428
|
+
let i = 0;
|
|
429
|
+
while (i < lines.length) {
|
|
430
|
+
if (isFenceLine(lines[i]!) && fenceLabel(lines[i]!) === "info") {
|
|
431
|
+
i++; // skip the opening ```info
|
|
432
|
+
while (i < lines.length && !isFenceLine(lines[i]!)) i++;
|
|
433
|
+
if (i < lines.length) i++; // skip the closing fence
|
|
434
|
+
continue;
|
|
435
|
+
}
|
|
436
|
+
out.push(lines[i]!);
|
|
437
|
+
i++;
|
|
438
|
+
}
|
|
439
|
+
return out.join("\n");
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
/** Count fenced ```info blocks in an article (the single-infobox invariant). */
|
|
443
|
+
export function countFencedInfoBlocks(content: string): number {
|
|
444
|
+
let count = 0;
|
|
445
|
+
let inFence = false;
|
|
446
|
+
for (const line of content.split(/\r?\n/)) {
|
|
447
|
+
if (isFenceLine(line)) {
|
|
448
|
+
if (!inFence && fenceLabel(line) === "info") count++;
|
|
449
|
+
inFence = !inFence;
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
return count;
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
/** Strip code fences, a repeated section heading, any model-emitted References /
|
|
456
|
+
* definition lines, a stray model-emitted infobox, and surrounding blank lines
|
|
457
|
+
* from a drafted section body. */
|
|
458
|
+
export function sanitizeSection(raw: string, title: string): string {
|
|
459
|
+
let s = stripFencedInfoBlocks(raw.trim()).trim();
|
|
460
|
+
s = s.replace(/^```[a-z]*\s*\n/i, "").replace(/\n```\s*$/i, "").trim();
|
|
461
|
+
// Drop any NON-numeric footnote marker (e.g. a parroted `[^N]` placeholder from
|
|
462
|
+
// the citation contract). Only `[^<digits>]` is a real, code-assigned marker.
|
|
463
|
+
s = s.replace(/ ?\[\^[^\]\d][^\]]*\]/g, "");
|
|
464
|
+
const wanted = title.trim().toLowerCase();
|
|
465
|
+
const kept: string[] = [];
|
|
466
|
+
let dropping = false;
|
|
467
|
+
for (const line of s.split("\n")) {
|
|
468
|
+
const t = line.trim();
|
|
469
|
+
const h = /^(#{1,6})\s+(.+)$/.exec(t);
|
|
470
|
+
if (h) {
|
|
471
|
+
const ht = h[2]!.trim().toLowerCase();
|
|
472
|
+
if (ht === wanted) continue; // model repeated our heading
|
|
473
|
+
if (ht === "references" || ht === "see also" || ht === "notes") {
|
|
474
|
+
dropping = true; // model spilled trailing meta sections — cut them
|
|
475
|
+
continue;
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
if (dropping) continue;
|
|
479
|
+
if (/^\[\^\d+\]:/.test(t)) continue; // a stray definition line — we own defs
|
|
480
|
+
kept.push(line);
|
|
481
|
+
}
|
|
482
|
+
// A drafted section that opened a ``` fence but never closed it would, once
|
|
483
|
+
// assembled, swallow every trailing section (## References / ## See also) of
|
|
484
|
+
// the WHOLE article. Balance the section's fences before it reaches assembly.
|
|
485
|
+
return repairFences(kept.join("\n").trim()).content.trim();
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
/** Strip fences / a leading heading / a stray infobox from a drafted lead,
|
|
489
|
+
* collapse to prose. */
|
|
490
|
+
export function sanitizeLead(raw: string): string {
|
|
491
|
+
let s = stripFencedInfoBlocks(raw.trim()).trim();
|
|
492
|
+
s = s.replace(/^```[a-z]*\s*\n/i, "").replace(/\n```\s*$/i, "").trim();
|
|
493
|
+
s = s.replace(/ ?\[\^[^\]\d][^\]]*\]/g, ""); // drop parroted non-numeric markers
|
|
494
|
+
const lines = s.split("\n").filter((l) => !/^#{1,6}\s+/.test(l.trim()) && !/^\[\^\d+\]:/.test(l.trim()));
|
|
495
|
+
// Balance any unclosed ``` so a lead fence can't swallow the rest of the article.
|
|
496
|
+
return repairFences(lines.join("\n").trim()).content.trim();
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
/**
|
|
500
|
+
* Physical spec-sheet keys describing a concrete object's hardware. They belong
|
|
501
|
+
* on a `thing` (a concrete named object) and must NEVER bleed onto a broad
|
|
502
|
+
* `domain` / `standard` / `person` / `project` entity — the policy-example
|
|
503
|
+
* parroting bug, where a broad DOMAIN article inherited a hardware spec key.
|
|
504
|
+
* Grounding the infobox in the entity's own kind blocks the leak deterministically.
|
|
505
|
+
*/
|
|
506
|
+
export const PHYSICAL_SPEC_KEYS = new Set([
|
|
507
|
+
"mount", "sensor", "sensor_size", "focal_length", "aperture", "format",
|
|
508
|
+
"resolution", "megapixels", "coverage", "compatible_with", "lens_mount",
|
|
509
|
+
]);
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* Parse model `key: value` lines into a clean info-block field list. Keeps only
|
|
513
|
+
* snake_case keys with a non-empty value (first occurrence wins), drops any
|
|
514
|
+
* model-supplied `kind`/`aliases` (we own those) and any physical-spec key that
|
|
515
|
+
* is ungrounded for this entity kind (a world-fact spec key on a `domain` is a
|
|
516
|
+
* parroted example, never a fact), then prepends `type:` (if any), the declared `kind:`,
|
|
517
|
+
* and appends `aliases:` when we have them. Always returns at least the `kind:`
|
|
518
|
+
* line, so the article always has a parseable infobox.
|
|
519
|
+
*/
|
|
520
|
+
export function buildInfoboxFields(raw: string, kind: EntityKind, aliases: string[] = []): string[] {
|
|
521
|
+
const fields = new Map<string, string>();
|
|
522
|
+
for (const line of raw.split("\n")) {
|
|
523
|
+
const t = line.trim().replace(/^[-*+]\s+/, "");
|
|
524
|
+
const idx = t.indexOf(":");
|
|
525
|
+
if (idx <= 0) continue;
|
|
526
|
+
const key = t.slice(0, idx).trim();
|
|
527
|
+
const value = t.slice(idx + 1).trim();
|
|
528
|
+
if (!/^[a-z][a-z0-9_]*$/.test(key)) continue;
|
|
529
|
+
if (!value) continue;
|
|
530
|
+
if (key === "kind" || key === "aliases") continue;
|
|
531
|
+
// Ground the box in the entity's kind: a hardware spec on a non-`thing`
|
|
532
|
+
// entity is a parroted example, not a fact established by its chunks.
|
|
533
|
+
if (kind !== "thing" && PHYSICAL_SPEC_KEYS.has(key)) continue;
|
|
534
|
+
if (!fields.has(key)) fields.set(key, value);
|
|
535
|
+
}
|
|
536
|
+
const lines: string[] = [];
|
|
537
|
+
const type = fields.get("type");
|
|
538
|
+
if (type) lines.push(`type: ${type}`);
|
|
539
|
+
lines.push(`kind: ${kind}`);
|
|
540
|
+
for (const [k, v] of fields) {
|
|
541
|
+
if (k === "type") continue;
|
|
542
|
+
lines.push(`${k}: ${v}`);
|
|
543
|
+
}
|
|
544
|
+
const cleanAliases = [...new Set(aliases.map((a) => a.trim()).filter(Boolean))];
|
|
545
|
+
if (cleanAliases.length) lines.push(`aliases: ${cleanAliases.join(", ")}`);
|
|
546
|
+
return lines;
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
// ---- assembly --------------------------------------------------------------
|
|
550
|
+
|
|
551
|
+
export interface AssembledSection {
|
|
552
|
+
title: string;
|
|
553
|
+
anchor: string;
|
|
554
|
+
body: string;
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
/** Distinct wikilink targets in `text` that resolve to a known stem (excluding
|
|
558
|
+
* self / Category: / conv:) — the See also list, sorted. */
|
|
559
|
+
export function deriveSeeAlso(text: string, stems: Set<string>, selfStem: string): string[] {
|
|
560
|
+
const out = new Set<string>();
|
|
561
|
+
for (const m of text.matchAll(/\[\[([^\]]+?)\]\]/g)) {
|
|
562
|
+
const inner = m[1]!.split("|")[0]!.split("#")[0]!.trim();
|
|
563
|
+
if (!inner || /^conv:/i.test(inner) || /^Category:/i.test(inner)) continue;
|
|
564
|
+
const r = resolveWikilinkToStem(inner, stems);
|
|
565
|
+
if (r && r !== selfStem) out.add(r);
|
|
566
|
+
}
|
|
567
|
+
return [...out].sort((a, b) => a.localeCompare(b));
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
/** Compose the fixed article skeleton from its parts. `## See also` /
|
|
571
|
+
* `## References` are emitted only when non-empty; References is always last. */
|
|
572
|
+
export function assembleArticle(parts: {
|
|
573
|
+
stem: string;
|
|
574
|
+
infoboxFields: string[];
|
|
575
|
+
lead: string;
|
|
576
|
+
sections: AssembledSection[];
|
|
577
|
+
seeAlso: string[];
|
|
578
|
+
referenceDefs: string[];
|
|
579
|
+
}): string {
|
|
580
|
+
const title = parts.stem.replace(/_/g, " ");
|
|
581
|
+
const out: string[] = [`# ${title}`, ""];
|
|
582
|
+
if (parts.infoboxFields.length) {
|
|
583
|
+
out.push("```info", ...parts.infoboxFields, "```", "");
|
|
584
|
+
}
|
|
585
|
+
if (parts.lead.trim()) out.push(parts.lead.trim(), "");
|
|
586
|
+
for (const s of parts.sections) {
|
|
587
|
+
if (!s.body.trim()) continue;
|
|
588
|
+
out.push(`## ${s.title}`, "", s.body.trim(), "");
|
|
589
|
+
}
|
|
590
|
+
if (parts.seeAlso.length) {
|
|
591
|
+
out.push("## See also", "", ...parts.seeAlso.map((s) => `- [[${s}]]`), "");
|
|
592
|
+
}
|
|
593
|
+
if (parts.referenceDefs.length) {
|
|
594
|
+
out.push("## References", "", ...parts.referenceDefs, "");
|
|
595
|
+
}
|
|
596
|
+
const assembled = out.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
|
|
597
|
+
// Final assembly guard: a drafted section may still carry an unbalanced ``` that
|
|
598
|
+
// the per-section sanitizer missed; never emit an article whose fences are odd
|
|
599
|
+
// (it would read its own trailing ## References / ## See also as code). Closing
|
|
600
|
+
// the dangling fence before the first heading it would swallow keeps every
|
|
601
|
+
// trailing section a recognized heading.
|
|
602
|
+
const repaired = repairFences(assembled).content;
|
|
603
|
+
return repaired.replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
/** Distinct `[^N]` marker numbers present in body prose (definition lines and
|
|
607
|
+
* fenced blocks excluded — markers live in prose). */
|
|
608
|
+
function usedMarkers(text: string): Set<number> {
|
|
609
|
+
const used = new Set<number>();
|
|
610
|
+
for (const line of text.split("\n")) {
|
|
611
|
+
if (/^\[\^\d+\]:/.test(line.trim())) continue;
|
|
612
|
+
for (const m of line.matchAll(/\[\^(\d+)\]/g)) used.add(parseInt(m[1]!, 10));
|
|
613
|
+
}
|
|
614
|
+
return used;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
/** Count sections (by the parsed skeleton) whose body carries ≥1 footnote
|
|
618
|
+
* marker — the "≥1 cited section" acceptance signal. */
|
|
619
|
+
export function countCitedSections(content: string): number {
|
|
620
|
+
const items = articleStructure(content);
|
|
621
|
+
const lines = content.split(/\r?\n/);
|
|
622
|
+
let cited = 0;
|
|
623
|
+
for (const it of items) {
|
|
624
|
+
if (it.kind !== "section") continue;
|
|
625
|
+
const body = lines.slice(it.startLine - 1, it.endLine).join("\n");
|
|
626
|
+
if (/\[\^\d+\](?!:)/.test(body)) cited++;
|
|
627
|
+
}
|
|
628
|
+
return cited;
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
// ---- CREATE (pure-ish: model via deps, no disk writes) ---------------------
|
|
632
|
+
|
|
633
|
+
/** Per-stage model call seam (defaults to the local synthesis stage). Injected
|
|
634
|
+
* by tests so the deterministic scaffolding runs with no GPU. */
|
|
635
|
+
export type SynthesisCall = (prompt: string, opts?: { maxTokens?: number }) => Promise<string>;
|
|
636
|
+
|
|
637
|
+
/** Retry instruction when a draft leaks meta/planning/refusal text. */
|
|
638
|
+
const ANTI_LEAK_HINT =
|
|
639
|
+
"Output ONLY the article prose for this section — no preamble, no meta-commentary, no plan, no restating the task.";
|
|
640
|
+
|
|
641
|
+
/**
|
|
642
|
+
* Draft prose that is NOT meta/planning/refusal leakage: draft once; if the
|
|
643
|
+
* sanitized body is {@link isLeakyDraft}, retry ONCE with a sharper instruction;
|
|
644
|
+
* if it still leaks, return "" so the caller DROPS the section rather than
|
|
645
|
+
* emitting garbage. The sanitizer (section vs lead) is injected.
|
|
646
|
+
*/
|
|
647
|
+
async function draftCleanProse(
|
|
648
|
+
call: SynthesisCall,
|
|
649
|
+
prompt: string,
|
|
650
|
+
opts: { maxTokens?: number },
|
|
651
|
+
sanitize: (raw: string) => string,
|
|
652
|
+
): Promise<string> {
|
|
653
|
+
let out = sanitize(await call(prompt, opts));
|
|
654
|
+
if (!isLeakyDraft(out)) return out;
|
|
655
|
+
out = sanitize(await call(`${prompt}\n\n${ANTI_LEAK_HINT}`, opts));
|
|
656
|
+
return isLeakyDraft(out) ? "" : out;
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
export interface CreateDeps {
|
|
660
|
+
/** Model call; defaults to `callLocal("synthesis", …)`. */
|
|
661
|
+
call?: SynthesisCall;
|
|
662
|
+
/** Other article stems (for wikilink resolution); defaults to the vault list. */
|
|
663
|
+
stems?: Set<string>;
|
|
664
|
+
/** Declared aliases to seed the infobox `aliases:` line. */
|
|
665
|
+
aliases?: string[];
|
|
666
|
+
/** Pre-existing article content for the gate's `before` (CREATE: ""). */
|
|
667
|
+
before?: string;
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
export interface CreateOutcome {
|
|
671
|
+
entity: string;
|
|
672
|
+
stem: string;
|
|
673
|
+
/** Final post-NORMALIZE markdown, or null when the gate vetoed the write. */
|
|
674
|
+
content: string | null;
|
|
675
|
+
action: "created" | "skipped";
|
|
676
|
+
/** Gate reason when skipped; null on success. */
|
|
677
|
+
reason: string | null;
|
|
678
|
+
/** (chunk_id, section_anchor) edges for the `synthesized_chunk_sections` ledger. */
|
|
679
|
+
chunkSections: { chunkId: string; anchor: string }[];
|
|
680
|
+
sectionAnchors: string[];
|
|
681
|
+
citedSections: number;
|
|
682
|
+
hasInfobox: boolean;
|
|
683
|
+
stub: boolean;
|
|
684
|
+
normalizeNotes: string[];
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/**
|
|
688
|
+
* Build ONE entity article from its accumulated chunks, end to end:
|
|
689
|
+
* sub-cluster → OUTLINE → per-section DRAFT → seed INFOBOX → assemble →
|
|
690
|
+
* NORMALIZE → gate. Returns the final content (or null when the gate vetoes a
|
|
691
|
+
* weak pass). Pure with respect to disk — the caller persists.
|
|
692
|
+
*/
|
|
693
|
+
export async function createArticle(
|
|
694
|
+
entity: string,
|
|
695
|
+
kind: EntityKind,
|
|
696
|
+
chunks: SynthesisChunk[],
|
|
697
|
+
deps: CreateDeps = {},
|
|
698
|
+
): Promise<CreateOutcome> {
|
|
699
|
+
const call: SynthesisCall = deps.call ?? ((p, o) => callLocal("synthesis", { user: p }, o));
|
|
700
|
+
const stem = entityStem(entity);
|
|
701
|
+
const stems = new Set(deps.stems ?? []);
|
|
702
|
+
stems.add(stem);
|
|
703
|
+
const otherArticles = otherArticlesBlock(stems, stem);
|
|
704
|
+
const policy = loadMetaPolicyQuiet(["Article_Conventions", "Infobox_Schemas", "Entities"]);
|
|
705
|
+
const { byConv, entries } = buildFootnoteMap(chunks);
|
|
706
|
+
|
|
707
|
+
// OUTLINE → sections; assign chunks; drop sections that won no chunk.
|
|
708
|
+
const outline = parseOutline(await call(buildOutlinePrompt(entity, chunks, policy), { maxTokens: MAX_OUTPUT_TOKENS }));
|
|
709
|
+
const clustered = subClusterChunks(chunks, outline);
|
|
710
|
+
const sections: AssembledSection[] = [];
|
|
711
|
+
const chunkSections: { chunkId: string; anchor: string }[] = [];
|
|
712
|
+
for (const title of outline) {
|
|
713
|
+
const anchor = sectionAnchor(title);
|
|
714
|
+
const secChunks = clustered.get(anchor);
|
|
715
|
+
if (!secChunks || secChunks.length === 0) continue;
|
|
716
|
+
const body = await draftCleanProse(
|
|
717
|
+
call,
|
|
718
|
+
buildSectionPrompt(entity, title, secChunks, byConv, otherArticles, policy),
|
|
719
|
+
{ maxTokens: MAX_OUTPUT_TOKENS },
|
|
720
|
+
(raw) => sanitizeSection(raw, title),
|
|
721
|
+
);
|
|
722
|
+
// A leaky draft survives to "" after one retry — DROP the section (and its
|
|
723
|
+
// chunk edges); never emit meta/planning/refusal prose into the article.
|
|
724
|
+
if (!body.trim()) continue;
|
|
725
|
+
sections.push({ title, anchor, body });
|
|
726
|
+
for (const c of secChunks) chunkSections.push({ chunkId: c.id, anchor });
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
// LEAD + INFOBOX (bounded, single-purpose calls). The lead runs the same
|
|
730
|
+
// leak-rejection so a refusal/plan never lands as the article's abstract.
|
|
731
|
+
const lead = await draftCleanProse(
|
|
732
|
+
call,
|
|
733
|
+
buildLeadPrompt(entity, chunks, byConv, otherArticles, policy),
|
|
734
|
+
{ maxTokens: MAX_OUTPUT_TOKENS },
|
|
735
|
+
sanitizeLead,
|
|
736
|
+
);
|
|
737
|
+
const infoboxFields = buildInfoboxFields(
|
|
738
|
+
await call(buildInfoboxPrompt(entity, kind, chunks, otherArticles, policy), { maxTokens: MAX_OUTPUT_TOKENS }),
|
|
739
|
+
kind,
|
|
740
|
+
deps.aliases ?? [],
|
|
741
|
+
);
|
|
742
|
+
|
|
743
|
+
// References: only the defs whose marker the model actually placed.
|
|
744
|
+
const bodyForMarkers = [lead, ...sections.map((s) => s.body)].join("\n");
|
|
745
|
+
const used = usedMarkers(bodyForMarkers);
|
|
746
|
+
const referenceDefs = entries.filter((e) => used.has(e.n)).map((e) => e.def);
|
|
747
|
+
|
|
748
|
+
const seeAlso = deriveSeeAlso([infoboxFields.join("\n"), bodyForMarkers].join("\n"), stems, stem);
|
|
749
|
+
|
|
750
|
+
const raw = assembleArticle({ stem, infoboxFields, lead, sections, seeAlso, referenceDefs });
|
|
751
|
+
|
|
752
|
+
// NORMALIZE (footnote bijection + renumber, wikilink canonicalization, infobox
|
|
753
|
+
// sort, structural guards, {{stub}} for thin), then the conservative gate.
|
|
754
|
+
const norm = normalizeArticle(raw, { stem, stems });
|
|
755
|
+
|
|
756
|
+
const base: Omit<CreateOutcome, "content" | "action" | "reason"> = {
|
|
757
|
+
entity,
|
|
758
|
+
stem,
|
|
759
|
+
chunkSections,
|
|
760
|
+
sectionAnchors: sections.map((s) => s.anchor),
|
|
761
|
+
citedSections: countCitedSections(norm.content),
|
|
762
|
+
hasInfobox: parseInfobox(norm.content) != null,
|
|
763
|
+
stub: hasStubMarker(norm.content),
|
|
764
|
+
normalizeNotes: norm.notes,
|
|
765
|
+
};
|
|
766
|
+
|
|
767
|
+
// Defect-2: every committed article MUST be footnote-bijective. NORMALIZE
|
|
768
|
+
// already repairs (drops orphan markers, renumbers, merges dup defs); if the
|
|
769
|
+
// result is STILL broken (an unrepairable malformed citation), veto the write
|
|
770
|
+
// rather than commit a citation-broken article.
|
|
771
|
+
const fn = checkFootnoteIntegrity(norm.content);
|
|
772
|
+
if (!fn.ok) {
|
|
773
|
+
return { ...base, content: null, action: "skipped", reason: `footnote bijection unrepairable: ${fn.errors.join("; ")}` };
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
// Defect-3: exactly ONE infobox — the code-constructed block. Drafted info
|
|
777
|
+
// blocks are stripped in sanitizeSection/sanitizeLead; this asserts they were,
|
|
778
|
+
// so a parroted second infobox can never reach the vault.
|
|
779
|
+
const infoboxCount = countFencedInfoBlocks(norm.content);
|
|
780
|
+
if (infoboxCount > 1) {
|
|
781
|
+
return { ...base, content: null, action: "skipped", reason: `multiple infoboxes (${infoboxCount})` };
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
const verdict: GateVerdict = gateEdit(deps.before ?? "", norm.content);
|
|
785
|
+
if (!verdict.ok) {
|
|
786
|
+
return { ...base, content: null, action: "skipped", reason: verdict.reason };
|
|
787
|
+
}
|
|
788
|
+
return { ...base, content: norm.content, action: "created", reason: null };
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
/** `loadMetaPolicy` that never throws — a missing Meta page yields an empty
|
|
792
|
+
* policy block so a fixture/smoke vault without seeded Meta still drafts. */
|
|
793
|
+
function loadMetaPolicyQuiet(names: string[]): string {
|
|
794
|
+
try {
|
|
795
|
+
return loadMetaPolicy(names);
|
|
796
|
+
} catch {
|
|
797
|
+
return "";
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
// ---- effectful entry: CREATE into the vault + ledger -----------------------
|
|
13
802
|
|
|
14
803
|
export interface SynthesizeResult {
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
skippedByGate:
|
|
804
|
+
created: boolean;
|
|
805
|
+
stem: string;
|
|
806
|
+
skippedByGate: boolean;
|
|
807
|
+
reason: string | null;
|
|
808
|
+
hasInfobox: boolean;
|
|
809
|
+
citedSections: number;
|
|
810
|
+
sectionAnchors: string[];
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
export interface SynthesizeCreateOpts {
|
|
814
|
+
entity: string;
|
|
815
|
+
kind?: EntityKind;
|
|
816
|
+
/** The entity's accumulated chunk ids (the CREATE rule already fired). */
|
|
817
|
+
chunkIds: string[];
|
|
818
|
+
/** Vault root (honors MLX_BUN_WIKI); defaults to `vaultRoot()`. */
|
|
819
|
+
root?: string;
|
|
820
|
+
/** Declared aliases for the infobox. */
|
|
821
|
+
aliases?: string[];
|
|
822
|
+
/** Model-call override (tests inject a fake). */
|
|
823
|
+
call?: SynthesisCall;
|
|
824
|
+
/** Skip the git commit (tests). */
|
|
825
|
+
commit?: boolean;
|
|
826
|
+
now?: number;
|
|
18
827
|
}
|
|
19
828
|
|
|
20
|
-
/**
|
|
21
|
-
export
|
|
22
|
-
|
|
23
|
-
|
|
829
|
+
/** Read the chunk + its conversation metadata into a {@link SynthesisChunk}. */
|
|
830
|
+
export function loadSynthesisChunk(store: MemoryStore, chunkId: string): SynthesisChunk | null {
|
|
831
|
+
const row = store.db
|
|
832
|
+
.query(
|
|
833
|
+
"SELECT c.id, c.conv, c.label, conv.title, conv.updated_at FROM chunks c " +
|
|
834
|
+
"JOIN conversations conv ON conv.conv = c.conv WHERE c.id = ?",
|
|
835
|
+
)
|
|
836
|
+
.get(chunkId) as
|
|
837
|
+
| { id: string; conv: string; label: string | null; title: string | null; updated_at: number | null }
|
|
838
|
+
| null;
|
|
839
|
+
if (!row) return null;
|
|
840
|
+
return {
|
|
841
|
+
id: row.id,
|
|
842
|
+
conv: row.conv,
|
|
843
|
+
label: row.label,
|
|
844
|
+
text: store.chunkText(chunkId),
|
|
845
|
+
title: row.title ?? "",
|
|
846
|
+
dateMs: row.updated_at,
|
|
847
|
+
};
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
/**
|
|
851
|
+
* CREATE an entity article from its chunks and persist it: draft via
|
|
852
|
+
* {@link createArticle}, write the markdown to the vault (unless the gate vetoed
|
|
853
|
+
* it), commit via `commitVault`, and record the integrated (chunk, section)
|
|
854
|
+
* edges in `synthesized_chunk_sections`. The CREATE decision itself is upstream
|
|
855
|
+
* (ROUTE's B8/P1-T5 gate) — this assumes the entity earned its article.
|
|
856
|
+
*/
|
|
857
|
+
export async function synthesizeCreate(
|
|
858
|
+
store: MemoryStore,
|
|
859
|
+
opts: SynthesizeCreateOpts,
|
|
24
860
|
): Promise<SynthesizeResult> {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
861
|
+
const root = opts.root ?? vaultRoot();
|
|
862
|
+
const stem = entityStem(opts.entity);
|
|
863
|
+
const chunks: SynthesisChunk[] = [];
|
|
864
|
+
for (const id of opts.chunkIds) {
|
|
865
|
+
const c = loadSynthesisChunk(store, id);
|
|
866
|
+
if (c) chunks.push(c);
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
const stems = new Set(await listArticles(root));
|
|
870
|
+
const outcome = await createArticle(opts.entity, opts.kind ?? "thing", chunks, {
|
|
871
|
+
call: opts.call,
|
|
872
|
+
stems,
|
|
873
|
+
aliases: opts.aliases,
|
|
874
|
+
});
|
|
875
|
+
|
|
876
|
+
const result: SynthesizeResult = {
|
|
877
|
+
created: outcome.action === "created",
|
|
878
|
+
stem,
|
|
879
|
+
skippedByGate: outcome.action === "skipped",
|
|
880
|
+
reason: outcome.reason,
|
|
881
|
+
hasInfobox: outcome.hasInfobox,
|
|
882
|
+
citedSections: outcome.citedSections,
|
|
883
|
+
sectionAnchors: outcome.sectionAnchors,
|
|
884
|
+
};
|
|
885
|
+
|
|
886
|
+
if (outcome.action !== "created" || outcome.content == null) return result;
|
|
887
|
+
|
|
888
|
+
// Write the article, then record the integration ledger + commit.
|
|
889
|
+
const dir = articlesDir(root);
|
|
890
|
+
await mkdir(dir, { recursive: true });
|
|
891
|
+
const path = join(dir, `${stem}.md`);
|
|
892
|
+
// Never clobber an existing article from the CREATE branch — that's UPDATE's job.
|
|
893
|
+
if (await fileExists(path)) {
|
|
894
|
+
result.created = false;
|
|
895
|
+
result.skippedByGate = false;
|
|
896
|
+
result.reason = "article already exists (CREATE is for new entities)";
|
|
897
|
+
return result;
|
|
898
|
+
}
|
|
899
|
+
await writeFile(path, outcome.content);
|
|
900
|
+
|
|
901
|
+
const now = opts.now ?? Date.now();
|
|
902
|
+
const insSec = store.db.query(
|
|
903
|
+
"INSERT OR IGNORE INTO chunk_sections (chunk_id, article_stem, section_anchor) VALUES (?, ?, ?)",
|
|
904
|
+
);
|
|
905
|
+
const insSynth = store.db.query(
|
|
906
|
+
"INSERT OR REPLACE INTO synthesized_chunk_sections " +
|
|
907
|
+
"(chunk_id, article_stem, section_anchor, synthesized_at) VALUES (?, ?, ?, ?)",
|
|
908
|
+
);
|
|
909
|
+
const tx = store.db.transaction((rows: { chunkId: string; anchor: string }[]) => {
|
|
910
|
+
for (const r of rows) {
|
|
911
|
+
insSec.run(r.chunkId, stem, r.anchor);
|
|
912
|
+
insSynth.run(r.chunkId, stem, r.anchor, now);
|
|
913
|
+
}
|
|
914
|
+
});
|
|
915
|
+
tx(outcome.chunkSections);
|
|
916
|
+
|
|
917
|
+
if (opts.commit !== false) {
|
|
918
|
+
await commitVault(root, `memory: create ${stem} (${outcome.chunkSections.length} chunk edges)`);
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
return result;
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
async function fileExists(path: string): Promise<boolean> {
|
|
925
|
+
try {
|
|
926
|
+
await readFile(path);
|
|
927
|
+
return true;
|
|
928
|
+
} catch {
|
|
929
|
+
return false;
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
// ===========================================================================
|
|
934
|
+
// PATCH — the UPDATE branch (P7-T2): fold ONE chunk into ONE existing section.
|
|
935
|
+
//
|
|
936
|
+
// The steady-state counterpart to CREATE. Where CREATE drafts a whole article
|
|
937
|
+
// from an entity's cold-start chunks, PATCH is the same bounded section-draft op
|
|
938
|
+
// run against an article that already exists: integrate THIS chunk into THIS
|
|
939
|
+
// (SECTION-ROUTE-chosen) section, citing it, changing nothing else. It is the
|
|
940
|
+
// self-healing mechanism — a user's in-conversation CORRECTION is just an
|
|
941
|
+
// ordinary chunk, so folding it rewrites the contradicted claim TOWARD the
|
|
942
|
+
// user's latest position (resolve-to-latest, not append-both, not ignore).
|
|
943
|
+
//
|
|
944
|
+
// Citations stay deterministic by construction exactly as in CREATE: the chunk's
|
|
945
|
+
// source conversation is assigned a stable `[^N]` (a re-cited conversation
|
|
946
|
+
// reuses its existing marker; a new one mints `maxN+1`) and we GENERATE the
|
|
947
|
+
// `[^N]:` definition line — the model only PLACES the marker. We swap only the
|
|
948
|
+
// target section's bytes via `extractSection`/`replaceSection` (every other
|
|
949
|
+
// section stays byte-identical) and append the def to `## References`, then run
|
|
950
|
+
// the SAME gate (weak → NO-OP, the vault is untouched and the chunk retries next
|
|
951
|
+
// run) + NORMALIZE as CREATE. Idempotency rides `synthesized_chunk_sections`'s PK
|
|
952
|
+
// — a second fold of the same (chunk, article, section) writes nothing.
|
|
953
|
+
// ===========================================================================
|
|
954
|
+
|
|
955
|
+
/** The article's H1 title (depth-1 TOC heading), or a neutral fallback. */
|
|
956
|
+
function articleH1Title(article: string): string {
|
|
957
|
+
const h1 = parseToc(article).find((e) => e.depth === 1);
|
|
958
|
+
return h1 ? h1.title : "this article";
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
/**
|
|
962
|
+
* The article's current footnote state: the highest `[^N]` number in use
|
|
963
|
+
* (markers ∪ definitions) and a map from each DEFINED `conv:HASH` to its number.
|
|
964
|
+
* PATCH assigns a re-cited conversation its existing marker (no duplicate def)
|
|
965
|
+
* and a brand-new one `maxN + 1`, so the article stays footnote-bijective and
|
|
966
|
+
* NORMALIZE's renumber short-circuits (no shuffle of other sections). Pure.
|
|
967
|
+
*/
|
|
968
|
+
export function articleFootnoteState(article: string): { maxN: number; byHash: Map<string, number> } {
|
|
969
|
+
let maxN = 0;
|
|
970
|
+
const byHash = new Map<string, number>();
|
|
971
|
+
for (const line of article.split("\n")) {
|
|
972
|
+
const d = /^\[\^(\d+)\]:(.*)$/.exec(line);
|
|
973
|
+
if (d) {
|
|
974
|
+
const n = parseInt(d[1]!, 10);
|
|
975
|
+
if (n > maxN) maxN = n;
|
|
976
|
+
const h = d[2]!.match(/conv:([0-9a-z]{8})/i);
|
|
977
|
+
if (h) byHash.set(h[1]!.toLowerCase(), n);
|
|
978
|
+
continue;
|
|
979
|
+
}
|
|
980
|
+
for (const m of line.matchAll(/\[\^(\d+)\]/g)) {
|
|
981
|
+
const n = parseInt(m[1]!, 10);
|
|
982
|
+
if (n > maxN) maxN = n;
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
return { maxN, byHash };
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
/**
|
|
989
|
+
* Map each `[^N]:` definition's marker number to the `YYYY-MM-DD` (or `undated`)
|
|
990
|
+
* date its generated def line carries — the dates we tagged via `footnoteDefLine`
|
|
991
|
+
* (`[^n]: \`conv:HASH\` (DATE, source)`). PATCH reads this so it can tell the model
|
|
992
|
+
* how old the claim it is correcting is, and resolve toward the LATER date. Pure.
|
|
993
|
+
*/
|
|
994
|
+
export function referenceDatesByMarker(article: string): Map<number, string> {
|
|
995
|
+
const out = new Map<number, string>();
|
|
996
|
+
for (const m of article.matchAll(/^\[\^(\d+)\]:[^\n]*?\((\d{4}-\d{2}-\d{2}|undated),/gm)) {
|
|
997
|
+
out.set(parseInt(m[1]!, 10), m[2]!);
|
|
998
|
+
}
|
|
999
|
+
return out;
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
/** The distinct `[^N]` marker numbers present in a single section/body block
|
|
1003
|
+
* (definition lines excluded), in first-appearance order. Pure. */
|
|
1004
|
+
function markersInBlock(body: string): number[] {
|
|
1005
|
+
const order: number[] = [];
|
|
1006
|
+
const seen = new Set<number>();
|
|
1007
|
+
for (const line of body.split("\n")) {
|
|
1008
|
+
if (/^\[\^\d+\]:/.test(line.trim())) continue;
|
|
1009
|
+
for (const m of line.matchAll(/\[\^(\d+)\]/g)) {
|
|
1010
|
+
const n = parseInt(m[1]!, 10);
|
|
1011
|
+
if (!seen.has(n)) {
|
|
1012
|
+
seen.add(n);
|
|
1013
|
+
order.push(n);
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
return order;
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
/**
|
|
1021
|
+
* Swap exactly the `anchor` section's block for `newBlock`, leaving every byte
|
|
1022
|
+
* OUTSIDE that section identical (the byte-preservation invariant PATCH and the
|
|
1023
|
+
* gate depend on). Returns null when the section is absent. Pure.
|
|
1024
|
+
*/
|
|
1025
|
+
export function replaceSection(article: string, anchor: string, newBlock: string): string | null {
|
|
1026
|
+
const block = extractSection(article, anchor);
|
|
1027
|
+
if (block == null) return null;
|
|
1028
|
+
const idx = article.indexOf(block);
|
|
1029
|
+
if (idx < 0) return null;
|
|
1030
|
+
return article.slice(0, idx) + newBlock + article.slice(idx + block.length);
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
/**
|
|
1034
|
+
* Insert a generated `[^N]:` definition line into the article's `## References`
|
|
1035
|
+
* section (after the last existing def), creating the section at the end when
|
|
1036
|
+
* the article has none. The model never writes defs — this is the only path that
|
|
1037
|
+
* does. Pure.
|
|
1038
|
+
*/
|
|
1039
|
+
export function appendReferenceDef(article: string, defLine: string): string {
|
|
1040
|
+
const refs = extractSection(article, "references");
|
|
1041
|
+
if (refs == null) {
|
|
1042
|
+
return `${article.replace(/\s+$/, "")}\n\n## References\n\n${defLine}\n`;
|
|
1043
|
+
}
|
|
1044
|
+
const newRefs = `${refs.replace(/\s+$/, "")}\n${defLine}\n`;
|
|
1045
|
+
return replaceSection(article, "references", newRefs) ?? article;
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
/** Retry hint when a patched section leaks meta/planning or fails to cite. */
|
|
1049
|
+
const PATCH_ANTI_LEAK_HINT =
|
|
1050
|
+
"Output ONLY the integrated section prose — no preamble, no meta-commentary, no plan, no restating the task.";
|
|
1051
|
+
|
|
1052
|
+
/**
|
|
1053
|
+
* Draft the integrated section body, rejecting leakage AND an uncited fold:
|
|
1054
|
+
* draft once; if the sanitized body is {@link isLeakyDraft} OR is missing the
|
|
1055
|
+
* assigned `[^marker]` (the integration didn't actually place the citation),
|
|
1056
|
+
* retry ONCE with a sharper instruction; if it still fails, return "" so the
|
|
1057
|
+
* caller NO-OPs (the vault is untouched; the chunk retries next run).
|
|
1058
|
+
*/
|
|
1059
|
+
async function draftPatchedProse(
|
|
1060
|
+
call: SynthesisCall,
|
|
1061
|
+
prompt: string,
|
|
1062
|
+
marker: number,
|
|
1063
|
+
sanitize: (raw: string) => string,
|
|
1064
|
+
): Promise<string> {
|
|
1065
|
+
const wants = new RegExp(`\\[\\^${marker}\\](?!:)`);
|
|
1066
|
+
const clean = (raw: string): string => {
|
|
1067
|
+
const body = sanitize(raw);
|
|
1068
|
+
return !isLeakyDraft(body) && wants.test(body) ? body : "";
|
|
1069
|
+
};
|
|
1070
|
+
const first = clean(await call(prompt, { maxTokens: MAX_OUTPUT_TOKENS }));
|
|
1071
|
+
if (first) return first;
|
|
1072
|
+
return clean(await call(`${prompt}\n\n${PATCH_ANTI_LEAK_HINT} Place the [^${marker}] marker for the new note.`, { maxTokens: MAX_OUTPUT_TOKENS }));
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
/** Draft an integrated SECTION body (sanitized as a section, must place the marker). */
|
|
1076
|
+
function draftPatchedSection(
|
|
1077
|
+
call: SynthesisCall,
|
|
1078
|
+
prompt: string,
|
|
1079
|
+
marker: number,
|
|
1080
|
+
heading: string,
|
|
1081
|
+
): Promise<string> {
|
|
1082
|
+
return draftPatchedProse(call, prompt, marker, (raw) => sanitizeSection(raw, heading));
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
/** The schematic PATCH prompt: current section markdown + the chunk + the inlined
|
|
1086
|
+
* footnote/correction contract. Kept free of copyable example values per the
|
|
1087
|
+
* small-model parroting failure mode. */
|
|
1088
|
+
export function buildPatchPrompt(
|
|
1089
|
+
title: string,
|
|
1090
|
+
heading: string,
|
|
1091
|
+
currentBody: string,
|
|
1092
|
+
chunk: SynthesisChunk,
|
|
1093
|
+
n: number,
|
|
1094
|
+
otherArticles: string,
|
|
1095
|
+
policy: string,
|
|
1096
|
+
priorDates: string[] = [],
|
|
1097
|
+
): string {
|
|
1098
|
+
const chunkDate = footnoteDate(chunk.dateMs);
|
|
1099
|
+
return (
|
|
1100
|
+
`${policy}\n\n${NO_COPY_GUARD}\n\n${WIKILINK_RULES}\n\n` +
|
|
1101
|
+
`You are integrating ONE new note (dated ${chunkDate}) into the "${heading}" section of the personal-wiki article about "${title}". ` +
|
|
1102
|
+
`Keep the existing prose and every existing [^k] citation marker intact; weave in only what the new note adds.\n\n` +
|
|
1103
|
+
`CITATION — cite the new note by placing the marker [^${n}] immediately after the specific claim it supports. ` +
|
|
1104
|
+
`Use [^${n}] for the new note, and reuse any [^k] markers already in the section exactly where they are. ` +
|
|
1105
|
+
`Do NOT write any [^k]: definition line — definitions are generated for you. ` +
|
|
1106
|
+
`[^N] is the only citation syntax; never emit HTML, bare links, or invent another number.\n\n` +
|
|
1107
|
+
correctionRule(n, chunkDate, priorDates) +
|
|
1108
|
+
`OTHER ARTICLES (link to these stems only):\n${otherArticles}\n\n` +
|
|
1109
|
+
`CURRENT "${heading}" SECTION:\n${currentBody || "(empty)"}\n\n` +
|
|
1110
|
+
`NEW NOTE (${chunkDate})${chunk.label ? ` — ${chunk.label}` : ""}:\n${chunk.text.slice(0, 1400).trim()}\n\n` +
|
|
1111
|
+
`Output ONLY the rewritten section body prose — do NOT repeat the "## ${heading}" heading, do NOT add other sections, do NOT write a References section.`
|
|
1112
|
+
);
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
/**
|
|
1116
|
+
* The PROVENANCE-PRESERVING correction rule (schematic). A correction resolves
|
|
1117
|
+
* toward the LATER-DATED statement so the latest position wins — but it KEEPS the
|
|
1118
|
+
* superseded claim's existing `[^k]` marker alongside the new `[^N]`, phrased as a
|
|
1119
|
+
* change over time, so no citation is lost. This is what lets the resolve fold pass
|
|
1120
|
+
* the gate's citation-survival check instead of being NO-OP'd for dropping `[^k]`.
|
|
1121
|
+
*/
|
|
1122
|
+
function correctionRule(n: number, chunkDate: string, priorDates: string[]): string {
|
|
1123
|
+
const prior = priorDates.filter((d) => d && d !== "undated");
|
|
1124
|
+
const priorClause = prior.length
|
|
1125
|
+
? `The claim it corrects is dated ${[...new Set(prior)].join(" / ")}, so the new note (dated ${chunkDate}) is LATER. `
|
|
1126
|
+
: `The new note is dated ${chunkDate}. `;
|
|
1127
|
+
return (
|
|
1128
|
+
`CORRECTION — if the new note CONTRADICTS a claim already in the section, this is a correction: resolve toward the LATER-DATED statement so the user's CURRENT position wins. ` +
|
|
1129
|
+
priorClause +
|
|
1130
|
+
`DEMOTE the superseded claim to the PAST TENSE — rewrite the OLD clause itself from present ("is my favourite", "currently", "the one I reach for") to past ("previously was my favourite", "originally", "used to be") — do NOT leave the old value asserted as still-current and merely append the new one. ` +
|
|
1131
|
+
`KEEP the superseded claim's existing [^k] citation marker in place ALONGSIDE the new note's [^${n}] marker, so no [^N] is lost — phrase it like "previously <earlier position>[^k]; as of <later date> <current position> is now my pick[^${n}]". ` +
|
|
1132
|
+
`Do NOT assert both as simultaneously true, and do NOT ignore the new note.\n\n`
|
|
1133
|
+
);
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
/**
|
|
1137
|
+
* The CLASSIFIED LEAD correction rule (schematic). The lead must read as a CLEAN
|
|
1138
|
+
* current-state summary — never scattered "previously …" caveats (RULE 1). So when
|
|
1139
|
+
* a contradiction is classified:
|
|
1140
|
+
* - EVOLUTION (the user changed their own position) → resolve the lead to the
|
|
1141
|
+
* CURRENT value in the present tense; KEEP the superseded claim's `[^k]` marker
|
|
1142
|
+
* so provenance survives the gate (the full trajectory is consolidated into a
|
|
1143
|
+
* `## History` section by the reconcile pass), but do NOT narrate the change in
|
|
1144
|
+
* the lead itself.
|
|
1145
|
+
* - ERROR (a wrong fact being corrected) → SILENTLY replace the wrong value with
|
|
1146
|
+
* the correct one; the wrong value's `[^k]` citation MAY be dropped (it is
|
|
1147
|
+
* pruned as if it had never been there).
|
|
1148
|
+
*/
|
|
1149
|
+
function leadCorrectionRule(kind: ContradictionKind, n: number, chunkDate: string): string {
|
|
1150
|
+
if (kind === "error") {
|
|
1151
|
+
return (
|
|
1152
|
+
`CORRECTION (factual error) — the new note (dated ${chunkDate}) corrects a WRONG fact already in the lead. ` +
|
|
1153
|
+
`Silently REPLACE the wrong value with the correct one and cite it [^${n}]; state ONLY the correct value. ` +
|
|
1154
|
+
`Do NOT mention the wrong value, do NOT write "previously" — it was never a real position. ` +
|
|
1155
|
+
`You MAY drop the wrong value's [^k] citation marker.\n\n`
|
|
1156
|
+
);
|
|
1157
|
+
}
|
|
1158
|
+
return (
|
|
1159
|
+
`CORRECTION (the user's view evolved) — the new note (dated ${chunkDate}) supersedes an earlier position in the lead. ` +
|
|
1160
|
+
`Resolve the lead to the user's CURRENT position in the PRESENT TENSE and cite it [^${n}]. ` +
|
|
1161
|
+
`Keep it a clean current-state summary: do NOT scatter "previously …" caveats through the lead — the trajectory is recorded separately in a History section. ` +
|
|
1162
|
+
`KEEP the superseded claim's existing [^k] citation marker somewhere in the lead so no [^N] is lost (provenance survives), but frame the lead around the current value, not the change.\n\n`
|
|
1163
|
+
);
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
/** The schematic LEAD-PATCH prompt: the article's current lead + the chunk + the
|
|
1167
|
+
* inlined footnote/correction contract. The lead is where an entity's headline
|
|
1168
|
+
* verdict lives, so the CORRECTION rule is the load-bearing one. When a `kind` is
|
|
1169
|
+
* classified, the lead resolves to a CLEAN current-state summary (evolution) or a
|
|
1170
|
+
* silent overwrite (error); otherwise the default provenance-preserving fold runs. */
|
|
1171
|
+
export function buildLeadPatchPrompt(
|
|
1172
|
+
title: string,
|
|
1173
|
+
currentLead: string,
|
|
1174
|
+
chunk: SynthesisChunk,
|
|
1175
|
+
n: number,
|
|
1176
|
+
otherArticles: string,
|
|
1177
|
+
policy: string,
|
|
1178
|
+
priorDates: string[] = [],
|
|
1179
|
+
kind?: ContradictionKind,
|
|
1180
|
+
): string {
|
|
1181
|
+
const chunkDate = footnoteDate(chunk.dateMs);
|
|
1182
|
+
return (
|
|
1183
|
+
`${policy}\n\n${NO_COPY_GUARD}\n\n${WIKILINK_RULES}\n\n` +
|
|
1184
|
+
`You are updating the LEAD (the opening summary) of the personal-wiki article about "${title}" with ONE new note (dated ${chunkDate}). ` +
|
|
1185
|
+
`The lead names **${title}** in bold and states what it is and how the user relates to it. ` +
|
|
1186
|
+
`Keep every existing [^k] citation marker that still applies; weave in only what the new note adds.\n\n` +
|
|
1187
|
+
`CITATION — cite the new note by placing the marker [^${n}] immediately after the specific claim it supports. ` +
|
|
1188
|
+
`Reuse any [^k] markers already in the lead exactly where they are. ` +
|
|
1189
|
+
`Do NOT write any [^k]: definition line — definitions are generated for you. ` +
|
|
1190
|
+
`[^N] is the only citation syntax; never emit HTML, bare links, or invent another number.\n\n` +
|
|
1191
|
+
(kind ? leadCorrectionRule(kind, n, chunkDate) : correctionRule(n, chunkDate, priorDates)) +
|
|
1192
|
+
`OTHER ARTICLES (link to these stems only):\n${otherArticles}\n\n` +
|
|
1193
|
+
`CURRENT LEAD:\n${currentLead || "(empty)"}\n\n` +
|
|
1194
|
+
`NEW NOTE (${chunkDate})${chunk.label ? ` — ${chunk.label}` : ""}:\n${chunk.text.slice(0, 1400).trim()}\n\n` +
|
|
1195
|
+
`Output ONLY the rewritten lead prose — 2–4 sentences, no heading, no list, no References section.`
|
|
1196
|
+
);
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
export interface PatchInput {
|
|
1200
|
+
/** The full current article markdown. */
|
|
1201
|
+
article: string;
|
|
1202
|
+
/** The target section's anchor (a `chunk_sections` row's section_anchor), or
|
|
1203
|
+
* {@link LEAD_ANCHOR} to fold into the article's opening lead. */
|
|
1204
|
+
anchor: string;
|
|
1205
|
+
/** The chunk to integrate (text + footnote metadata). */
|
|
1206
|
+
chunk: SynthesisChunk;
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
export interface PatchDeps {
|
|
1210
|
+
/** Model call; defaults to `callLocal("synthesis", …)` (same stage as CREATE). */
|
|
1211
|
+
call?: SynthesisCall;
|
|
1212
|
+
/** Other article stems for wikilink resolution. */
|
|
1213
|
+
stems?: Set<string>;
|
|
1214
|
+
/** Article stem (H1/NORMALIZE); defaults to the entity stem of the H1 title. */
|
|
1215
|
+
stem?: string;
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
export interface PatchOutcome {
|
|
1219
|
+
anchor: string;
|
|
1220
|
+
/** Final post-NORMALIZE article, or null when the patch was a NO-OP. */
|
|
1221
|
+
content: string | null;
|
|
1222
|
+
action: "patched" | "skipped";
|
|
1223
|
+
/** Skip reason; null on success. */
|
|
1224
|
+
reason: string | null;
|
|
1225
|
+
/** The `[^N]` number assigned to the chunk (null when never reached). */
|
|
1226
|
+
footnote: number | null;
|
|
1227
|
+
/** True when the chunk's conversation was already cited (reused marker, no new def). */
|
|
1228
|
+
reusedFootnote: boolean;
|
|
1229
|
+
/** True when a new `[^N]:` definition line was appended. */
|
|
1230
|
+
addedDef: boolean;
|
|
1231
|
+
normalizeNotes: string[];
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
/**
|
|
1235
|
+
* Fold ONE chunk into ONE existing section of an article, end to end:
|
|
1236
|
+
* assign the chunk's `[^N]` → draft the integrated body (leak/uncited → retry →
|
|
1237
|
+
* NO-OP) → swap ONLY that section's bytes → append the generated `[^N]:` def →
|
|
1238
|
+
* NORMALIZE → gate. Returns the patched article (or null when the gate vetoes a
|
|
1239
|
+
* weak pass). Pure with respect to disk — the caller persists.
|
|
1240
|
+
*/
|
|
1241
|
+
export async function patchSection(input: PatchInput, deps: PatchDeps = {}): Promise<PatchOutcome> {
|
|
1242
|
+
const call: SynthesisCall = deps.call ?? ((p, o) => callLocal("synthesis", { user: p }, o));
|
|
1243
|
+
const { article, anchor, chunk } = input;
|
|
1244
|
+
// The LEAD is patched by swapping exactly the lead bytes (no `## heading`).
|
|
1245
|
+
if (anchor === LEAD_ANCHOR) return patchLead(article, chunk, deps, call);
|
|
1246
|
+
const skip = (reason: string, footnote: number | null = null, reusedFootnote = false, addedDef = false): PatchOutcome => ({
|
|
1247
|
+
anchor,
|
|
1248
|
+
content: null,
|
|
1249
|
+
action: "skipped",
|
|
1250
|
+
reason,
|
|
1251
|
+
footnote,
|
|
1252
|
+
reusedFootnote,
|
|
1253
|
+
addedDef,
|
|
1254
|
+
normalizeNotes: [],
|
|
1255
|
+
});
|
|
1256
|
+
|
|
1257
|
+
const block = extractSection(article, anchor);
|
|
1258
|
+
if (block == null) return skip(`section not found: ${anchor}`);
|
|
1259
|
+
const headingLine = block.split("\n")[0] ?? `## ${anchor}`;
|
|
1260
|
+
const heading = headingLine.replace(/^#{1,6}\s+/, "").trim();
|
|
1261
|
+
const currentBody = block.split("\n").slice(1).join("\n").trim();
|
|
1262
|
+
|
|
1263
|
+
const title = articleH1Title(article);
|
|
1264
|
+
const stem = deps.stem ?? entityStem(title);
|
|
1265
|
+
const stems = new Set(deps.stems ?? []);
|
|
1266
|
+
stems.add(stem);
|
|
1267
|
+
const otherArticles = otherArticlesBlock(stems, stem);
|
|
1268
|
+
const policy = loadMetaPolicyQuiet(["Article_Conventions"]);
|
|
1269
|
+
|
|
1270
|
+
// Assign the chunk's footnote: reuse the conversation's existing marker, else
|
|
1271
|
+
// mint maxN+1 (keeps the article bijective+contiguous → NORMALIZE no-ops).
|
|
1272
|
+
const state = articleFootnoteState(article);
|
|
1273
|
+
const hash = convHash(chunk.conv);
|
|
1274
|
+
const reused = state.byHash.get(hash);
|
|
1275
|
+
const n = reused ?? state.maxN + 1;
|
|
1276
|
+
const addedDef = reused === undefined;
|
|
1277
|
+
|
|
1278
|
+
// The dates of the claims this section already cites — so a contradicting fold
|
|
1279
|
+
// can be told it is the LATER statement and resolve toward itself (provenance
|
|
1280
|
+
// preserved: it keeps the old [^k] marker).
|
|
1281
|
+
const refDates = referenceDatesByMarker(article);
|
|
1282
|
+
const priorDates = markersInBlock(currentBody)
|
|
1283
|
+
.map((m) => refDates.get(m))
|
|
1284
|
+
.filter((d): d is string => d != null);
|
|
1285
|
+
|
|
1286
|
+
const newBody = await draftPatchedSection(
|
|
1287
|
+
call,
|
|
1288
|
+
buildPatchPrompt(title, heading, currentBody, chunk, n, otherArticles, policy, priorDates),
|
|
1289
|
+
n,
|
|
1290
|
+
heading,
|
|
1291
|
+
);
|
|
1292
|
+
// Empty after one retry = leaky/uncited → NO-OP (vault untouched, chunk retries).
|
|
1293
|
+
if (!newBody.trim()) return skip("patched section leaked, refused, or did not cite the new note (NO-OP)", n, reused !== undefined, addedDef);
|
|
1294
|
+
|
|
1295
|
+
let next = replaceSection(article, anchor, `${headingLine}\n\n${newBody}\n`);
|
|
1296
|
+
if (next == null) return skip(`section vanished on swap: ${anchor}`, n, reused !== undefined, addedDef);
|
|
1297
|
+
if (addedDef) next = appendReferenceDef(next, footnoteDefLine(n, chunk.conv, chunk.title, chunk.dateMs));
|
|
1298
|
+
|
|
1299
|
+
const norm = normalizeArticle(next, { stem, stems });
|
|
1300
|
+
|
|
1301
|
+
// Footnote bijection must hold (a hallucinated marker number the gate cannot
|
|
1302
|
+
// repair → NO-OP rather than a citation-broken commit).
|
|
1303
|
+
const fn = checkFootnoteIntegrity(norm.content);
|
|
1304
|
+
if (!fn.ok) return skip(`footnote bijection unrepairable: ${fn.errors.join("; ")}`, n, reused !== undefined, addedDef);
|
|
1305
|
+
|
|
1306
|
+
// The conservative gate over (before = original article, after = patched).
|
|
1307
|
+
const verdict: GateVerdict = gateEdit(article, norm.content);
|
|
1308
|
+
if (!verdict.ok) return skip(verdict.reason ?? "gate veto", n, reused !== undefined, addedDef);
|
|
1309
|
+
|
|
1310
|
+
return {
|
|
1311
|
+
anchor,
|
|
1312
|
+
content: norm.content,
|
|
1313
|
+
action: "patched",
|
|
1314
|
+
reason: null,
|
|
1315
|
+
footnote: n,
|
|
1316
|
+
reusedFootnote: reused !== undefined,
|
|
1317
|
+
addedDef,
|
|
1318
|
+
normalizeNotes: norm.notes,
|
|
1319
|
+
};
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
/**
|
|
1323
|
+
* Fold ONE chunk into the article's LEAD (its opening summary), swapping exactly
|
|
1324
|
+
* the lead bytes — every section and the References stay byte-identical except for
|
|
1325
|
+
* the appended def. Same assign-footnote → draft (leak/uncited → retry → NO-OP) →
|
|
1326
|
+
* NORMALIZE → gate contract as {@link patchSection}, but the lead has no `##`
|
|
1327
|
+
* heading so we locate it via the structural skeleton and replace those lines.
|
|
1328
|
+
*/
|
|
1329
|
+
async function patchLead(
|
|
1330
|
+
article: string,
|
|
1331
|
+
chunk: SynthesisChunk,
|
|
1332
|
+
deps: PatchDeps,
|
|
1333
|
+
call: SynthesisCall,
|
|
1334
|
+
): Promise<PatchOutcome> {
|
|
1335
|
+
const skip = (reason: string, footnote: number | null = null, reusedFootnote = false, addedDef = false): PatchOutcome => ({
|
|
1336
|
+
anchor: LEAD_ANCHOR,
|
|
1337
|
+
content: null,
|
|
1338
|
+
action: "skipped",
|
|
1339
|
+
reason,
|
|
1340
|
+
footnote,
|
|
1341
|
+
reusedFootnote,
|
|
1342
|
+
addedDef,
|
|
1343
|
+
normalizeNotes: [],
|
|
1344
|
+
});
|
|
1345
|
+
|
|
1346
|
+
const leadItem = articleStructure(article).find((it) => it.kind === "lead");
|
|
1347
|
+
if (!leadItem) return skip("article has no lead to patch");
|
|
1348
|
+
const lines = article.split("\n");
|
|
1349
|
+
const start = leadItem.startLine - 1;
|
|
1350
|
+
const end = leadItem.endLine - 1; // 1-based inclusive → 0-based inclusive
|
|
1351
|
+
const currentLead = lines.slice(start, end + 1).join("\n").trim();
|
|
1352
|
+
|
|
1353
|
+
const title = articleH1Title(article);
|
|
1354
|
+
const stem = deps.stem ?? entityStem(title);
|
|
1355
|
+
const stems = new Set(deps.stems ?? []);
|
|
1356
|
+
stems.add(stem);
|
|
1357
|
+
const otherArticles = otherArticlesBlock(stems, stem);
|
|
1358
|
+
const policy = loadMetaPolicyQuiet(["Article_Conventions"]);
|
|
1359
|
+
|
|
1360
|
+
// Assign the chunk's footnote exactly as the section path does.
|
|
1361
|
+
const state = articleFootnoteState(article);
|
|
1362
|
+
const hash = convHash(chunk.conv);
|
|
1363
|
+
const reused = state.byHash.get(hash);
|
|
1364
|
+
const n = reused ?? state.maxN + 1;
|
|
1365
|
+
const addedDef = reused === undefined;
|
|
1366
|
+
|
|
1367
|
+
const refDates = referenceDatesByMarker(article);
|
|
1368
|
+
const priorMarkers = markersInBlock(currentLead);
|
|
1369
|
+
const priorDates = priorMarkers
|
|
1370
|
+
.map((m) => refDates.get(m))
|
|
1371
|
+
.filter((d): d is string => d != null);
|
|
1372
|
+
|
|
1373
|
+
// CLASSIFY the contradiction when the lead already cites a prior claim the new
|
|
1374
|
+
// note may conflict with: is the user CHANGING THEIR OWN view (evolution — the
|
|
1375
|
+
// lead resolves to current-state, the trajectory preserved in History by the
|
|
1376
|
+
// reconcile pass) or correcting a WRONG fact (error — silent overwrite, the
|
|
1377
|
+
// wrong value's citation may be pruned)?
|
|
1378
|
+
let kind: ContradictionKind | undefined;
|
|
1379
|
+
const allowDroppedHashes = new Set<string>();
|
|
1380
|
+
if (priorMarkers.length) {
|
|
1381
|
+
kind = classifyContradiction(
|
|
1382
|
+
await call(buildContradictionClassifierPrompt(currentLead, chunk.text.slice(0, 1400).trim()), { maxTokens: MAX_OUTPUT_TOKENS }),
|
|
1383
|
+
);
|
|
1384
|
+
if (kind === "error") {
|
|
1385
|
+
// A factual fix may drop the wrong value's citation. Permit dropping the
|
|
1386
|
+
// hashes the corrected lead cited (bounded to the lead's own prior claims).
|
|
1387
|
+
const hashByMarker = new Map<number, string>();
|
|
1388
|
+
for (const [h, m] of state.byHash) hashByMarker.set(m, h);
|
|
1389
|
+
for (const m of priorMarkers) {
|
|
1390
|
+
const h = hashByMarker.get(m);
|
|
1391
|
+
if (h) allowDroppedHashes.add(h);
|
|
1392
|
+
}
|
|
1393
|
+
}
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
const newLead = await draftPatchedProse(
|
|
1397
|
+
call,
|
|
1398
|
+
buildLeadPatchPrompt(title, currentLead, chunk, n, otherArticles, policy, priorDates, kind),
|
|
1399
|
+
n,
|
|
1400
|
+
sanitizeLead,
|
|
1401
|
+
);
|
|
1402
|
+
if (!newLead.trim()) return skip("patched lead leaked, refused, or did not cite the new note (NO-OP)", n, reused !== undefined, addedDef);
|
|
1403
|
+
|
|
1404
|
+
// Swap exactly the lead lines; every other byte is preserved.
|
|
1405
|
+
let next = [...lines.slice(0, start), newLead, ...lines.slice(end + 1)].join("\n");
|
|
1406
|
+
if (addedDef) next = appendReferenceDef(next, footnoteDefLine(n, chunk.conv, chunk.title, chunk.dateMs));
|
|
1407
|
+
|
|
1408
|
+
const norm = normalizeArticle(next, { stem, stems });
|
|
1409
|
+
|
|
1410
|
+
const fn = checkFootnoteIntegrity(norm.content);
|
|
1411
|
+
if (!fn.ok) return skip(`footnote bijection unrepairable: ${fn.errors.join("; ")}`, n, reused !== undefined, addedDef);
|
|
1412
|
+
|
|
1413
|
+
const verdict: GateVerdict = gateEdit(article, norm.content, { allowDroppedHashes });
|
|
1414
|
+
if (!verdict.ok) return skip(verdict.reason ?? "gate veto", n, reused !== undefined, addedDef);
|
|
1415
|
+
|
|
1416
|
+
return {
|
|
1417
|
+
anchor: LEAD_ANCHOR,
|
|
1418
|
+
content: norm.content,
|
|
1419
|
+
action: "patched",
|
|
1420
|
+
reason: null,
|
|
1421
|
+
footnote: n,
|
|
1422
|
+
reusedFootnote: reused !== undefined,
|
|
1423
|
+
addedDef,
|
|
1424
|
+
normalizeNotes: norm.notes,
|
|
1425
|
+
};
|
|
1426
|
+
}
|
|
1427
|
+
|
|
1428
|
+
// ---- effectful entry: PATCH a vault article + ledger -----------------------
|
|
1429
|
+
|
|
1430
|
+
export interface SynthesizePatchOpts {
|
|
1431
|
+
/** The article to patch (its filename stem). */
|
|
1432
|
+
stem: string;
|
|
1433
|
+
/** The section anchor to fold into (a `chunk_sections` row). */
|
|
1434
|
+
anchor: string;
|
|
1435
|
+
/** The chunk id to integrate. */
|
|
1436
|
+
chunkId: string;
|
|
1437
|
+
/** Vault root (honors MLX_BUN_WIKI); defaults to `vaultRoot()`. */
|
|
1438
|
+
root?: string;
|
|
1439
|
+
/** Model-call override (tests inject a fake). */
|
|
1440
|
+
call?: SynthesisCall;
|
|
1441
|
+
/** Skip the git commit (tests). */
|
|
1442
|
+
commit?: boolean;
|
|
1443
|
+
now?: number;
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
export interface PatchResult {
|
|
1447
|
+
patched: boolean;
|
|
1448
|
+
stem: string;
|
|
1449
|
+
anchor: string;
|
|
1450
|
+
/** The gate (or a draft NO-OP) vetoed the write. */
|
|
1451
|
+
skippedByGate: boolean;
|
|
1452
|
+
/** The (chunk, article, section) edge was already folded (idempotent NO-OP). */
|
|
1453
|
+
alreadyIntegrated: boolean;
|
|
1454
|
+
reason: string | null;
|
|
1455
|
+
footnote: number | null;
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
/**
|
|
1459
|
+
* PATCH one chunk into one section of a vault article and persist it. Idempotent
|
|
1460
|
+
* BEFORE the model: a present `synthesized_chunk_sections` PK row short-circuits
|
|
1461
|
+
* to a NO-OP (no GPU, no write). Otherwise drafts via {@link patchSection}, writes
|
|
1462
|
+
* the swapped article (unless the gate vetoed), records the integration edge, and
|
|
1463
|
+
* commits.
|
|
1464
|
+
*/
|
|
1465
|
+
export async function synthesizePatch(store: MemoryStore, opts: SynthesizePatchOpts): Promise<PatchResult> {
|
|
1466
|
+
const root = opts.root ?? vaultRoot();
|
|
1467
|
+
const base: PatchResult = {
|
|
1468
|
+
patched: false,
|
|
1469
|
+
stem: opts.stem,
|
|
1470
|
+
anchor: opts.anchor,
|
|
1471
|
+
skippedByGate: false,
|
|
1472
|
+
alreadyIntegrated: false,
|
|
1473
|
+
reason: null,
|
|
1474
|
+
footnote: null,
|
|
1475
|
+
};
|
|
1476
|
+
|
|
1477
|
+
// Idempotency: the (chunk, article, section) PK blocks a double-fold up front.
|
|
1478
|
+
const already = store.db
|
|
1479
|
+
.query("SELECT 1 FROM synthesized_chunk_sections WHERE chunk_id = ? AND article_stem = ? AND section_anchor = ?")
|
|
1480
|
+
.get(opts.chunkId, opts.stem, opts.anchor);
|
|
1481
|
+
if (already) return { ...base, alreadyIntegrated: true, reason: "already integrated (synthesized_chunk_sections PK)" };
|
|
1482
|
+
|
|
1483
|
+
const path = join(articlesDir(root), `${opts.stem}.md`);
|
|
1484
|
+
let article: string;
|
|
1485
|
+
try {
|
|
1486
|
+
article = await readFile(path, "utf8");
|
|
1487
|
+
} catch {
|
|
1488
|
+
return { ...base, reason: `article not found: ${opts.stem}` };
|
|
1489
|
+
}
|
|
1490
|
+
|
|
1491
|
+
const chunk = loadSynthesisChunk(store, opts.chunkId);
|
|
1492
|
+
if (!chunk) return { ...base, reason: `chunk not found: ${opts.chunkId}` };
|
|
1493
|
+
|
|
1494
|
+
const stems = new Set(await listArticles(root));
|
|
1495
|
+
const outcome = await patchSection({ article, anchor: opts.anchor, chunk }, { call: opts.call, stems, stem: opts.stem });
|
|
1496
|
+
|
|
1497
|
+
if (outcome.action !== "patched" || outcome.content == null) {
|
|
1498
|
+
return { ...base, skippedByGate: true, reason: outcome.reason, footnote: outcome.footnote };
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1501
|
+
await writeFile(path, outcome.content);
|
|
1502
|
+
|
|
1503
|
+
const now = opts.now ?? Date.now();
|
|
1504
|
+
const insSec = store.db.query(
|
|
1505
|
+
"INSERT OR IGNORE INTO chunk_sections (chunk_id, article_stem, section_anchor) VALUES (?, ?, ?)",
|
|
1506
|
+
);
|
|
1507
|
+
const insSynth = store.db.query(
|
|
1508
|
+
"INSERT OR REPLACE INTO synthesized_chunk_sections " +
|
|
1509
|
+
"(chunk_id, article_stem, section_anchor, synthesized_at) VALUES (?, ?, ?, ?)",
|
|
1510
|
+
);
|
|
1511
|
+
const tx = store.db.transaction(() => {
|
|
1512
|
+
insSec.run(opts.chunkId, opts.stem, opts.anchor);
|
|
1513
|
+
insSynth.run(opts.chunkId, opts.stem, opts.anchor, now);
|
|
1514
|
+
});
|
|
1515
|
+
tx();
|
|
1516
|
+
|
|
1517
|
+
if (opts.commit !== false) {
|
|
1518
|
+
await commitVault(root, `memory: patch ${opts.stem} #${opts.anchor} (+chunk ${opts.chunkId})`);
|
|
1519
|
+
}
|
|
1520
|
+
|
|
1521
|
+
return { ...base, patched: true, footnote: outcome.footnote };
|
|
1522
|
+
}
|
|
1523
|
+
|
|
1524
|
+
// ===========================================================================
|
|
1525
|
+
// NEW SECTION — the third SECTION-ROUTE outcome (P7): a chunk that fits NO
|
|
1526
|
+
// existing section AND is not a verdict-level correction earns its OWN section.
|
|
1527
|
+
//
|
|
1528
|
+
// SECTION-ROUTE returns a NAMED new section for a substantive chunk that matched
|
|
1529
|
+
// nothing; honoring it (rather than dropping the chunk) is the difference between
|
|
1530
|
+
// self-healing and silent loss. We mint an EMPTY `## Title` section, then run the
|
|
1531
|
+
// SAME bounded fold as PATCH against it — so the new section is drafted, cited,
|
|
1532
|
+
// gated, and normalized exactly like every other integration, never a raw dump.
|
|
1533
|
+
// ===========================================================================
|
|
1534
|
+
|
|
1535
|
+
/** Insert an empty `## title` section into the article just before its `## See
|
|
1536
|
+
* also` / `## References` tail (or at the end when neither exists), so a fresh
|
|
1537
|
+
* topical section lands among the body sections, not after the references. The
|
|
1538
|
+
* body is left empty — {@link patchSection} fills it by folding the chunk in. */
|
|
1539
|
+
export function insertEmptySection(article: string, title: string): string {
|
|
1540
|
+
const lines = article.replace(/\s+$/, "").split("\n");
|
|
1541
|
+
const idx = lines.findIndex((l) => /^##\s+(see also|references)\s*$/i.test(l.trim()));
|
|
1542
|
+
if (idx < 0) return [...lines, "", `## ${title}`, ""].join("\n") + "\n";
|
|
1543
|
+
return [...lines.slice(0, idx), `## ${title}`, "", "", ...lines.slice(idx)].join("\n") + "\n";
|
|
1544
|
+
}
|
|
1545
|
+
|
|
1546
|
+
export interface SynthesizeNewSectionOpts {
|
|
1547
|
+
/** The article to grow (its filename stem). */
|
|
1548
|
+
stem: string;
|
|
1549
|
+
/** The new section's display title. */
|
|
1550
|
+
title: string;
|
|
1551
|
+
/** The new section's anchor (slug of `title`); also the ledger key. */
|
|
1552
|
+
anchor: string;
|
|
1553
|
+
/** The chunk id to fold into the new section. */
|
|
1554
|
+
chunkId: string;
|
|
1555
|
+
/** Vault root (honors MLX_BUN_WIKI); defaults to `vaultRoot()`. */
|
|
1556
|
+
root?: string;
|
|
1557
|
+
/** Model-call override (tests inject a fake). */
|
|
1558
|
+
call?: SynthesisCall;
|
|
1559
|
+
/** Skip the git commit (tests). */
|
|
1560
|
+
commit?: boolean;
|
|
1561
|
+
now?: number;
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
/**
|
|
1565
|
+
* CREATE a new section in an existing article and fold ONE chunk into it. Mints
|
|
1566
|
+
* the `## Title` heading then drafts its body via {@link patchSection} (so the
|
|
1567
|
+
* fold is cited, gated, NORMALIZED, and footnote-bijective just like a PATCH),
|
|
1568
|
+
* writes the grown article, and records the (chunk, section) edge. Idempotent via
|
|
1569
|
+
* `synthesized_chunk_sections`'s PK — a re-run of the same (chunk, article,
|
|
1570
|
+
* section) writes nothing. Used by SYNTHESIZE when SECTION-ROUTE returns a
|
|
1571
|
+
* `newSection` for a chunk that fit no existing section.
|
|
1572
|
+
*/
|
|
1573
|
+
export async function synthesizeNewSection(
|
|
1574
|
+
store: MemoryStore,
|
|
1575
|
+
opts: SynthesizeNewSectionOpts,
|
|
1576
|
+
): Promise<PatchResult> {
|
|
1577
|
+
const root = opts.root ?? vaultRoot();
|
|
1578
|
+
const base: PatchResult = {
|
|
1579
|
+
patched: false,
|
|
1580
|
+
stem: opts.stem,
|
|
1581
|
+
anchor: opts.anchor,
|
|
1582
|
+
skippedByGate: false,
|
|
1583
|
+
alreadyIntegrated: false,
|
|
1584
|
+
reason: null,
|
|
1585
|
+
footnote: null,
|
|
1586
|
+
};
|
|
1587
|
+
|
|
1588
|
+
const already = store.db
|
|
1589
|
+
.query("SELECT 1 FROM synthesized_chunk_sections WHERE chunk_id = ? AND article_stem = ? AND section_anchor = ?")
|
|
1590
|
+
.get(opts.chunkId, opts.stem, opts.anchor);
|
|
1591
|
+
if (already) return { ...base, alreadyIntegrated: true, reason: "already integrated (synthesized_chunk_sections PK)" };
|
|
1592
|
+
|
|
1593
|
+
const path = join(articlesDir(root), `${opts.stem}.md`);
|
|
1594
|
+
let article: string;
|
|
1595
|
+
try {
|
|
1596
|
+
article = await readFile(path, "utf8");
|
|
1597
|
+
} catch {
|
|
1598
|
+
return { ...base, reason: `article not found: ${opts.stem}` };
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
const chunk = loadSynthesisChunk(store, opts.chunkId);
|
|
1602
|
+
if (!chunk) return { ...base, reason: `chunk not found: ${opts.chunkId}` };
|
|
1603
|
+
|
|
1604
|
+
// Mint the empty section unless the article already carries it (idempotent
|
|
1605
|
+
// across the same name), then fold the chunk into it via the PATCH op.
|
|
1606
|
+
const withSection = extractSection(article, opts.anchor) != null ? article : insertEmptySection(article, opts.title);
|
|
1607
|
+
const stems = new Set(await listArticles(root));
|
|
1608
|
+
const outcome = await patchSection(
|
|
1609
|
+
{ article: withSection, anchor: opts.anchor, chunk },
|
|
1610
|
+
{ call: opts.call, stems, stem: opts.stem },
|
|
1611
|
+
);
|
|
1612
|
+
|
|
1613
|
+
if (outcome.action !== "patched" || outcome.content == null) {
|
|
1614
|
+
return { ...base, skippedByGate: true, reason: outcome.reason, footnote: outcome.footnote };
|
|
1615
|
+
}
|
|
1616
|
+
|
|
1617
|
+
await writeFile(path, outcome.content);
|
|
1618
|
+
|
|
1619
|
+
const now = opts.now ?? Date.now();
|
|
1620
|
+
const insSec = store.db.query(
|
|
1621
|
+
"INSERT OR IGNORE INTO chunk_sections (chunk_id, article_stem, section_anchor) VALUES (?, ?, ?)",
|
|
1622
|
+
);
|
|
1623
|
+
const insSynth = store.db.query(
|
|
1624
|
+
"INSERT OR REPLACE INTO synthesized_chunk_sections " +
|
|
1625
|
+
"(chunk_id, article_stem, section_anchor, synthesized_at) VALUES (?, ?, ?, ?)",
|
|
1626
|
+
);
|
|
1627
|
+
const tx = store.db.transaction(() => {
|
|
1628
|
+
insSec.run(opts.chunkId, opts.stem, opts.anchor);
|
|
1629
|
+
insSynth.run(opts.chunkId, opts.stem, opts.anchor, now);
|
|
1630
|
+
});
|
|
1631
|
+
tx();
|
|
1632
|
+
|
|
1633
|
+
if (opts.commit !== false) {
|
|
1634
|
+
await commitVault(root, `memory: new section ${opts.stem} #${opts.anchor} (+chunk ${opts.chunkId})`);
|
|
1635
|
+
}
|
|
1636
|
+
|
|
1637
|
+
return { ...base, patched: true, footnote: outcome.footnote };
|
|
29
1638
|
}
|