dsh-reference-session-collector 0.1.0 → 0.2.0

package/lib/index.js

@@ -1,9 +1,9 @@
 import { DatabaseSync } from "node:sqlite";
 import { existsSync, mkdirSync, readFileSync, readdirSync, statSync } from "node:fs";
 import { join } from "node:path";
+import { createHash } from "node:crypto";
 import { homedir } from "node:os";
 import zlib from "node:zlib";
-import { createHash } from "node:crypto";
 import { Tokenizer } from "@huggingface/tokenizers";
 //#region src/segment.ts
 /**
@@ -159,16 +159,56 @@ function parseTurnRange(raw) {
 	};
 }
 /**
-* segments 明细。
-* @param turn - `turn=` 区间过滤（`a-b` 闭区间或单值）。
-* @param fields - `'search'` 返检索面字段（time/search_text/on_surface）。
+* v5 读侧列集：hash 引用列名对（缺省原文臂 / redact=1 脱敏臂）——
+* 常数字面量拼入 SQL，无注入面（R3「redact=1 切 redacted 列」）。
+*/
+const PLAIN_FACE = {
+	textHash: "text_hash",
+	searchTextHash: "search_text_hash"
+};
+const REDACTED_FACE = {
+	textHash: "redacted_text_hash",
+	searchTextHash: "redacted_search_text_hash"
+};
+/** v4 segments DDL 15 列的还原 SELECT（v5 逐字段同形回归地基；列序一致；
+* 取材列对 bt/bs/hb 由 faceJoins 按 redact 臂选定，本模板不感知）。 */
+const DETAIL_COLUMNS = `s.session_id, s.seq, s.turn, s.step, s.kind, s.role, s.time,
+    COALESCE(bt.body, '') AS text, s.tokens,
+    COALESCE(bs.body, '') AS search_text,
+    s.on_surface, s.compressed,
+    CASE WHEN s.compressed = 1 THEN COALESCE(hb.body, '') ELSE NULL END AS surface_text,
+    s.surface_tokens, s.replaced_by`;
+/** blocks/链头两跳 JOIN 子句（replaced_by 恒带 session_id，撞键防御）。 */
+function faceJoins(face) {
+	return `FROM segments s
+    LEFT JOIN blocks bt ON bt.hash = s.${face.textHash}
+    LEFT JOIN blocks bs ON bs.hash = s.${face.searchTextHash}
+    LEFT JOIN segments h ON h.session_id = s.session_id AND h.seq = s.replaced_by AND s.compressed = 1
+    LEFT JOIN blocks hb ON hb.hash = h.${face.textHash}`;
+}
+/**
+* segments 明细（v4 逐字段同形还原）。
+* @param opts.turn - `turn=` 区间过滤（`a-b` 闭区间或单值）。
+* @param opts.fields - `'search'` 返检索面字段（time/search_text/on_surface）。
+* @param opts.tail - 尾窗 N：先取 seq 降序 N 行再翻升序（与会话文件真实尾部对拍）。
+* @param opts.redact - 脱敏臂：text/search_text/surface_text 取 redacted_* 列。
 */
 function sessionDetail(ctx, sessionId, opts = {}) {
 	const store = ctx.store;
 	const session = store.summaries.get(sessionId);
+	const face = opts.redact === true ? REDACTED_FACE : PLAIN_FACE;
 	const range = parseTurnRange(opts.turn);
-	const where = range === void 0 ? "SELECT * FROM segments WHERE session_id=? ORDER BY seq" : "SELECT * FROM segments WHERE session_id=? AND turn BETWEEN ? AND ? ORDER BY seq";
-	const rows = range === void 0 ? store.db.prepare(where).all(sessionId) : store.db.prepare(where).all(sessionId, range.min, range.max);
+	const tail = typeof opts.tail === "number" && Number.isFinite(opts.tail) && opts.tail > 0 ? Math.floor(opts.tail) : void 0;
+	const where = range === void 0 ? "WHERE s.session_id=?" : "WHERE s.session_id=? AND s.turn BETWEEN ? AND ?";
+	const params = range === void 0 ? [sessionId] : [
+		sessionId,
+		range.min,
+		range.max
+	];
+	const order = tail === void 0 ? "ORDER BY s.seq ASC" : "ORDER BY s.seq DESC LIMIT ?";
+	if (tail !== void 0) params.push(tail);
+	const fetched = store.db.prepare(`SELECT ${DETAIL_COLUMNS} ${faceJoins(face)} ${where} ${order}`).all(...params);
+	const rows = tail === void 0 ? fetched : fetched.reverse();
 	if (opts.fields !== "search") return {
 		session,
 		rows
@@ -194,8 +234,17 @@ function surfaceLedger(ctx, sessionId) {
 	if (cached !== void 0) return cached;
 	const value = {
 		session_id: sessionId,
-		on_surface: store.db.prepare("SELECT seq, turn, step, kind, role, time, tokens, text FROM segments WHERE session_id=? AND on_surface=1 ORDER BY seq").all(sessionId),
-		compressed_pointers: store.db.prepare("SELECT seq, replaced_by, surface_text, surface_tokens FROM segments WHERE session_id=? AND compressed=1 ORDER BY seq").all(sessionId)
+		on_surface: store.db.prepare(`SELECT s.seq, s.turn, s.step, s.kind, s.role, s.time, s.tokens,
+        COALESCE(b.body, '') AS text
+     FROM segments s
+     LEFT JOIN blocks b ON b.hash = s.text_hash
+     WHERE s.session_id=? AND s.on_surface=1 ORDER BY s.seq`).all(sessionId),
+		compressed_pointers: store.db.prepare(`SELECT s.seq, s.replaced_by,
+        COALESCE(hb.body, '') AS surface_text, s.surface_tokens
+     FROM segments s
+     LEFT JOIN segments h ON h.session_id = s.session_id AND h.seq = s.replaced_by
+     LEFT JOIN blocks hb ON hb.hash = h.text_hash
+     WHERE s.session_id=? AND s.compressed=1 ORDER BY s.seq`).all(sessionId)
 	};
 	ctx.surfaceLru.set(key, value);
 	return value;
@@ -239,7 +288,11 @@ function recallReport(ctx, sessionId) {
 	ctx.recallLru.set(key, value);
 	return value;
 }
-/** FTS5 BM25 内容搜索（中英同构切词，走 sanitizeFtsQuery）。 */
+/**
+* FTS5 BM25 内容搜索（中英同构切词，走 sanitizeFtsQuery）。
+* v5：fts 仍索引原文（R3）；snippet 经 blocks 还原，`redact` 只切 snippet
+* 取材列（redacted_search_text_hash），命中/排序语义两臂一致。
+*/
 function search(ctx, query, opts = {}) {
 	const match = sanitizeFtsQuery(query);
 	if (match === "") return {
@@ -249,9 +302,10 @@ function search(ctx, query, opts = {}) {
 	};
 	const limit = Math.min(Math.max(1, Math.floor(opts.limit ?? 20)), 100);
 	const sql = `SELECT segments_fts.session_id AS session_id, segments_fts.seq AS seq, segments_fts.kind AS kind,
-      bm25(segments_fts) AS rank, substr(s.search_text, 1, 200) AS snippet, s.time, s.on_surface, s.turn
+      bm25(segments_fts) AS rank, substr(COALESCE(b.body, ''), 1, 200) AS snippet, s.time, s.on_surface, s.turn
     FROM segments_fts
     JOIN segments s ON s.session_id = segments_fts.session_id AND s.seq = segments_fts.seq
+    LEFT JOIN blocks b ON b.hash = s.${(opts.redact === true ? REDACTED_FACE : PLAIN_FACE).searchTextHash}
     WHERE segments_fts MATCH ? ${opts.session !== void 0 && opts.session !== "" ? "AND segments_fts.session_id = ?" : ""}
     ORDER BY rank LIMIT ?`;
 	const stmt = ctx.store.db.prepare(sql);
@@ -297,6 +351,7 @@ function buildSearchSnippet(text, query, max = SEARCH_SNIPPET_CHARS) {
 * 会话分组全文搜索（op=search&group=session）：BM25 候选窗 → 按会话取
 * 加权最强命中 → 排序分页。数学/视界与 steward engine.search 逐字同构：
 * score = −rank × weight(type)，时间排序键是会话 updated_at（非命中 time）。
+* snippet 锚定文本经 blocks 还原（redact 臂切 redacted_search_text_hash）。
 */
 function searchGrouped(ctx, request) {
 	const match = sanitizeFtsQuery(request.query);
@@ -319,27 +374,31 @@ function searchGrouped(ctx, request) {
 		dateClauses.push("AND sess.last_time_ms < ?");
 		dateParams.push(request.to);
 	}
+	const dateClause = dateClauses.join(" ");
 	const sql = `SELECT seg.session_id AS session_id, seg.seq AS seq, seg.kind AS kind, seg.time AS time,
-      seg.search_text AS search_text, bm25(segments_fts) AS rank,
+      COALESCE(b.body, '') AS search_text, bm25(segments_fts) AS rank,
       sess.title AS title, sess.last_time_ms AS updated_at
     FROM segments_fts
     JOIN segments seg ON seg.session_id = segments_fts.session_id AND seg.seq = segments_fts.seq
+    LEFT JOIN blocks b ON b.hash = seg.${(request.redact === true ? REDACTED_FACE : PLAIN_FACE).searchTextHash}
     JOIN sessions sess ON sess.session_id = seg.session_id
     WHERE segments_fts MATCH ?
-      AND seg.kind IN (${placeholders}) ${surfaceClause} ${archivedClause} ${dateClauses.join(" ")}
+      AND seg.kind IN (${placeholders}) ${surfaceClause} ${archivedClause} ${dateClause}
     ORDER BY rank LIMIT ?`;
-	const docs = ctx.store.db.prepare(sql).all(match, ...kinds, ...dateParams, SEARCH_SCAN_LIMIT);
-	const bestBySession = /* @__PURE__ */ new Map();
-	for (const doc of docs) {
-		const weight = SEARCH_TYPE_WEIGHT[doc.kind] ?? 1;
-		const score = -Number(doc.rank) * weight;
-		const best = bestBySession.get(doc.session_id);
-		if (best === void 0 || score > best.score) bestBySession.set(doc.session_id, {
-			doc,
-			score
-		});
+	const scored = ctx.store.db.prepare(sql).all(match, ...kinds, ...dateParams, SEARCH_SCAN_LIMIT).map((doc) => ({
+		doc,
+		score: -Number(doc.rank) * (SEARCH_TYPE_WEIGHT[doc.kind] ?? 1)
+	}));
+	let grouped;
+	if (request.expand === true) grouped = scored;
+	else {
+		const bestBySession = /* @__PURE__ */ new Map();
+		for (const entry of scored) {
+			const best = bestBySession.get(entry.doc.session_id);
+			if (best === void 0 || entry.score > best.score) bestBySession.set(entry.doc.session_id, entry);
+		}
+		grouped = [...bestBySession.values()];
 	}
-	const grouped = [...bestBySession.values()];
 	if (request.sortBy === "time") grouped.sort((a, b) => b.doc.updated_at - a.doc.updated_at || b.score - a.score);
 	else grouped.sort((a, b) => b.score - a.score);
 	const hits = grouped.slice(0, limit).map(({ doc }) => ({
@@ -357,22 +416,301 @@ function searchGrouped(ctx, request) {
 	};
 }
 //#endregion
+//#region src/blocks.ts
+/**
+* 字块字典（schema v5，2026-10-20 用户定标「整条一起存相同字块」）：
+* segments 的 text/search_text/redacted_* 一律以 sha256 全串等值 hash 入
+* 本字典，行内只存引用；同文=同 hash=单份存储（INSERT OR IGNORE 幂等）。
+*
+* 选型依据（真机实测）：surface_text 1768 行仅 23 distinct（99.2%）；
+* text 11% / search_text 12% 整段重复；重复组字节跨度大→前缀键/分块切分
+* 会撕裂等值判定，全串 hash 是唯一可靠粒度。hash 用 node:crypto sha256
+* （零新增依赖；弃 blake3）。
+*
+* 纪律：字典只增不删（deleteSession 清行不清块——共享块归属不可知），
+* 投影可重建、体积有上界（语料 distinct 集）；GC 属明示不做项。
+*/
+/** blocks 表 DDL（单一事实源：schema.ts 与测试拼库都引本常量）。 */
+const BLOCKS_DDL = `CREATE TABLE IF NOT EXISTS blocks(
+  hash TEXT PRIMARY KEY, body TEXT NOT NULL, bytes INTEGER NOT NULL,
+  created_at TEXT DEFAULT (datetime('now')));`;
+/** sha256 hex（64 字符）——全串等值键。 */
+function hashOf(text) {
+	return createHash("sha256").update(text, "utf8").digest("hex");
+}
+var BlocksStore = class {
+	insert;
+	constructor(db) {
+		this.insert = db.prepare("INSERT OR IGNORE INTO blocks(hash, body, bytes) VALUES(?,?,?)");
+	}
+	/**
+	* 入字典并返回引用 hash；空文本不入典返回 null（hash 列存 NULL，读侧还原
+	* 为 ''——「未落行文本」与「空文本」在 v4 就靠此区分，语义不变）。
+	* 幂等：同文重复 ensure 只落一份（INSERT OR IGNORE + hash 主键）。
+	*/
+	ensure(text) {
+		if (text === "") return null;
+		const hash = hashOf(text);
+		this.insert.run(hash, text, Buffer.byteLength(text, "utf8"));
+		return hash;
+	}
+};
+//#endregion
+//#region src/redact.ts
+const PLACEHOLDER_VALUE_PATTERN = /^\[REDACTED:[A-Z_]+\]$/;
+const SENSITIVE_KEY_SOURCE = String.raw`(?:[A-Za-z_$][A-Za-z0-9_$-]*)?(?:password|passwd|pwd|client[-_$]?secret|consumer[-_$]?secret|api[-_$]?key|access[-_$]?token|refresh[-_$]?token|auth[-_$]?token|id[-_$]?token|private[-_$]?key|secret[-_$]?access[-_$]?key|secret[-_$]?key|credential|secret|token)`;
+const CREDENTIAL_VALUE_SOURCE = String.raw`(?:\[REDACTED:[A-Z_]+\]|\$\{\{[^\r\n]*?\}\}|\{\{[^\r\n]*?\}\}|\$\{[^}\r\n]+\}|\$[A-Za-z_][A-Za-z0-9_]*|"(?:\\.|[^"\\\r\n])*"|'(?:\\.|[^'\\\r\n])*'|[^,;#}\]\)\r\n]+)`;
+/** 安全值 allowlist:命中即放行(示例/占位/环境变量引用/类型名等,脱了只会毁记忆)。 */
+const SAFE_CREDENTIAL_VALUE_PATTERN = /^(?:\[REDACTED:[A-Z_]+\]|\$\{\{[^\r\n]*\}\}|\{\{[^\r\n]*\}\}|\$\{[A-Z_][A-Z0-9_]*\}|\$[A-Z_][A-Z0-9_]*|(?:process\.)?env\.[A-Z_][A-Z0-9_]*|<[^>]+>|--\S+|x{2,}|\*{2,}|\.{3}|sk[_-]x+|example|sample|change-?me|(?:your|replace)[-_][a-z0-9_-]+|string|str|number|boolean|unknown|any|null|undefined|none)$/i;
+const RULES = [
+	{
+		pattern: /-----BEGIN ((?:(?:RSA|EC|DSA|OPENSSH|ENCRYPTED) )?PRIVATE KEY|PGP PRIVATE KEY BLOCK)-----[\s\S]*?(?:-----END \1-----|$)/gi,
+		kind: "PRIVATE_KEY",
+		priority: 100
+	},
+	{
+		pattern: /(?:^|[\r\n])[ \t]*(?:Proxy-)?Authorization[ \t]*:[ \t]*([^ \t\r\n][^\r\n]*)/gim,
+		group: 1,
+		kind: "AUTH_TOKEN",
+		priority: 95
+	},
+	{
+		pattern: /(?:^|[^A-Za-z0-9_-])(?:Bearer|Basic)\s+["']?([A-Za-z0-9._~+/=-]{16,})(?![A-Za-z0-9._~+/=-])/gi,
+		group: 1,
+		kind: "AUTH_TOKEN",
+		priority: 94
+	},
+	{
+		pattern: /\beyJ[A-Za-z0-9_-]{5,}\.eyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{16,}\b/g,
+		kind: "AUTH_TOKEN",
+		priority: 94
+	},
+	{
+		pattern: /(?:^|[\r\n])[ \t]*(?:Set-Cookie|Cookie)\s*:\s*([^\r\n]+)/gim,
+		group: 1,
+		kind: "COOKIE",
+		priority: 92
+	},
+	{
+		pattern: new RegExp(String.raw`(?:^|[\r\n;,{}])[ \t]*(?:-[ \t]+)?(?:(?:export|const|let|var)[ \t]+)*["']?${SENSITIVE_KEY_SOURCE}["']?[ \t]*[:=][ \t]*(${CREDENTIAL_VALUE_SOURCE})`, "gim"),
+		group: 1,
+		kind: "CREDENTIAL",
+		priority: 85,
+		accept: (value) => !isSafeCredentialValue(value)
+	},
+	{
+		pattern: new RegExp(String.raw`--${SENSITIVE_KEY_SOURCE}(?![A-Za-z0-9_$-])(?:[ \t]+|=[ \t]*)(${CREDENTIAL_VALUE_SOURCE})`, "gi"),
+		group: 1,
+		kind: "CREDENTIAL",
+		priority: 85,
+		accept: (value) => !isSafeCredentialValue(value)
+	},
+	{
+		pattern: new RegExp(String.raw`[?&]${SENSITIVE_KEY_SOURCE}(?![A-Za-z0-9_$-])=([^&#\s"'<>]+)`, "gi"),
+		group: 1,
+		kind: "CREDENTIAL",
+		priority: 85,
+		accept: (value) => !isSafeCredentialValue(value)
+	},
+	{
+		pattern: /\b[A-Za-z][A-Za-z0-9+.-]*:\/\/[^\s/:@]*:([^\s/@]+)@/g,
+		group: 1,
+		kind: "CREDENTIAL",
+		priority: 85,
+		accept: (value) => !isSafeCredentialValue(value)
+	},
+	{
+		pattern: /\bsk-(?:proj-)?[A-Za-z0-9_-]{16,}[A-Za-z0-9]\b/g,
+		kind: "API_KEY",
+		priority: 80
+	},
+	{
+		pattern: /\bsk_[A-Za-z0-9_-]{8,}\b/g,
+		kind: "API_KEY",
+		priority: 80
+	},
+	{
+		pattern: /\b(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})\b/g,
+		kind: "API_KEY",
+		priority: 80
+	},
+	{
+		pattern: /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g,
+		kind: "API_KEY",
+		priority: 80
+	},
+	{
+		pattern: /\bxox[baprs]-[A-Za-z0-9-]{16,}[A-Za-z0-9]\b/g,
+		kind: "API_KEY",
+		priority: 80
+	},
+	{
+		pattern: /\b(?:(?:sk|rk)_live_[A-Za-z0-9]{16,}|whsec_[A-Za-z0-9]{16,})\b/g,
+		kind: "API_KEY",
+		priority: 80
+	},
+	{
+		pattern: /\bglpat-[A-Za-z0-9_-]{16,}[A-Za-z0-9]\b/g,
+		kind: "API_KEY",
+		priority: 80
+	},
+	{
+		pattern: /\bAIza[0-9A-Za-z_-]{35}\b/g,
+		kind: "API_KEY",
+		priority: 80
+	},
+	{
+		pattern: /\bnpm_[A-Za-z0-9]{36}\b/g,
+		kind: "API_KEY",
+		priority: 80
+	},
+	{
+		pattern: /(?<![A-Z0-9._%+-])[A-Z0-9._%+-]{1,64}@[A-Z0-9](?:[A-Z0-9.-]{0,251}[A-Z0-9])?\.[A-Z]{2,63}(?![A-Z0-9-]|\.[A-Z0-9])/gi,
+		kind: "EMAIL",
+		priority: 70
+	},
+	{
+		pattern: /(?<![A-Za-z0-9])[0-9A-Fa-f]{8}-(?:[0-9A-Fa-f]{4}-){3}[0-9A-Fa-f]{12}(?![A-Za-z0-9])/g,
+		kind: "OPAQUE_ID",
+		priority: 60
+	},
+	{
+		pattern: /(?<![A-Za-z0-9])(?:[0-9A-Fa-f]{64}|[0-9A-Fa-f]{40}|[0-9A-Fa-f]{32})(?![A-Za-z0-9])/g,
+		kind: "OPAQUE_ID",
+		priority: 60,
+		accept: (value) => /[A-Fa-f]/.test(value)
+	},
+	{
+		pattern: /(?<![A-Za-z0-9])[A-Za-z0-9]{24,}(?![A-Za-z0-9])/g,
+		kind: "OPAQUE_ID",
+		priority: 60,
+		accept: looksLikeHighEntropyOpaqueId
+	},
+	{
+		pattern: /(?<![A-Za-z0-9.])\+?(?:\d|\(\d)[\d ()-]*\d[Xx]?\)?(?![A-Za-z0-9]|\.\d)/g,
+		kind: "LONG_NUMBER",
+		priority: 50,
+		accept: (value) => {
+			if (value.replace(/\D/g, "").length < 9) return false;
+			const normalized = value.trim().replace(/[()]/g, "");
+			return !/^(?:19|20)\d{2}-\d{1,2}-\d{1,2}(?:[ T-]\d{1,2})?$/.test(normalized) && !/^(?:19|20)\d{2}\s+\d{1,2}\s+\d{1,2}(?:\s+\d{1,2})?$/.test(normalized);
+		}
+	}
+];
+/**
+* 对文本执行 8 类脱敏。规则各自扫描 → 收集值级 span → 合并重叠 → 重拼插入占位符。
+* 同一位置被多条规则命中时取更长/更高优先级的 span。
+*/
+function redactSecrets(text) {
+	if (!text) return {
+		text,
+		counts: {},
+		redacted: false
+	};
+	const spans = [];
+	for (const rule of RULES) collectRuleSpans(text, rule, spans);
+	const selected = mergeOverlappingSpans(spans);
+	if (selected.length === 0) return {
+		text,
+		counts: {},
+		redacted: false
+	};
+	const counts = {};
+	const output = [];
+	let offset = 0;
+	for (const span of selected) {
+		output.push(text.slice(offset, span.start), `[REDACTED:${span.kind}]`);
+		counts[span.kind] = (counts[span.kind] ?? 0) + 1;
+		offset = span.end;
+	}
+	output.push(text.slice(offset));
+	return {
+		text: output.join(""),
+		counts,
+		redacted: true
+	};
+}
+function collectRuleSpans(text, rule, spans) {
+	rule.pattern.lastIndex = 0;
+	for (const match of text.matchAll(rule.pattern)) {
+		const value = match[rule.group ?? 0];
+		if (match.index === void 0 || !value || PLACEHOLDER_VALUE_PATTERN.test(value.trim())) continue;
+		if (rule.accept && !rule.accept(value)) continue;
+		const relativeStart = match[0].lastIndexOf(value);
+		if (relativeStart < 0) continue;
+		spans.push({
+			start: match.index + relativeStart,
+			end: match.index + relativeStart + value.length,
+			kind: rule.kind,
+			priority: rule.priority
+		});
+	}
+}
+function isSafeCredentialValue(rawValue) {
+	let value = rawValue.trim();
+	if (value.startsWith("\"") && value.endsWith("\"") || value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1).trim();
+	return !value || SAFE_CREDENTIAL_VALUE_PATTERN.test(value);
+}
+/** 高熵判定:小写≥2 ∧ 大写≥2 ∧ 数字≥2 ∧ 字符类切换≥4 ∧ 最长连续字母段≤5 ∧ 去重字符≥12。 */
+function looksLikeHighEntropyOpaqueId(value) {
+	let lowercase = 0;
+	let uppercase = 0;
+	let digits = 0;
+	let transitions = 0;
+	let letterRun = 0;
+	let longestLetterRun = 0;
+	let previousWasDigit;
+	const unique = /* @__PURE__ */ new Set();
+	for (const character of value) {
+		const isDigit = character >= "0" && character <= "9";
+		if (isDigit) {
+			digits += 1;
+			letterRun = 0;
+		} else {
+			if (character >= "a" && character <= "z") lowercase += 1;
+			else uppercase += 1;
+			letterRun += 1;
+			longestLetterRun = Math.max(longestLetterRun, letterRun);
+		}
+		if (previousWasDigit !== void 0 && previousWasDigit !== isDigit) transitions += 1;
+		previousWasDigit = isDigit;
+		unique.add(character);
+	}
+	return lowercase >= 2 && uppercase >= 2 && digits >= 2 && transitions >= 4 && longestLetterRun <= 5 && unique.size >= 12;
+}
+function mergeOverlappingSpans(spans) {
+	const sorted = [...spans].sort((left, right) => left.start - right.start || right.end - left.end || right.priority - left.priority);
+	const merged = [];
+	for (const span of sorted) {
+		const previous = merged.at(-1);
+		if (!previous || span.start >= previous.end) {
+			merged.push({ ...span });
+			continue;
+		}
+		previous.end = Math.max(previous.end, span.end);
+	}
+	return merged;
+}
+//#endregion
 //#region src/schema.ts
 /**
-* 投影库装配（spec §DDL）：node:sqlite 四表一虚表 + WAL/mmap 缓存层 1 +
-* sessions 汇总常驻 Map（缓存层 2）。写纪律：每 session 全量重写式替换
-* （事务内 DELETE+INSERT），幂等。
+* 投影库装配（spec §DDL）：node:sqlite 五表一虚表（sessions/segments/replaces/
+* blocks/meta + segments_fts）+ WAL/mmap 缓存层 1 + sessions 汇总常驻 Map
+* （缓存层 2）。写纪律：每 session 全量重写式替换（事务内 DELETE+INSERT），
+* 幂等；字块一律走 blocks 字典 hash 引用（v5），脱敏派生臂同源生成。
 *
 * 本库只落在 ~/.dsh-reference-session-collector/projection.db ——绝不触碰会话文件。
 */
 /** schema 版本（meta 表；断裂式迁移的比对基线）。
 * v2=加 sessions.format_generation；
-* v3=search_text 视界变更（剥离宿主注入样板段，strip-injected horizon v2）；
-* v4=账目双修（真机取证 2026-10-20）：① text/tokens 视界补 data.content 双臂
-*   （checkpoint 替身/用户消息旧视界全计 0）；② replaces 加 live 列——替身行
-*   被后续区间再覆盖时 delta 清除不入 gross（用户定标：checkpoint 只影响其后
-*   的积分）。旧行两项口径均错，整库清空重建（投影可重建，丢行无损）。 */
-const PROJECTION_SCHEMA_VERSION = "4";
+* v3=search_text 视界变更（剥离宿主注入样板段）；
+* v4=账目双修（data.content 双臂 + replaces.live）；
+* v5=字块共享 + 脱敏派生臂（2026-10-20 用户定标）：text/search_text/
+*   surface_text 实列退役→blocks 字典 hash 引用（surface 经 replaced_by
+*   链派生）；新 redacted_text_hash/redacted_search_text_hash（redactSecrets
+*   派生臂，CCI 口径 text 的 token 数学不动）；旧库列形状断裂→DROP 重建
+*   + VACUUM（投影可重建，丢行无损）。 */
+const PROJECTION_SCHEMA_VERSION = "5";
 const DDL = `
 CREATE TABLE IF NOT EXISTS sessions(
   session_id TEXT PRIMARY KEY, dir_id TEXT NOT NULL, title TEXT DEFAULT '',
@@ -391,11 +729,12 @@ CREATE TABLE IF NOT EXISTS segments(
   session_id TEXT NOT NULL, seq INTEGER NOT NULL, turn INTEGER DEFAULT 0,
   step INTEGER DEFAULT 0, kind TEXT NOT NULL, role TEXT DEFAULT '',
   time INTEGER DEFAULT 0,
-  text TEXT DEFAULT '', tokens INTEGER DEFAULT 0,
-  search_text TEXT DEFAULT '',
+  text_hash TEXT, tokens INTEGER DEFAULT 0,
+  search_text_hash TEXT,
+  redacted_text_hash TEXT, redacted_search_text_hash TEXT,
   on_surface INTEGER DEFAULT 1,
   compressed INTEGER DEFAULT 0,
-  surface_text TEXT, surface_tokens INTEGER, replaced_by INTEGER,
+  surface_tokens INTEGER, replaced_by INTEGER,
   PRIMARY KEY(session_id, seq)) WITHOUT ROWID;
 CREATE INDEX IF NOT EXISTS idx_segments_turn ON segments(session_id, turn, step);
 CREATE TABLE IF NOT EXISTS replaces(
@@ -410,6 +749,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS segments_fts USING fts5(
   index_text, session_id UNINDEXED, seq UNINDEXED, kind UNINDEXED,
   tokenize='unicode61');
 CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT);
+${BLOCKS_DDL}
 `;
 function summaryToRow(s, artifactBytes, artifactMtimeMs, extra) {
 	return {
@@ -451,24 +791,21 @@ PRAGMA cache_size=-32000;
 PRAGMA mmap_size=268435456;
 PRAGMA application_id=0x44525343;
 `);
-	db.exec(DDL);
-	if (!db.prepare("PRAGMA table_info(sessions)").all().some((c) => c.name === "format_generation")) db.exec("ALTER TABLE sessions ADD COLUMN format_generation INTEGER DEFAULT -1");
-	if (!db.prepare("PRAGMA table_info(replaces)").all().some((c) => c.name === "live")) db.exec("ALTER TABLE replaces ADD COLUMN live INTEGER NOT NULL DEFAULT 1");
-	const previous = db.prepare("SELECT value FROM meta WHERE key='schema_version'").get();
-	if (previous !== void 0 && previous.value !== "4") {
-		db.exec("BEGIN");
-		try {
-			db.prepare("DELETE FROM sessions").run();
-			db.prepare("DELETE FROM segments").run();
-			db.prepare("DELETE FROM replaces").run();
-			db.prepare("DELETE FROM segments_fts").run();
-			db.exec("COMMIT");
-		} catch (error) {
-			db.exec("ROLLBACK");
-			throw error;
-		}
+	let previous;
+	try {
+		previous = db.prepare("SELECT value FROM meta WHERE key='schema_version'").get();
+	} catch {
+		previous = void 0;
 	}
-	db.prepare("INSERT INTO meta(key, value) VALUES(?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run("schema_version", "4");
+	if (previous !== void 0 && previous.value !== "5") db.exec(`DROP TABLE IF EXISTS segments_fts;
+             DROP TABLE IF EXISTS segments;
+             DROP TABLE IF EXISTS replaces;
+             DROP TABLE IF EXISTS sessions;
+             DROP TABLE IF EXISTS blocks;
+             DROP TABLE IF EXISTS meta`);
+	db.exec(DDL);
+	if (previous !== void 0 && previous.value !== "5") db.exec("VACUUM");
+	db.prepare("INSERT INTO meta(key, value) VALUES(?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run("schema_version", "5");
 	const summaries = /* @__PURE__ */ new Map();
 	const loadAll = () => {
 		summaries.clear();
@@ -477,9 +814,11 @@ PRAGMA application_id=0x44525343;
 	};
 	loadAll();
 	const insertSegment = db.prepare(`INSERT INTO segments(
-    session_id, seq, turn, step, kind, role, time, text, tokens, search_text,
-    on_surface, compressed, surface_text, surface_tokens, replaced_by)
-    VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`);
+    session_id, seq, turn, step, kind, role, time, text_hash, tokens, search_text_hash,
+    redacted_text_hash, redacted_search_text_hash,
+    on_surface, compressed, surface_tokens, replaced_by)
+    VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`);
+	const blocks = new BlocksStore(db);
 	const insertReplace = db.prepare(`INSERT INTO replaces(
     session_id, seq, start_seq, end_seq, a_tokens, b_tokens, kind, turn, step, live)
     VALUES(?,?,?,?,?,?,?,?,?,?)`);
@@ -511,7 +850,11 @@ PRAGMA application_id=0x44525343;
 				db.prepare("DELETE FROM segments WHERE session_id=?").run(row.session_id);
 				db.prepare("DELETE FROM replaces WHERE session_id=?").run(row.session_id);
 				db.prepare("DELETE FROM segments_fts WHERE session_id=?").run(row.session_id);
-				for (const seg of segments) insertSegment.run(row.session_id, seg.seq, seg.turn, seg.step, seg.kind, seg.role, seg.time, seg.text, seg.tokens, seg.searchText, seg.onSurface, seg.compressed, seg.surfaceText, seg.surfaceTokens, seg.replacedBy);
+				for (const seg of segments) {
+					const redText = redactSecrets(seg.text).text;
+					const redSearch = redactSecrets(seg.searchText).text;
+					insertSegment.run(row.session_id, seg.seq, seg.turn, seg.step, seg.kind, seg.role, seg.time, blocks.ensure(seg.text), seg.tokens, blocks.ensure(seg.searchText), blocks.ensure(redText), blocks.ensure(redSearch), seg.onSurface, seg.compressed, seg.surfaceTokens, seg.replacedBy);
+				}
 				for (const rep of replaces) insertReplace.run(row.session_id, rep.seq, rep.startSeq, rep.endSeq, rep.aTokens, rep.bTokens, rep.kind, rep.turn, rep.step, rep.live);
 				for (const seg of segments) {
 					if (seg.searchText === "") continue;
@@ -1656,6 +1999,8 @@ var ProjectionSync = class {
 			const discovered = discoverSessionLogs(dshHome, limit);
 			const exact = this.counterMode === "off" ? void 0 : resolveExactCounter();
 			const basis = exact !== void 0 ? "ds-token" : "chars/4";
+			const dict = this.store.blocksWatermark();
+			const dictEvicted = dict.current < dict.last;
 			const result = {
 				ok: true,
 				busy: false,
@@ -1673,7 +2018,8 @@ var ProjectionSync = class {
 					this.syncOne(found, {
 						archivedSet,
 						exact,
-						result
+						result,
+						dictEvicted
 					});
 				} catch (error) {
 					result.failures.push({
@@ -1704,6 +2050,7 @@ var ProjectionSync = class {
 				}
 			}
 			result.durationMs = Date.now() - startedAt;
+			this.store.setBlocksWatermark(this.store.blocksWatermark().current);
 			this.lastResult = result;
 			return result;
 		} finally {
@@ -1712,7 +2059,7 @@ var ProjectionSync = class {
 	}
 	syncOne(found, ctx) {
 		const record = this.store.getSessionByDir(found.dirSessionId);
-		const statHit = record !== void 0 && record.artifactBytes === found.artifact.bytes && record.artifactMtimeMs === found.artifact.mtimeMs;
+		const statHit = !ctx.dictEvicted && record !== void 0 && record.artifactBytes === found.artifact.bytes && record.artifactMtimeMs === found.artifact.mtimeMs;
 		if (record !== void 0) {
 			const archived = ctx.archivedSet.has(record.sessionId) ? 1 : 0;
 			if (record.archived !== archived) this.store.setSessionArchived(record.sessionId, archived);
@@ -1810,6 +2157,8 @@ function isTrustedRequest(req, trustedHosts) {
 	if (typeof host !== "string") return false;
 	return trustedHosts.some((trusted) => trusted.toLowerCase() === host.toLowerCase());
 }
+/** R5 水位在 meta 表的键（DDL 冻结不动，kv 面落）。 */
+const BLOCKS_WATERMARK_META_KEY = "blocks_watermark";
 /** sync 端口 → ProjectionStore 真身适配（dir 水位经常驻 Map 扫描）。 */
 function createStorePort(store) {
 	return {
@@ -1840,6 +2189,18 @@ function createStorePort(store) {
 		},
 		deleteSession(sessionId) {
 			store.deleteSession(sessionId);
+		},
+		blocksWatermark() {
+			const current = Number(store.db.prepare("SELECT COALESCE(MAX(rowid), 0) AS m FROM blocks").get().m);
+			const row = store.db.prepare("SELECT value FROM meta WHERE key=?").get(BLOCKS_WATERMARK_META_KEY);
+			const parsed = row === void 0 ? 0 : Number(row.value);
+			return {
+				current,
+				last: Number.isFinite(parsed) ? parsed : 0
+			};
+		},
+		setBlocksWatermark(value) {
+			store.db.prepare("INSERT INTO meta(key, value) VALUES(?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(BLOCKS_WATERMARK_META_KEY, String(value));
 		}
 	};
 }
@@ -1859,6 +2220,20 @@ function resolveLatestSessionId(query) {
 	return bestId;
 }
 /**
+* v5 参数解析（R3/R4）：`redact` 仅 '1' 为真，其余/非法=0（缺省原文臂）；
+* `tail` 正整数生效，非法/≤0=忽略（全量）。
+*/
+function parseRedact(url) {
+	return url.searchParams.get("redact") === "1";
+}
+function parseTail(url) {
+	const raw = url.searchParams.get("tail");
+	if (raw === null) return void 0;
+	const parsed = Number(raw);
+	if (!Number.isFinite(parsed) || parsed <= 0) return void 0;
+	return Math.floor(parsed);
+}
+/**
 * op 分发：GET `?op=…`；POST `{op:'sync'}` 触发后台同步。
 * 未知 op / 缺参 → 400（不留空壳）。
 */
@@ -1924,9 +2299,13 @@ async function handleDrsRequest(deps, req, res) {
 			}
 			const turn = url.searchParams.get("turn") ?? void 0;
 			const fields = url.searchParams.get("fields") ?? void 0;
+			const redact = parseRedact(url);
+			const tail = parseTail(url);
 			const detail = sessionDetail(deps.query, sessionId, {
 				...turn === void 0 ? {} : { turn },
-				...fields === void 0 ? {} : { fields }
+				...fields === void 0 ? {} : { fields },
+				...tail === void 0 ? {} : { tail },
+				...redact ? { redact: true } : {}
 			});
 			if (detail.session === void 0) {
 				writeJson(res, 200, {
@@ -1993,6 +2372,7 @@ async function handleDrsRequest(deps, req, res) {
 				});
 				return;
 			}
+			const redact = parseRedact(url);
 			if (url.searchParams.get("group") === "session") {
 				const kindsRaw = url.searchParams.get("kinds") ?? "";
 				const archivedRaw = url.searchParams.get("archived");
@@ -2009,7 +2389,9 @@ async function handleDrsRequest(deps, req, res) {
 						...toRaw !== null && Number.isFinite(Number(toRaw)) ? { to: Number(toRaw) } : {},
 						...url.searchParams.get("surface") === "0" ? { surfaceOnly: false } : {},
 						...url.searchParams.get("sort") === "time" ? { sortBy: "time" } : {},
-						...limitRaw !== null && Number.isFinite(Number(limitRaw)) ? { limit: Number(limitRaw) } : {}
+						...limitRaw !== null && Number.isFinite(Number(limitRaw)) ? { limit: Number(limitRaw) } : {},
+						...url.searchParams.get("expand") === "1" ? { expand: true } : {},
+						...redact ? { redact: true } : {}
 					})
 				});
 				return;
@@ -2019,7 +2401,8 @@ async function handleDrsRequest(deps, req, res) {
 				ok: true,
 				...search(deps.query, q, {
 					...sessionId === "" ? {} : { session: sessionId },
-					...limitRaw === null ? {} : { limit: Number(limitRaw) }
+					...limitRaw === null ? {} : { limit: Number(limitRaw) },
+					...redact ? { redact: true } : {}
 				})
 			});
 			return;