@yottameta/yotta-memory 0.23.3 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -22,12 +22,13 @@ const os = require('os');
22
22
  const crypto = require('crypto');
23
23
  const http = require('http');
24
24
  const net = require('net');
25
+ const zlib = require('zlib');
25
26
  const child_process = require('child_process');
26
27
  const { AsyncLocalStorage } = require('async_hooks');
27
28
 
28
- const VERSION = '0.23.3';
29
- const CLI_VALUE_OPTS = new Set(['--type', '--limit', '--days', '--out', '--owner', '--agent', '--agent-id', '--agent-key', '--agent-key-file', '--plugin-data', '--threshold', '--scope', '--host', '--port', '--dir', '--name', '--user', '--relationship', '--source', '--weight', '--budget', '--password', '--new-password', '--recovery-key', '--recovery-key-out', '--reason', '--merge', '--model', '--subject', '--embedding', '--focus', '--embedding-timeout', '--min-age', '--min-idle', '--max-utility', '--min-group', '--period', '--to', '--id', '--time', '--tools', '--mcp-config', '--skill-dir', '--year', '--evalset', '--k', '--seed', '--bootstrap', '--gate', '--against', '--template', '--path', '--from']);
30
- const CLI_FLAG_OPTS = new Set(['--project', '--all', '--unsafe', '--no-auth', '--stdio', '--onstart', '--from-current', '--restart', '--force', '--attach', '--allow-same-volume', '--verify', '--no-hint', '--encrypt', '--no-encrypt', '--password-stdin', '--json', '--manual', '--skip-schedule', '--useful', '--useless', '--undo', '--dry-run', '--apply', '--purge', '--dedup', '--batches', '--propose', '--audit', '--capacity', '--rules', '--explain', '--semantic', '--runtime', '--ablate', '--timing', '--baseline', '--probe', '--quarantine', '--restore', '--no-usage', '--yes', '--keep-memories', '--keep-identity']);
29
+ const VERSION = '0.24.0';
30
+ const CLI_VALUE_OPTS = new Set(['--type', '--limit', '--days', '--out', '--owner', '--agent', '--agent-id', '--agent-key', '--agent-key-file', '--plugin-data', '--threshold', '--scope', '--host', '--port', '--dir', '--name', '--user', '--relationship', '--source', '--weight', '--budget', '--password', '--new-password', '--recovery-key', '--recovery-key-out', '--reason', '--merge', '--model', '--subject', '--embedding', '--focus', '--embedding-timeout', '--min-age', '--min-idle', '--max-utility', '--min-group', '--period', '--to', '--id', '--time', '--tools', '--mcp-config', '--skill-dir', '--year', '--evalset', '--k', '--seed', '--bootstrap', '--gate', '--against', '--template', '--path', '--from', '--from-plan', '--batch']);
31
+ const CLI_FLAG_OPTS = new Set(['--project', '--all', '--unsafe', '--no-auth', '--stdio', '--onstart', '--from-current', '--restart', '--force', '--attach', '--allow-same-volume', '--verify', '--no-hint', '--encrypt', '--no-encrypt', '--password-stdin', '--json', '--manual', '--skip-schedule', '--useful', '--useless', '--undo', '--dry-run', '--apply', '--purge', '--dedup', '--batches', '--propose', '--audit', '--capacity', '--rules', '--explain', '--semantic', '--runtime', '--ablate', '--timing', '--baseline', '--probe', '--quarantine', '--restore', '--no-usage', '--yes', '--keep-memories', '--keep-identity', '--stats', '--include-flat', '--no-policy', '--policy-candidates']);
31
32
 
32
33
  function helpOption(flag, arg, what, when, caution) {
33
34
  return { flag: flag, arg: arg || '', what: what, when: when || '', caution: caution || '' };
@@ -119,6 +120,28 @@ const HELP_MODEL = [
119
120
  helpOption('--template', '<文件>', '用模板声明显式期望(owner / 最小条数 / 类型条数 / 查询)', '有明确验收清单、要按清单核对时', '模板只加严判定,不会放宽或改写探针'),
120
121
  helpOption('--json', '', '输出 JSON', '脚本或自动化读取体检结果时', ''),
121
122
  ] },
123
+ { name: 'index', usage: 'index --stats [--json]', what: '只读分析公共索引体积、字段占比与 gzip 分片预估', when: '判断是否需要升级索引格式、压缩或分片时', options: [
124
+ helpOption('--stats', '', '输出索引统计报告', '分析索引膨胀根因与预估压缩收益时', '只读:不重建索引,不改任何记忆文件'),
125
+ helpOption('--json', '', '输出 JSON', '脚本读取字段体积、token 与压缩预估时', ''),
126
+ ] },
127
+ { name: 'layout', usage: 'layout <plan|apply|rollback> [选项]', what: '预演、执行或回滚日级目录重平衡', when: '单个年/月目录文件数超过阈值、需要可回滚迁移时', subcommands: [
128
+ helpSub('plan', 'plan [--threshold <数量>] [--include-flat] [--out <文件>] [--json]', '生成布局迁移计划', '先确定会移动哪些文件、目标路径和冲突时', [
129
+ helpOption('--threshold', '<数量>', '目录文件数超过该值时进入计划', '默认阈值不合适时', ''),
130
+ helpOption('--include-flat', '', '把旧平铺根也纳入计划', '确认愿意迁移旧平铺文件时', '默认只处理年/月桶,旧平铺保持原位'),
131
+ helpOption('--out', '<文件>', '把计划写到指定位置', '需要把计划交给维护窗口审阅时', ''),
132
+ helpOption('--json', '', '输出 JSON', '自动化读取候选、操作与冲突时', ''),
133
+ ]),
134
+ helpSub('apply', 'apply --from-plan <文件> --yes [--json]', '执行已审阅的布局迁移计划', '维护窗口内确认计划无冲突后', [
135
+ helpOption('--from-plan', '<文件>', '指定布局迁移计划文件', '执行 plan 生成的计划时', ''),
136
+ helpOption('--yes', '', '跳过交互确认', '计划已审阅且维护窗口已开始时', '会移动记忆文件;失败时自动逆序回滚'),
137
+ helpOption('--json', '', '输出 JSON', '自动化执行迁移时', ''),
138
+ ]),
139
+ helpSub('rollback', 'rollback --batch <id> --yes [--json]', '按 journal 回滚一次布局迁移', '迁移后发现问题、需要恢复原路径时', [
140
+ helpOption('--batch', '<id>', '指定布局迁移批次', '找回滚 journal 时', ''),
141
+ helpOption('--yes', '', '跳过交互确认', '已确认要恢复原布局时', '会移动记忆文件;按 journal 逆序恢复'),
142
+ helpOption('--json', '', '输出 JSON', '自动化回滚时', ''),
143
+ ]),
144
+ ] },
122
145
  { name: 'maintain', usage: 'maintain [选项]', what: '记忆自组织:归档、遗忘候选、去重和合并', when: '定期清理低价值记忆、合并重复条目时', options: [
123
146
  helpOption('--capacity', '', '查看容量水位与淘汰 / 晋升建议', '库变大,想看哪些条目值得清理或提升时', '只读报告,不会自动归档或改权重'),
124
147
  helpOption('--rules', '', '查看重复踩坑 → 建议升级为规则的清单', '同一类问题反复出现、想把它固化成 BOUND 规则时', '只读报告,不自动写规则;阈值走 config maintain_rule_min_hits'),
@@ -183,6 +206,9 @@ const HELP_MODEL = [
183
206
  helpOption('--yes', '', '跳过交互确认', '自动化里执行隔离时', '跳过确认后不再二次询问,请先看一遍命中清单'),
184
207
  helpOption('--restore', '', '还原最近一个隔离批次', '隔离后发现问题、要恢复原文时', '会覆盖当前文件;批次已还原过则拒绝重复还原'),
185
208
  helpOption('--id', '<id>', '指定要还原的隔离批次', '要还原的不是最近一批时', ''),
209
+ helpOption('--no-policy', '', '忽略扫描例外策略做全量复核', '怀疑策略过期或想重新确认所有历史命中时', '会重新显示已抑制命中,不改策略文件'),
210
+ helpOption('--policy-candidates', '', '只生成例外策略候选清单', '历史安全语料需要人工复核后配置例外时', '只读扫描;不自动写入 security/scan-policy.json'),
211
+ helpOption('--out', '<文件>', '把候选清单写到文件', '想把候选交给人工复核或评审时', '仅用于 --policy-candidates;不会写策略文件'),
186
212
  ] },
187
213
  { name: 'reindex', usage: 'reindex', what: '重建记忆索引', when: '索引损坏、手动改过记忆文件或 doctor 提示索引异常时', options: [] },
188
214
  { name: 'export', usage: 'export --out <文件.json>', what: '导出全部记忆', when: '备份、迁移或做离线检查时', options: [helpOption('--out', '<文件>', '指定导出文件', '导出时必须给出目标 JSON 文件', '')] },
@@ -332,6 +358,11 @@ const VIEW_HELP_QUICK = [
332
358
  { cmd: 'backup restore', example: 'yotta-memory backup restore <id> --to <目录>', danger: true, note: '恢复会写入目标目录;覆盖非空目录需显式 --force,先确认目标和备份' },
333
359
  { cmd: 'backup drill', example: 'yotta-memory backup drill <id>', note: '先演练确认备份可用,再正式恢复' },
334
360
  { cmd: 'doctor', example: 'yotta-memory doctor' },
361
+ { cmd: 'index', example: 'yotta-memory index --stats --json', note: '只读统计;--stats 不会重建索引' },
362
+ { cmd: 'layout', example: 'yotta-memory layout <子命令>' },
363
+ { cmd: 'layout plan', example: 'yotta-memory layout plan --json' },
364
+ { cmd: 'layout apply', example: 'yotta-memory layout apply --from-plan <文件> --yes', danger: true, note: '会移动记忆文件;先审阅计划,失败时自动逆序回滚' },
365
+ { cmd: 'layout rollback', example: 'yotta-memory layout rollback --batch <id> --yes', danger: true, note: '按 journal 逆序恢复原路径;先确认批次' },
335
366
  { cmd: 'maintain', example: 'yotta-memory maintain --capacity', note: '只读报告,不会自动归档或改权重' },
336
367
  { cmd: 'consolidate', example: 'yotta-memory consolidate --propose' },
337
368
  { cmd: 'distill', example: 'yotta-memory distill --model <命令>' },
@@ -1077,12 +1108,13 @@ function typeSubdir(type, owner) {
1077
1108
  const id = owner || '';
1078
1109
  return path.join(PRIVATE_DIR, id, TYPE_DIRS[t]);
1079
1110
  }
1080
- // v0.17.0 规模:新写入按 <yyyy>/<mm> 分层(公共 facts/<yyyy>/<mm>/,私密 private/<owner>/<type>/<yyyy>/<mm>/)。
1111
+ // v0.24.0 规模:新写入按 <yyyy>/<mm>/<dd> 分层(公共 facts/<yyyy>/<mm>/<dd>/,
1112
+ // 私密 private/<owner>/<type>/<yyyy>/<mm>/<dd>/)。
1081
1113
  // 只影响新写入;旧平铺文件保持原位继续可读,不自动迁移。
1082
1114
  function dateSubdir(dateStr) {
1083
1115
  const d = String(dateStr || '');
1084
1116
  if (!/^\d{4}-\d{2}-\d{2}$/.test(d)) return '';
1085
- return path.join(d.slice(0, 4), d.slice(5, 7));
1117
+ return path.join(d.slice(0, 4), d.slice(5, 7), d.slice(8, 10));
1086
1118
  }
1087
1119
  function entryWriteDir(root, type, owner, dateStr) {
1088
1120
  const base = path.join(root, typeSubdir(type, owner));
@@ -1717,10 +1749,7 @@ function recentlyWrittenFiles(typeDir, writeDir) {
1717
1749
  const yearDir = path.dirname(writeDir);
1718
1750
  const seen = new Set(out);
1719
1751
  if (fs.existsSync(yearDir)) {
1720
- for (const fp of fs.readdirSync(yearDir)) {
1721
- const full = path.join(yearDir, fp);
1722
- if (fs.statSync(full).isFile() && MEMORY_FILE_RE.test(fp) && !seen.has(full)) out.push(full);
1723
- }
1752
+ walkEntryFiles(yearDir, out);
1724
1753
  }
1725
1754
  if (fs.existsSync(typeDir)) {
1726
1755
  for (const f of fs.readdirSync(typeDir)) {
@@ -2287,14 +2316,38 @@ function writeProfileText(root, owner, text) {
2287
2316
 
2288
2317
  // ---- 索引(index.json)----
2289
2318
  function indexPath(root) { return path.join(root, INDEX_FILE); }
2290
- const INDEX_VERSION = 4;
2291
- // v0.8.1 超大库索引分片:公共 index.json 超过阈值按年份分片(index-<year>.json),避免单文件膨胀。
2319
+ const INDEX_VERSION = 5;
2320
+ const INDEX_MIN_SUPPORTED_VERSION = 4;
2321
+ // v0.24.0:索引按逻辑体积或条目数触发年份分片;大库使用 gzip 分片。
2292
2322
  const INDEX_SHARD_THRESHOLD = 5000;
2293
- // 安全校验:分片文件名必须匹配 index-YYYY.json,防 manifest 篡改导致路径穿越。
2294
- const SHARD_RE = /^index-\d{4}\.json$/;
2323
+ const INDEX_COMPRESS_TRIGGER_BYTES = 4 * 1024 * 1024;
2324
+ // 安全校验:只接受 index-YYYY.json(.gz),防 manifest 篡改导致路径穿越。
2325
+ const SHARD_RE = /^index-\d{4}\.json(?:\.gz)?$/;
2295
2326
  function isSafeShardName(name) { return typeof name === 'string' && SHARD_RE.test(name); }
2296
- function indexShardName(year) { return 'index-' + year + '.json'; }
2297
- function shardYear(name) { return String(name).slice('index-'.length, -'.json'.length); }
2327
+ function isGzipShardName(name) { return typeof name === 'string' && /\.json\.gz$/.test(name); }
2328
+ function indexShardName(year, compressed) { return 'index-' + year + '.json' + (compressed === false ? '' : '.gz'); }
2329
+ function shardYear(name) {
2330
+ const match = String(name || '').match(/^index-(\d{4})\.json(?:\.gz)?$/);
2331
+ return match ? match[1] : '';
2332
+ }
2333
+ function readIndexShard(root, name) {
2334
+ if (!isSafeShardName(name)) return null;
2335
+ const full = path.join(root, name);
2336
+ if (!fs.existsSync(full)) return null;
2337
+ try {
2338
+ const raw = fs.readFileSync(full);
2339
+ const text = isGzipShardName(name) ? zlib.gunzipSync(raw).toString('utf8') : raw.toString('utf8');
2340
+ const parsed = JSON.parse(text);
2341
+ return parsed && Array.isArray(parsed.entries) ? parsed : null;
2342
+ } catch (error) {
2343
+ return null;
2344
+ }
2345
+ }
2346
+ function writeIndexShard(root, name, payload) {
2347
+ const raw = Buffer.from(JSON.stringify(payload), 'utf8');
2348
+ const data = isGzipShardName(name) ? zlib.gzipSync(raw, { level: 9, mtime: 0 }) : raw;
2349
+ atomicWriteFileSync(path.join(root, name), data);
2350
+ }
2298
2351
  function entryYear(entry) {
2299
2352
  return String((entry && entry.created) || '').slice(0, 4) || String(today()).slice(0, 4);
2300
2353
  }
@@ -2317,7 +2370,7 @@ function loadIndexManifest(root) {
2317
2370
  if (!fs.existsSync(p)) return null;
2318
2371
  try {
2319
2372
  const d = JSON.parse(fs.readFileSync(p, 'utf8'));
2320
- if (d && Array.isArray(d.shards)) return d;
2373
+ if (d && Array.isArray(d.shards) && (!d.version || d.version >= INDEX_MIN_SUPPORTED_VERSION)) return d;
2321
2374
  } catch (e) { /* ignore */ }
2322
2375
  return null;
2323
2376
  }
@@ -2328,20 +2381,17 @@ function loadIndex(root) {
2328
2381
  const d = JSON.parse(fs.readFileSync(p, 'utf8'));
2329
2382
  if (!d) return null;
2330
2383
  if (Array.isArray(d.entries)) {
2331
- // v0.8.0 语义索引(字段加权 + 拼音 token)version=3;v0.8.1 分片 version=4;旧版索引返回 null 触发重建
2332
- if (d.version && d.version >= INDEX_VERSION) return d.entries;
2384
+ // v4 平铺索引与 v5 平铺索引都继续读取;更旧版本返回 null 触发重建。
2385
+ if (d.version && d.version >= INDEX_MIN_SUPPORTED_VERSION) return d.entries;
2333
2386
  return null;
2334
2387
  }
2335
2388
  if (Array.isArray(d.shards)) {
2336
- // v0.8.1 分片 manifest:串联各分片条目
2337
- if (d.version && d.version < INDEX_VERSION) return null;
2389
+ if (d.version && d.version < INDEX_MIN_SUPPORTED_VERSION) return null;
2338
2390
  const out = [];
2339
2391
  for (const sh of d.shards) {
2340
- if (!isSafeShardName(sh)) continue;
2341
- const sp = path.join(root, sh);
2342
- if (!fs.existsSync(sp)) continue;
2343
- const sd = JSON.parse(fs.readFileSync(sp, 'utf8'));
2344
- if (sd && Array.isArray(sd.entries)) out.push.apply(out, sd.entries);
2392
+ const sd = readIndexShard(root, sh);
2393
+ if (!sd) return null;
2394
+ out.push.apply(out, sd.entries);
2345
2395
  }
2346
2396
  return out;
2347
2397
  }
@@ -2361,21 +2411,20 @@ function loadIndexFor(root, options) {
2361
2411
  const d = JSON.parse(fs.readFileSync(p, 'utf8'));
2362
2412
  if (!d) return null;
2363
2413
  if (Array.isArray(d.entries)) {
2364
- if (d.version && d.version < INDEX_VERSION) return null;
2414
+ if (d.version && d.version < INDEX_MIN_SUPPORTED_VERSION) return null;
2365
2415
  const want = new Set(years);
2366
2416
  return d.entries.filter(function (e) { return want.has(entryYear(e)); });
2367
2417
  }
2368
2418
  if (Array.isArray(d.shards)) {
2369
- if (d.version && d.version < INDEX_VERSION) return null;
2419
+ if (d.version && d.version < INDEX_MIN_SUPPORTED_VERSION) return null;
2370
2420
  const want = new Set(years);
2371
2421
  const out = [];
2372
2422
  for (const sh of d.shards) {
2373
2423
  if (!isSafeShardName(sh)) continue;
2374
2424
  if (!want.has(shardYear(sh))) continue; // 只读取命中年份的分片文件
2375
- const sp = path.join(root, sh);
2376
- if (!fs.existsSync(sp)) continue;
2377
- const sd = JSON.parse(fs.readFileSync(sp, 'utf8'));
2378
- if (sd && Array.isArray(sd.entries)) out.push.apply(out, sd.entries);
2425
+ const sd = readIndexShard(root, sh);
2426
+ if (!sd) return null;
2427
+ out.push.apply(out, sd.entries);
2379
2428
  }
2380
2429
  return out;
2381
2430
  }
@@ -2395,10 +2444,15 @@ function indexEntriesFor(root, options) {
2395
2444
  return { entries: all.filter(function (e) { return want.has(entryYear(e)); }), error: '' };
2396
2445
  }
2397
2446
  function getIndex(root) { return loadIndex(root) || []; }
2398
- function saveIndex(root, entries) {
2447
+ function saveIndex(root, entries, options) {
2448
+ options = options || {};
2399
2449
  const clean = entries.map(function (e) { const c = Object.assign({}, e); delete c.meta; return c; });
2400
2450
  const old = loadIndexManifest(root);
2401
- if (clean.length > INDEX_SHARD_THRESHOLD) {
2451
+ const shardThreshold = options.shardThresholdEntries === undefined ? INDEX_SHARD_THRESHOLD : Number(options.shardThresholdEntries);
2452
+ const byteThreshold = options.compressThresholdBytes === undefined ? INDEX_COMPRESS_TRIGGER_BYTES : Number(options.compressThresholdBytes);
2453
+ const compress = options.compress !== false;
2454
+ const logicalBytes = jsonByteLength(clean);
2455
+ if (clean.length > shardThreshold || logicalBytes > byteThreshold) {
2402
2456
  const byYear = {};
2403
2457
  for (const e of clean) {
2404
2458
  const y = entryYear(e);
@@ -2406,21 +2460,206 @@ function saveIndex(root, entries) {
2406
2460
  }
2407
2461
  const newShards = [];
2408
2462
  for (const y of Object.keys(byYear).sort()) {
2409
- const name = indexShardName(y);
2410
- atomicWriteFileSync(path.join(root, name), JSON.stringify({ version: INDEX_VERSION, year: (parseInt(y, 10) || 0), entries: byYear[y] }, null, 2), 'utf8');
2463
+ const name = indexShardName(y, compress);
2464
+ writeIndexShard(root, name, { version: INDEX_VERSION, year: (parseInt(y, 10) || 0), entries: byYear[y] });
2411
2465
  newShards.push(name);
2412
2466
  }
2413
- atomicWriteFileSync(indexPath(root), JSON.stringify({ version: INDEX_VERSION, updated: today(), shards: newShards, count: clean.length }, null, 2), 'utf8');
2467
+ atomicWriteFileSync(indexPath(root), JSON.stringify({
2468
+ version: INDEX_VERSION,
2469
+ updated: today(),
2470
+ format: compress ? 'gzip-year' : 'json-year',
2471
+ shards: newShards,
2472
+ count: clean.length,
2473
+ }, null, 2), 'utf8');
2414
2474
  if (old && Array.isArray(old.shards)) {
2415
2475
  for (const sh of old.shards) if (newShards.indexOf(sh) === -1 && isSafeShardName(sh)) { try { fs.unlinkSync(path.join(root, sh)); } catch (e) {} }
2416
2476
  }
2417
2477
  } else {
2478
+ atomicWriteFileSync(indexPath(root), JSON.stringify({ version: INDEX_VERSION, updated: today(), entries: clean }, null, 2), 'utf8');
2418
2479
  if (old && Array.isArray(old.shards)) {
2419
2480
  for (const sh of old.shards) { try { fs.unlinkSync(path.join(root, sh)); } catch (e) {} }
2420
2481
  }
2421
- atomicWriteFileSync(indexPath(root), JSON.stringify({ version: INDEX_VERSION, updated: today(), entries: clean }, null, 2), 'utf8');
2422
2482
  }
2423
2483
  }
2484
+ function jsonByteLength(value) {
2485
+ return Buffer.byteLength(JSON.stringify(value === undefined ? null : value), 'utf8');
2486
+ }
2487
+ function indexFieldBytes(entries) {
2488
+ const groups = {
2489
+ tokens: ['tokens'],
2490
+ statement: ['statement'],
2491
+ subject: ['subject'],
2492
+ tags: ['tags'],
2493
+ usage: ['access_count', 'last_accessed', 'feedback_net', 'hit_days', 'hit_queries'],
2494
+ };
2495
+ const grouped = new Set();
2496
+ for (const names of Object.values(groups)) for (const name of names) grouped.add(name);
2497
+ const out = { tokens: 0, statement: 0, subject: 0, tags: 0, usage: 0, other: 0, total: 0, overhead: 0 };
2498
+ for (const entry of entries) {
2499
+ out.total += jsonByteLength(entry);
2500
+ for (const group of Object.keys(groups)) {
2501
+ for (const name of groups[group]) out[group] += jsonByteLength(entry[name]);
2502
+ }
2503
+ for (const name of Object.keys(entry)) {
2504
+ if (!grouped.has(name)) out.other += jsonByteLength(entry[name]);
2505
+ }
2506
+ }
2507
+ out.overhead = Math.max(0, out.total - (out.tokens + out.statement + out.subject + out.tags + out.usage + out.other));
2508
+ return out;
2509
+ }
2510
+ function indexTokenStats(entries) {
2511
+ let unique = 0;
2512
+ let weighted = 0;
2513
+ let pinyin = 0;
2514
+ for (const entry of entries) {
2515
+ const tokens = entry && entry.tokens && typeof entry.tokens === 'object' ? entry.tokens : {};
2516
+ for (const key of Object.keys(tokens)) {
2517
+ unique += 1;
2518
+ weighted += Number(tokens[key]) || 0;
2519
+ if (key.indexOf('py:') === 0 || key.indexOf('pyi:') === 0) pinyin += 1;
2520
+ }
2521
+ }
2522
+ return {
2523
+ unique: unique,
2524
+ weighted: Math.round(weighted * 1000) / 1000,
2525
+ pinyin: pinyin,
2526
+ averageUniquePerEntry: entries.length ? Math.round((unique / entries.length) * 1000) / 1000 : 0,
2527
+ };
2528
+ }
2529
+ function indexGzipProjection(entries) {
2530
+ const byYear = {};
2531
+ for (const entry of entries) {
2532
+ const clean = Object.assign({}, entry);
2533
+ delete clean.meta;
2534
+ const year = entryYear(clean);
2535
+ (byYear[year] = byYear[year] || []).push(clean);
2536
+ }
2537
+ const shards = [];
2538
+ let gzipBytes = 0;
2539
+ for (const year of Object.keys(byYear).sort()) {
2540
+ const raw = Buffer.from(JSON.stringify({ version: 5, year: parseInt(year, 10) || 0, entries: byYear[year] }), 'utf8');
2541
+ const compressed = zlib.gzipSync(raw, { level: 9, mtime: 0 });
2542
+ gzipBytes += compressed.length;
2543
+ shards.push({
2544
+ year: year,
2545
+ entries: byYear[year].length,
2546
+ jsonBytes: raw.length,
2547
+ gzipBytes: compressed.length,
2548
+ });
2549
+ }
2550
+ const manifest = Buffer.from(JSON.stringify({
2551
+ version: 5,
2552
+ updated: '',
2553
+ shards: shards.map(function (item) { return 'index-' + item.year + '.json.gz'; }),
2554
+ count: entries.length,
2555
+ }), 'utf8');
2556
+ const storageBytes = gzipBytes + manifest.length;
2557
+ return {
2558
+ format: 'index-v5-gzip-year',
2559
+ storageBytes: storageBytes,
2560
+ manifestBytes: manifest.length,
2561
+ shards: shards,
2562
+ };
2563
+ }
2564
+ function indexStatsCore(opts) {
2565
+ opts = opts || {};
2566
+ const root = path.resolve(opts.root || userRoot());
2567
+ const idx = indexPath(root);
2568
+ if (!fs.existsSync(idx)) {
2569
+ return { error: true, format: 'missing', entryCount: 0, storageBytes: 0, text: '索引不存在: ' + idx + '。请先运行 yotta-memory reindex。' };
2570
+ }
2571
+ let manifestBytes = 0;
2572
+ let data = null;
2573
+ try {
2574
+ manifestBytes = fs.statSync(idx).size;
2575
+ data = JSON.parse(fs.readFileSync(idx, 'utf8'));
2576
+ } catch (error) {
2577
+ return { error: true, format: 'invalid', entryCount: 0, storageBytes: manifestBytes, text: '索引无法解析: ' + error.message };
2578
+ }
2579
+ if (!data || (!Array.isArray(data.entries) && !Array.isArray(data.shards))) {
2580
+ return { error: true, format: 'invalid', entryCount: 0, storageBytes: manifestBytes, text: '索引结构无效:缺少 entries 或 shards 数组。' };
2581
+ }
2582
+ const warnings = [];
2583
+ const entries = [];
2584
+ let shardBytes = 0;
2585
+ const shards = [];
2586
+ if (Array.isArray(data.entries)) {
2587
+ entries.push.apply(entries, data.entries);
2588
+ } else {
2589
+ for (const shard of data.shards) {
2590
+ if (!isSafeShardName(shard)) {
2591
+ warnings.push('忽略不安全分片名: ' + String(shard));
2592
+ continue;
2593
+ }
2594
+ const full = path.join(root, shard);
2595
+ if (!fs.existsSync(full)) {
2596
+ warnings.push('分片缺失: ' + shard);
2597
+ continue;
2598
+ }
2599
+ try {
2600
+ const bytes = fs.statSync(full).size;
2601
+ const parsed = readIndexShard(root, shard);
2602
+ shardBytes += bytes;
2603
+ if (parsed) {
2604
+ entries.push.apply(entries, parsed.entries);
2605
+ shards.push({ name: shard, bytes: bytes, entries: parsed.entries.length });
2606
+ } else {
2607
+ warnings.push('分片结构无效: ' + shard);
2608
+ }
2609
+ } catch (error) {
2610
+ warnings.push('分片无法解析: ' + shard + '(' + error.message + ')');
2611
+ }
2612
+ }
2613
+ }
2614
+ const storageBytes = manifestBytes + shardBytes;
2615
+ const logicalBytes = jsonByteLength(entries);
2616
+ const fieldBytes = indexFieldBytes(entries);
2617
+ const tokens = indexTokenStats(entries);
2618
+ const gzipProjection = indexGzipProjection(entries);
2619
+ const warnBytes = scaleThreshold(loadConfig(), 'scale_warn_index_bytes');
2620
+ gzipProjection.underWarningThreshold = gzipProjection.storageBytes <= warnBytes;
2621
+ gzipProjection.warningThresholdBytes = warnBytes;
2622
+ gzipProjection.compressionRatio = storageBytes ? Math.round((gzipProjection.storageBytes / storageBytes) * 1000) / 1000 : 0;
2623
+ const triggerReasons = [];
2624
+ if (entries.length > INDEX_SHARD_THRESHOLD) triggerReasons.push('entries');
2625
+ if (logicalBytes > INDEX_COMPRESS_TRIGGER_BYTES) triggerReasons.push('logical_bytes');
2626
+ const lines = [
2627
+ '索引统计(只读)',
2628
+ '- 格式: ' + (Array.isArray(data.entries) ? 'inline' : 'sharded'),
2629
+ '- 条目: ' + entries.length,
2630
+ '- 索引存储: ' + storageBytes + ' 字节(manifest ' + manifestBytes + ' / 分片 ' + shardBytes + ')',
2631
+ '- 逻辑原始体积: ' + logicalBytes + ' 字节',
2632
+ '- token: ' + tokens.unique + ' 个唯一 token(加权 ' + tokens.weighted + ' / 拼音 ' + tokens.pinyin + ' / 平均 ' + tokens.averageUniquePerEntry + ' 每条例)',
2633
+ '- 字段体积: tokens ' + fieldBytes.tokens + ' / statement ' + fieldBytes.statement + ' / subject ' + fieldBytes.subject + ' / tags ' + fieldBytes.tags + ' / usage ' + fieldBytes.usage + ' / other ' + fieldBytes.other,
2634
+ '- gzip 年份分片预估: ' + gzipProjection.storageBytes + ' 字节(压缩比 ' + gzipProjection.compressionRatio + ' / 低于告警线 ' + (gzipProjection.underWarningThreshold ? '是' : '否') + ')',
2635
+ '- 触发建议: ' + (triggerReasons.length ? triggerReasons.join(' + ') : '当前无需按现有阈值分片'),
2636
+ ];
2637
+ if (warnings.length) lines.push('- 告警: ' + warnings.join(';'));
2638
+ return {
2639
+ error: false,
2640
+ root: root,
2641
+ format: Array.isArray(data.entries) ? 'inline' : 'sharded',
2642
+ version: Number(data.version || 0),
2643
+ updated: data.updated || '',
2644
+ entryCount: entries.length,
2645
+ shardCount: shards.length,
2646
+ shards: shards,
2647
+ manifestBytes: manifestBytes,
2648
+ shardBytes: shardBytes,
2649
+ storageBytes: storageBytes,
2650
+ logicalBytes: logicalBytes,
2651
+ fieldBytes: fieldBytes,
2652
+ tokens: tokens,
2653
+ gzipProjection: gzipProjection,
2654
+ trigger: {
2655
+ entryThreshold: INDEX_SHARD_THRESHOLD,
2656
+ logicalByteThreshold: INDEX_COMPRESS_TRIGGER_BYTES,
2657
+ reasons: triggerReasons,
2658
+ },
2659
+ warnings: warnings,
2660
+ text: lines.join('\n'),
2661
+ };
2662
+ }
2424
2663
  function collectEntryFiles(root) {
2425
2664
  const dirs = [];
2426
2665
  const facts = path.join(root, PUBLIC_DIR);
@@ -2534,6 +2773,318 @@ function layoutCollisionReport(root) {
2534
2773
  report.ok = report.conflicts === 0 && report.unverified === 0;
2535
2774
  return report;
2536
2775
  }
2776
+ const LAYOUT_PLAN_DIR = path.join('.maintenance', 'layout-plans');
2777
+ const LAYOUT_JOURNAL_DIR = path.join('.maintenance', 'layout-journals');
2778
+ function safeLayoutRel(value) {
2779
+ const rel = String(value || '').replace(/\\/g, '/');
2780
+ if (!rel || path.isAbsolute(rel) || rel.split('/').includes('..')) return false;
2781
+ if (!MEMORY_FILE_RE.test(path.posix.basename(rel))) return false;
2782
+ return rel.indexOf(PUBLIC_DIR + '/') === 0 || rel.indexOf(PRIVATE_DIR + '/') === 0;
2783
+ }
2784
+ function layoutInsideRoot(root, rel) {
2785
+ const base = path.resolve(root);
2786
+ const full = path.resolve(root, String(rel || ''));
2787
+ return full === base || full.indexOf(base + path.sep) === 0;
2788
+ }
2789
+ function layoutBaseInfo(rel) {
2790
+ const parts = String(rel || '').split('/');
2791
+ if (parts[0] === PUBLIC_DIR) return { kind: 'public', baseDepth: 1, owner: '', type: 'facts' };
2792
+ if (parts[0] === PRIVATE_DIR && parts.length >= 3) {
2793
+ return { kind: 'private', baseDepth: 3, owner: parts[1], type: parts[2] };
2794
+ }
2795
+ return null;
2796
+ }
2797
+ function isLayoutMonthBucket(rel, includeFlat) {
2798
+ const parts = String(rel || '').split('/');
2799
+ const info = layoutBaseInfo(rel);
2800
+ if (!info) return false;
2801
+ if (parts.length === info.baseDepth) return !!includeFlat;
2802
+ if (parts.length !== info.baseDepth + 2) return false;
2803
+ return /^\d{4}$/.test(parts[info.baseDepth]) && /^\d{2}$/.test(parts[info.baseDepth + 1]);
2804
+ }
2805
+ function layoutTargetFor(rel) {
2806
+ const parts = String(rel || '').split('/');
2807
+ const info = layoutBaseInfo(rel);
2808
+ const base = path.basename(rel);
2809
+ const match = base.match(/^(\d{4})-(\d{2})-(\d{2})-\d{4}\.md(?:\.enc)?$/);
2810
+ if (!info || !match) return '';
2811
+ return parts.slice(0, info.baseDepth).concat([match[1], match[2], match[3], base]).join('/');
2812
+ }
2813
+ function layoutCandidateDirs(root, includeFlat, threshold) {
2814
+ const seeds = [];
2815
+ const facts = path.join(root, PUBLIC_DIR);
2816
+ if (fs.existsSync(facts)) seeds.push({ dir: facts, baseDepth: 1, owner: '' });
2817
+ const pdir = path.join(root, PRIVATE_DIR);
2818
+ if (fs.existsSync(pdir)) {
2819
+ for (const owner of fs.readdirSync(pdir)) {
2820
+ const ownerDir = path.join(pdir, owner);
2821
+ if (!fs.statSync(ownerDir).isDirectory()) continue;
2822
+ for (const type of PRIVATE_LEAF) {
2823
+ const typeDir = path.join(ownerDir, type);
2824
+ if (fs.existsSync(typeDir)) seeds.push({ dir: typeDir, baseDepth: 3, owner: owner });
2825
+ }
2826
+ }
2827
+ }
2828
+ const found = [];
2829
+ const visit = function (dir) {
2830
+ const rel = path.relative(root, dir).replace(/\\/g, '/');
2831
+ if (isLayoutMonthBucket(rel, includeFlat)) {
2832
+ const files = fs.readdirSync(dir, { withFileTypes: true })
2833
+ .filter(function (entry) { return entry.isFile() && MEMORY_FILE_RE.test(entry.name); })
2834
+ .map(function (entry) { return entry.name; });
2835
+ if (files.length > threshold) {
2836
+ const info = layoutBaseInfo(rel);
2837
+ if (info && info.kind === 'private' && isEncrypted(root) && !getOwnerKeyFor(root, info.owner)) {
2838
+ found.push({ path: rel, files: files.length, skipped: true, reason: 'encrypted_owner_without_key' });
2839
+ } else {
2840
+ found.push({ path: rel, files: files.length, skipped: false, reason: '' });
2841
+ }
2842
+ }
2843
+ }
2844
+ for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
2845
+ if (entry.isDirectory()) visit(path.join(dir, entry.name));
2846
+ }
2847
+ };
2848
+ for (const seed of seeds) visit(seed.dir);
2849
+ return found.sort(function (a, b) { return a.path.localeCompare(b.path); });
2850
+ }
2851
+ function layoutBatchId(now) {
2852
+ return 'layout-' + now.toISOString().replace(/[:.]/g, '-') + '-' + crypto.randomBytes(3).toString('hex');
2853
+ }
2854
+ function layoutPlanCore(opts) {
2855
+ opts = opts || {};
2856
+ const root = path.resolve(opts.root || userRoot());
2857
+ const cfg = loadConfig();
2858
+ const threshold = opts.threshold === undefined ? scaleThreshold(cfg, 'scale_warn_files_per_dir') : Number(opts.threshold);
2859
+ const includeFlat = !!opts.includeFlat;
2860
+ const now = opts.now ? new Date(opts.now) : new Date();
2861
+ const batch = layoutBatchId(now);
2862
+ const candidates = layoutCandidateDirs(root, includeFlat, threshold);
2863
+ const operations = [];
2864
+ const conflicts = [];
2865
+ const warnings = [];
2866
+ for (const candidate of candidates) {
2867
+ if (candidate.skipped) {
2868
+ warnings.push('跳过 ' + candidate.path + ': 加密 owner 无可用密钥');
2869
+ continue;
2870
+ }
2871
+ const dir = path.join(root, candidate.path);
2872
+ for (const name of fs.readdirSync(dir)) {
2873
+ if (!MEMORY_FILE_RE.test(name)) continue;
2874
+ const from = path.posix.join(candidate.path, name);
2875
+ const to = layoutTargetFor(from);
2876
+ if (!to || !safeLayoutRel(to)) {
2877
+ conflicts.push({ from: from, to: to, reason: 'target_invalid' });
2878
+ continue;
2879
+ }
2880
+ if (fs.existsSync(path.join(root, to))) {
2881
+ conflicts.push({ from: from, to: to, reason: 'target_exists' });
2882
+ continue;
2883
+ }
2884
+ operations.push({
2885
+ from: from,
2886
+ to: to,
2887
+ sha256: normalizedLayoutDigest(fs.readFileSync(path.join(root, from))),
2888
+ status: 'pending',
2889
+ });
2890
+ }
2891
+ }
2892
+ operations.sort(function (a, b) { return a.from.localeCompare(b.from); });
2893
+ const plan = {
2894
+ batch: batch,
2895
+ createdAt: now.toISOString(),
2896
+ root: root,
2897
+ threshold: threshold,
2898
+ includeFlat: includeFlat,
2899
+ candidates: candidates,
2900
+ operations: operations,
2901
+ conflicts: conflicts,
2902
+ warnings: warnings,
2903
+ planFile: '',
2904
+ };
2905
+ if (conflicts.length) {
2906
+ plan.error = true;
2907
+ plan.text = '布局迁移计划存在 ' + conflicts.length + ' 个冲突,拒绝生成可执行计划。\n'
2908
+ + conflicts.map(function (item) { return '- ' + item.from + ' -> ' + item.to + '(' + item.reason + ')'; }).join('\n');
2909
+ return plan;
2910
+ }
2911
+ const planFile = opts.out
2912
+ ? path.resolve(String(opts.out))
2913
+ : path.join(root, LAYOUT_PLAN_DIR, batch + '.json');
2914
+ fs.mkdirSync(path.dirname(planFile), { recursive: true });
2915
+ plan.planFile = planFile;
2916
+ atomicWriteFileSync(planFile, JSON.stringify(plan, null, 2), 'utf8');
2917
+ plan.error = false;
2918
+ plan.text = '布局迁移计划: ' + batch + '\n- 目录: ' + candidates.length + ' 个候选'
2919
+ + '(可迁移 ' + candidates.filter(function (item) { return !item.skipped; }).length + ')\n'
2920
+ + '- 文件: ' + operations.length + ' 个\n- 计划: ' + planFile
2921
+ + (warnings.length ? '\n- 告警: ' + warnings.join(';') : '');
2922
+ return plan;
2923
+ }
2924
+ function validateLayoutPlan(root, plan) {
2925
+ if (!plan || plan.root !== root) return '计划 root 与当前记忆库不一致,拒绝执行。';
2926
+ if (!/^layout-[A-Za-z0-9-]+$/.test(String(plan.batch || ''))) return '计划 batch 不安全。';
2927
+ if (!Array.isArray(plan.operations) || !plan.operations.length) return '计划没有可执行文件。';
2928
+ if (Array.isArray(plan.conflicts) && plan.conflicts.length) return '计划包含冲突,拒绝执行。';
2929
+ for (const op of plan.operations) {
2930
+ if (!safeLayoutRel(op.from) || !safeLayoutRel(op.to) || !layoutInsideRoot(root, op.from) || !layoutInsideRoot(root, op.to)) {
2931
+ return '计划包含不安全路径: ' + String(op.from) + ' -> ' + String(op.to);
2932
+ }
2933
+ }
2934
+ return '';
2935
+ }
2936
+ function writeLayoutJournal(pathname, journal) {
2937
+ atomicWriteFileSync(pathname, JSON.stringify(journal, null, 2), 'utf8');
2938
+ }
2939
+ function restoreLayoutDone(root, journal, pathname) {
2940
+ let ok = true;
2941
+ for (let i = journal.operations.length - 1; i >= 0; i--) {
2942
+ const op = journal.operations[i];
2943
+ if (op.status !== 'done') continue;
2944
+ const from = path.join(root, op.from);
2945
+ const to = path.join(root, op.to);
2946
+ if (!fs.existsSync(to) || fs.existsSync(from)) {
2947
+ op.status = 'rollback_conflict';
2948
+ ok = false;
2949
+ continue;
2950
+ }
2951
+ fs.mkdirSync(path.dirname(from), { recursive: true });
2952
+ fs.renameSync(to, from);
2953
+ op.status = 'rolled_back';
2954
+ writeLayoutJournal(pathname, journal);
2955
+ }
2956
+ return ok;
2957
+ }
2958
+ function withLayoutLock(root, fn) {
2959
+ const lock = path.join(root, '.maintenance', 'layout.lock');
2960
+ fs.mkdirSync(path.dirname(lock), { recursive: true });
2961
+ try {
2962
+ fs.writeFileSync(lock, JSON.stringify({ pid: process.pid, created: new Date().toISOString() }), { encoding: 'utf8', flag: 'wx' });
2963
+ } catch (error) {
2964
+ return { error: true, text: '已有布局迁移在运行或上次未清理 lock: ' + lock };
2965
+ }
2966
+ try {
2967
+ return fn();
2968
+ } finally {
2969
+ try { fs.unlinkSync(lock); } catch (error) { /* ignore */ }
2970
+ }
2971
+ }
2972
+ function layoutApplyCore(opts) {
2973
+ opts = opts || {};
2974
+ const root = path.resolve(opts.root || userRoot());
2975
+ if (!opts.yes) return { error: true, text: '拒绝执行布局迁移:请复核计划后显式加 --yes。' };
2976
+ if (!opts.planFile) return { error: true, text: '请用 --from-plan <文件> 指定布局迁移计划。' };
2977
+ const planFile = path.resolve(String(opts.planFile));
2978
+ let plan = null;
2979
+ try { plan = JSON.parse(fs.readFileSync(planFile, 'utf8')); } catch (error) {
2980
+ return { error: true, text: '无法读取布局迁移计划: ' + error.message };
2981
+ }
2982
+ const invalid = validateLayoutPlan(root, plan);
2983
+ if (invalid) return { error: true, text: invalid };
2984
+ return withLayoutLock(root, function () {
2985
+ const journalFile = path.join(root, LAYOUT_JOURNAL_DIR, plan.batch + '.json');
2986
+ if (fs.existsSync(journalFile)) return { error: true, text: '该计划已有 journal,拒绝重复执行: ' + journalFile };
2987
+ const journal = {
2988
+ batch: plan.batch,
2989
+ createdAt: plan.createdAt,
2990
+ startedAt: new Date().toISOString(),
2991
+ root: root,
2992
+ status: 'applying',
2993
+ operations: plan.operations.map(function (op) {
2994
+ return { from: op.from, to: op.to, sha256: op.sha256, status: 'pending' };
2995
+ }),
2996
+ };
2997
+ fs.mkdirSync(path.dirname(journalFile), { recursive: true });
2998
+ writeLayoutJournal(journalFile, journal);
2999
+ let moved = 0;
3000
+ try {
3001
+ for (const op of journal.operations) {
3002
+ const from = path.join(root, op.from);
3003
+ const to = path.join(root, op.to);
3004
+ if (!fs.existsSync(from)) throw new Error('源文件缺失: ' + op.from);
3005
+ const digest = normalizedLayoutDigest(fs.readFileSync(from));
3006
+ if (digest !== op.sha256) throw new Error('源文件哈希变化: ' + op.from);
3007
+ if (fs.existsSync(to)) throw new Error('目标已存在: ' + op.to);
3008
+ fs.mkdirSync(path.dirname(to), { recursive: true });
3009
+ fs.renameSync(from, to);
3010
+ op.status = 'done';
3011
+ moved += 1;
3012
+ writeLayoutJournal(journalFile, journal);
3013
+ }
3014
+ buildIndex(root);
3015
+ journal.status = 'applied';
3016
+ journal.completedAt = new Date().toISOString();
3017
+ writeLayoutJournal(journalFile, journal);
3018
+ return {
3019
+ error: false,
3020
+ batch: plan.batch,
3021
+ moved: moved,
3022
+ rolledBack: false,
3023
+ planFile: planFile,
3024
+ journalFile: journalFile,
3025
+ text: '布局迁移完成: ' + plan.batch + '(' + moved + ' 文件)\n计划: ' + planFile + '\nJournal: ' + journalFile,
3026
+ };
3027
+ } catch (error) {
3028
+ const rolledBack = restoreLayoutDone(root, journal, journalFile);
3029
+ journal.status = rolledBack ? 'rolled_back' : 'rollback_failed';
3030
+ journal.error = error.message;
3031
+ journal.completedAt = new Date().toISOString();
3032
+ writeLayoutJournal(journalFile, journal);
3033
+ try { buildIndex(root); } catch (e) { /* ignore */ }
3034
+ return {
3035
+ error: true,
3036
+ batch: plan.batch,
3037
+ moved: moved,
3038
+ rolledBack: rolledBack,
3039
+ planFile: planFile,
3040
+ journalFile: journalFile,
3041
+ text: '布局迁移失败并' + (rolledBack ? '已回滚' : '回滚不完整') + ': ' + error.message + '\nJournal: ' + journalFile,
3042
+ };
3043
+ }
3044
+ });
3045
+ }
3046
+ function layoutRollbackCore(opts) {
3047
+ opts = opts || {};
3048
+ const root = path.resolve(opts.root || userRoot());
3049
+ if (!opts.yes) return { error: true, text: '拒绝回滚布局迁移:请显式加 --yes。' };
3050
+ if (!/^layout-[A-Za-z0-9-]+$/.test(String(opts.batch || ''))) return { error: true, text: '请用 --batch <id> 指定安全的布局迁移批次。' };
3051
+ const journalFile = path.join(root, LAYOUT_JOURNAL_DIR, opts.batch + '.json');
3052
+ let journal = null;
3053
+ try { journal = JSON.parse(fs.readFileSync(journalFile, 'utf8')); } catch (error) {
3054
+ return { error: true, text: '无法读取布局迁移 journal: ' + error.message };
3055
+ }
3056
+ if (journal.root !== root) return { error: true, text: 'journal root 与当前记忆库不一致。' };
3057
+ if (journal.status === 'rolled_back') return { error: false, batch: opts.batch, rolledBack: 0, journalFile: journalFile, text: '布局迁移批次已回滚,无需重复操作。' };
3058
+ if (journal.status !== 'applied') return { error: true, text: '批次状态不是 applied,不能常规回滚: ' + journal.status };
3059
+ return withLayoutLock(root, function () {
3060
+ let restored = 0;
3061
+ try {
3062
+ for (let i = journal.operations.length - 1; i >= 0; i--) {
3063
+ const op = journal.operations[i];
3064
+ if (op.status !== 'done') continue;
3065
+ const from = path.join(root, op.from);
3066
+ const to = path.join(root, op.to);
3067
+ if (!fs.existsSync(to)) throw new Error('迁移目标缺失: ' + op.to);
3068
+ if (fs.existsSync(from)) throw new Error('回滚目标已存在: ' + op.from);
3069
+ fs.mkdirSync(path.dirname(from), { recursive: true });
3070
+ fs.renameSync(to, from);
3071
+ op.status = 'rolled_back';
3072
+ restored += 1;
3073
+ writeLayoutJournal(journalFile, journal);
3074
+ }
3075
+ buildIndex(root);
3076
+ journal.status = 'rolled_back';
3077
+ journal.rolledBackAt = new Date().toISOString();
3078
+ writeLayoutJournal(journalFile, journal);
3079
+ return { error: false, batch: opts.batch, restored: restored, journalFile: journalFile, text: '布局迁移已回滚: ' + opts.batch + '(' + restored + ' 文件)' };
3080
+ } catch (error) {
3081
+ journal.status = 'rollback_failed';
3082
+ journal.rollbackError = error.message;
3083
+ writeLayoutJournal(journalFile, journal);
3084
+ return { error: true, batch: opts.batch, restored: restored, journalFile: journalFile, text: '布局回滚失败: ' + error.message };
3085
+ }
3086
+ });
3087
+ }
2537
3088
  // 迁移:把根下旧平铺 prefs|bounds|commits/*.md 按 frontmatter owner 迁入 private/<owner>/<type>/
2538
3089
  function migrateLayout(root) {
2539
3090
  let moved = 0;
@@ -2804,15 +3355,13 @@ function touchIndex(root, relFiles, options) {
2804
3355
  for (const sh of manifest.shards) {
2805
3356
  if (!isSafeShardName(sh)) continue;
2806
3357
  if (yearSet && !yearSet.has(shardYear(sh))) continue;
2807
- const sp = path.join(root, sh);
2808
- if (!fs.existsSync(sp)) continue;
2809
- let sd = null; try { sd = JSON.parse(fs.readFileSync(sp, 'utf8')); } catch (e) { continue; }
2810
- if (!sd || !Array.isArray(sd.entries)) continue;
3358
+ const sd = readIndexShard(root, sh);
3359
+ if (!sd) continue;
2811
3360
  let dirty = false;
2812
3361
  for (const e of sd.entries) {
2813
3362
  if (set.has(e.file)) { bumpEntry(e); dirty = true; }
2814
3363
  }
2815
- if (dirty) atomicWriteFileSync(sp, JSON.stringify(sd, null, 2), 'utf8');
3364
+ if (dirty) writeIndexShard(root, sh, sd);
2816
3365
  }
2817
3366
  });
2818
3367
  return;
@@ -3429,6 +3978,53 @@ function backupSetupCore(opts) {
3429
3978
  + (schedule && schedule.error ? '\n[警告] ' + schedule.text + ';serve 补跑仍会在启动后检查。' : ''),
3430
3979
  };
3431
3980
  }
3981
+ const BACKUP_FUTURE_TOLERANCE_MS = 15 * 60 * 1000;
3982
+ function backupCreatedTs(value) {
3983
+ const ts = Date.parse(String(value || ''));
3984
+ return Number.isFinite(ts) ? ts : null;
3985
+ }
3986
+ function compareBackupCreatedDesc(a, b) {
3987
+ const at = backupCreatedTs(a && a.created);
3988
+ const bt = backupCreatedTs(b && b.created);
3989
+ if (at !== null && bt !== null && at !== bt) return bt - at;
3990
+ if (at !== null && bt === null) return -1;
3991
+ if (at === null && bt !== null) return 1;
3992
+ return String((b && b.id) || '').localeCompare(String((a && a.id) || ''));
3993
+ }
3994
+ function selectLatestValidBackup(backups, nowMs) {
3995
+ const now = Number.isFinite(nowMs) ? nowMs : Date.now();
3996
+ const valid = [];
3997
+ const anomalies = [];
3998
+ for (const backup of backups || []) {
3999
+ if (!backup || !backup.manifest) {
4000
+ anomalies.push({ id: backup && backup.id || '', type: 'manifest_missing', created: backup && backup.created || '' });
4001
+ continue;
4002
+ }
4003
+ if (!Array.isArray(backup.manifest.files)) {
4004
+ anomalies.push({ id: backup.id, type: 'manifest_incomplete', created: backup.created || '' });
4005
+ continue;
4006
+ }
4007
+ const createdTs = backupCreatedTs(backup.created);
4008
+ if (createdTs === null) {
4009
+ anomalies.push({ id: backup.id, type: 'invalid_created', created: backup.created || '' });
4010
+ continue;
4011
+ }
4012
+ if (createdTs > now + BACKUP_FUTURE_TOLERANCE_MS) {
4013
+ anomalies.push({ id: backup.id, type: 'future_timestamp', created: backup.created || '' });
4014
+ continue;
4015
+ }
4016
+ valid.push(Object.assign({}, backup, { createdTs: createdTs }));
4017
+ }
4018
+ valid.sort(compareBackupCreatedDesc);
4019
+ return { latestValid: valid[0] || null, anomalies: anomalies };
4020
+ }
4021
+ function backupAnomalyLabel(type) {
4022
+ if (type === 'future_timestamp') return '未来时间';
4023
+ if (type === 'manifest_missing') return 'manifest 缺失';
4024
+ if (type === 'manifest_incomplete') return 'manifest 不完整';
4025
+ if (type === 'invalid_created') return 'created 无效';
4026
+ return type;
4027
+ }
3432
4028
  function backupStatusCore(opts) {
3433
4029
  opts = opts || {};
3434
4030
  const root = path.resolve(opts.root || userRoot());
@@ -3450,14 +4046,20 @@ function backupStatusCore(opts) {
3450
4046
  };
3451
4047
  }
3452
4048
  const listed = backupListCore({ dir: dir });
3453
- const latest = listed.backups[0] || null;
3454
4049
  const now = opts.now ? new Date(opts.now).getTime() : Date.now();
3455
- const createdTs = latest && latest.created ? new Date(latest.created).getTime() : NaN;
3456
- const ageHours = isNaN(createdTs) ? null : (now - createdTs) / 3600000;
4050
+ const selection = selectLatestValidBackup(listed.backups, now);
4051
+ const latest = selection.latestValid;
4052
+ const ageHours = latest ? (now - latest.createdTs) / 3600000 : null;
3457
4053
  const fresh = ageHours !== null && ageHours <= maxAge;
3458
4054
  const same = opts.sameVolumeFn ? !!opts.sameVolumeFn(root, dir) : sameVolume(root, dir);
3459
4055
  const independent = !same;
3460
4056
  const healthy = !!latest && fresh && independent && !cfg.backup_last_error && !cfg.backup_scheduler_error;
4057
+ const configLastSuccess = cfg.backup_last_success || null;
4058
+ const effectiveLastSuccess = latest ? latest.created : configLastSuccess;
4059
+ const source = latest ? 'latest_valid' : (configLastSuccess ? 'config' : 'none');
4060
+ const anomalyLines = selection.anomalies.map(function (item) {
4061
+ return item.id + '(' + backupAnomalyLabel(item.type) + (item.created ? ' / ' + item.created : '') + ')';
4062
+ });
3461
4063
  const lines = [
3462
4064
  '备份状态: ' + (cfg.backup_enabled ? '已启用' : '已关闭'),
3463
4065
  '- 备份目录: ' + dir,
@@ -3465,8 +4067,12 @@ function backupStatusCore(opts) {
3465
4067
  '- 计划: ' + (cfg.backup_schedule || 'manual') + (cfg.backup_time ? ' ' + cfg.backup_time : ''),
3466
4068
  '- 调度器: ' + (cfg.backup_scheduler || '(未注册)'),
3467
4069
  '- 调度异常: ' + (cfg.backup_scheduler_error || '(无)'),
3468
- '- 上次成功: ' + (cfg.backup_last_success || (latest && latest.created) || '(无)'),
4070
+ '- 配置缓存上次成功: ' + (configLastSuccess || '(无)'),
4071
+ '- 有效备份: ' + (latest ? latest.id + ' / ' + latest.created : '(无)'),
4072
+ '- 有效上次成功: ' + (effectiveLastSuccess || '(无)'),
4073
+ '- 来源: ' + source,
3469
4074
  '- 备份年龄: ' + (ageHours === null ? '(无可用备份)' : ageHours.toFixed(1) + ' 小时'),
4075
+ '- 异常备份: ' + (anomalyLines.length ? anomalyLines.join(';') : '(无)'),
3470
4076
  '- 最近失败: ' + (cfg.backup_last_error || '(无)'),
3471
4077
  '- 健康: ' + (healthy ? '正常' : '异常'),
3472
4078
  ];
@@ -3477,6 +4083,11 @@ function backupStatusCore(opts) {
3477
4083
  healthy: healthy,
3478
4084
  independent: independent,
3479
4085
  latest: latest,
4086
+ latest_valid: latest,
4087
+ config_last_success: configLastSuccess,
4088
+ effective_last_success: effectiveLastSuccess,
4089
+ source: source,
4090
+ anomalies: selection.anomalies,
3480
4091
  ageHours: ageHours,
3481
4092
  text: lines.join('\n'),
3482
4093
  };
@@ -3487,9 +4098,7 @@ function localDateKey(value) {
3487
4098
  return d.getFullYear() + '-' + p(d.getMonth() + 1) + '-' + p(d.getDate());
3488
4099
  }
3489
4100
  function latestBackupByCreated(backups) {
3490
- return (backups || []).slice().sort(function (a, b) {
3491
- return String(b.created || '').localeCompare(String(a.created || ''));
3492
- })[0] || null;
4101
+ return (backups || []).slice().sort(compareBackupCreatedDesc)[0] || null;
3493
4102
  }
3494
4103
  function acquireBackupLock(lockPath, now, staleHours) {
3495
4104
  const nowMs = now instanceof Date ? now.getTime() : new Date(now).getTime();
@@ -3560,7 +4169,7 @@ function backupEnsureDailyCore(opts) {
3560
4169
  const listed = backupListCore({ dir: dir });
3561
4170
  if (listed.error) throw new Error(listed.text);
3562
4171
  const todayKey = localDateKey(now);
3563
- const latest = latestBackupByCreated(listed.backups);
4172
+ const latest = selectLatestValidBackup(listed.backups, now.getTime()).latestValid;
3564
4173
  if (latest && latest.created && localDateKey(latest.created) === todayKey) {
3565
4174
  cfg.backup_last_check = nowIso;
3566
4175
  cfg.backup_last_success = latest.created;
@@ -3609,7 +4218,14 @@ function backupHealthCore(opts) {
3609
4218
  const maxAge = Number(cfg.backup_max_age_hours) || 36;
3610
4219
  if (cfg.backup_last_error) return { level: 'warning', needsSetup: false, text: '最近一次备份失败: ' + cfg.backup_last_error };
3611
4220
  if (cfg.backup_scheduler_error) return { level: 'warning', needsSetup: false, text: '每日备份调度注册异常: ' + cfg.backup_scheduler_error };
3612
- if (!status.latest) return { level: 'warning', needsSetup: true, text: '自动备份已启用,但还没有成功备份。请检查 backup 目录与调度器。' };
4221
+ if (!status.latest_valid) {
4222
+ const anomalies = (status.anomalies || []).map(function (item) { return item.id + '(' + item.type + ')'; });
4223
+ return {
4224
+ level: 'warning',
4225
+ needsSetup: true,
4226
+ text: '自动备份已启用,但没有可用备份。' + (anomalies.length ? '检测到异常备份: ' + anomalies.join(';') + '。' : '请检查 backup 目录与调度器。'),
4227
+ };
4228
+ }
3613
4229
  if (status.ageHours !== null && status.ageHours > maxAge) {
3614
4230
  return { level: 'warning', needsSetup: false, text: '最近备份已过期(' + status.ageHours.toFixed(1) + ' 小时,超过 ' + maxAge + ' 小时);请检查备份盘与每日调度。' };
3615
4231
  }
@@ -4024,6 +4640,56 @@ function scaleMetric(key, value, unit, info, warn) {
4024
4640
  const level = value > warn ? 'warning' : (value > info ? 'info' : 'ok');
4025
4641
  return { key: key, value: value, unit: unit, info: info, warn: warn, level: level };
4026
4642
  }
4643
+ function typePressureReport(entries, cfg) {
4644
+ const archiveUtility = parseFloat(cfg && cfg.maintain_archived_utility || ARCHIVE_UTILITY_DEFAULT);
4645
+ const archiveAge = parseInt(cfg && cfg.maintain_archived_age || '180', 10) || 180;
4646
+ const report = {};
4647
+ for (const entry of entries) {
4648
+ const type = String(entry.type || 'FACT').toUpperCase();
4649
+ if (!report[type]) {
4650
+ report[type] = {
4651
+ total: 0,
4652
+ stale_180d: 0,
4653
+ archive_candidates: 0,
4654
+ growth_30d: 0,
4655
+ stale_ratio: 0,
4656
+ growth_ratio_30d: 0,
4657
+ pressure_level: 'ok',
4658
+ observation_days: 7,
4659
+ final_thresholds_deferred: true,
4660
+ warnings: [],
4661
+ };
4662
+ }
4663
+ const row = report[type];
4664
+ row.total += 1;
4665
+ const last = entry.last_accessed || entry.updated || entry.created || '';
4666
+ if (last && daysBetween(last, today()) > 180) row.stale_180d += 1;
4667
+ if (entry.created && daysBetween(entry.created, today()) <= 30) row.growth_30d += 1;
4668
+ const createdAge = entry.created ? daysBetween(entry.created, today()) : 0;
4669
+ if (type !== 'BOUND' && !entry.immutable && createdAge > archiveAge && utilityScore(entry) < archiveUtility) {
4670
+ row.archive_candidates += 1;
4671
+ }
4672
+ }
4673
+ for (const type of Object.keys(report)) {
4674
+ const row = report[type];
4675
+ row.stale_ratio = row.total ? Math.round((row.stale_180d / row.total) * 1000) / 1000 : 0;
4676
+ const prior = Math.max(0, row.total - row.growth_30d);
4677
+ row.growth_ratio_30d = prior ? Math.round((row.growth_30d / prior) * 1000) / 1000 : (row.growth_30d ? 1 : 0);
4678
+ if (row.total > 5000) {
4679
+ row.pressure_level = 'warning';
4680
+ row.warnings.push('类型压力: ' + type + ' 总量 ' + row.total + ' 超过 5000,需要分类型瘦身或归档。');
4681
+ } else if (row.total > 500 && row.stale_ratio >= 0.3 && row.archive_candidates > 0) {
4682
+ row.pressure_level = 'warning';
4683
+ row.warnings.push('类型压力: ' + type + ' ' + row.total + ' 条中 ' + row.stale_180d
4684
+ + ' 条超过 180 天未活跃,且有 ' + row.archive_candidates + ' 条归档候选。');
4685
+ } else if (row.total > 500 || row.growth_ratio_30d >= 0.2) {
4686
+ row.pressure_level = 'info';
4687
+ } else {
4688
+ row.pressure_level = 'observe';
4689
+ }
4690
+ }
4691
+ return report;
4692
+ }
4027
4693
  function scaleReport(root, cfg) {
4028
4694
  const thresholds = {};
4029
4695
  for (const key of Object.keys(SCALE_THRESHOLD_DEFAULTS)) thresholds[key] = scaleThreshold(cfg, key);
@@ -4032,6 +4698,13 @@ function scaleReport(root, cfg) {
4032
4698
  const indexBytes = indexSizeBytes(root);
4033
4699
  // v0.23.0 D3:超长 statement(> 2000 字)与单类型堆积(读索引,不逐条解密读文件)
4034
4700
  const scaleEntries = loadIndex(root) || [];
4701
+ let indexData = null;
4702
+ try { indexData = JSON.parse(fs.readFileSync(indexPath(root), 'utf8')); } catch (e) { indexData = null; }
4703
+ const indexShards = indexData && Array.isArray(indexData.shards) ? indexData.shards.length : 0;
4704
+ const indexFormat = indexData
4705
+ ? String(indexData.format || (Array.isArray(indexData.entries) ? 'inline' : (indexShards && indexData.shards.every(isGzipShardName) ? 'gzip-year' : 'json-year')))
4706
+ : 'missing';
4707
+ const indexLogicalBytes = jsonByteLength(scaleEntries);
4035
4708
  let longStatements = 0;
4036
4709
  const typeCounts = { FACT: 0, PREF: 0, BOUND: 0, COMMIT: 0 };
4037
4710
  for (const e of scaleEntries) {
@@ -4049,6 +4722,7 @@ function scaleReport(root, cfg) {
4049
4722
  for (let i = 0; i < 3; i++) samples.push(measureIndexColdStart(root));
4050
4723
  samples.sort(function (a, b) { return a - b; });
4051
4724
  const coldStartMs = samples[Math.floor(samples.length / 2)];
4725
+ const typePressure = typePressureReport(scaleEntries, cfg);
4052
4726
  const metrics = [
4053
4727
  scaleMetric('entries', entries, '条', thresholds.scale_info_entries, thresholds.scale_warn_entries),
4054
4728
  scaleMetric('files_per_dir', filesPerDir, '个文件', thresholds.scale_info_files_per_dir, thresholds.scale_warn_files_per_dir),
@@ -4076,17 +4750,25 @@ function scaleReport(root, cfg) {
4076
4750
  if (metrics[5].level === 'warning') {
4077
4751
  warnings.push('规模: 单类型堆积(' + typeMaxName + ' ' + typeMax + ' 条)超过阈值 ' + thresholds.scale_warn_type_count + '(考虑归档 / 合并 / 精简;COMMIT 收工锚点建议短写)');
4078
4752
  }
4753
+ for (const type of Object.keys(typePressure)) {
4754
+ for (const message of typePressure[type].warnings) warnings.push(message);
4755
+ }
4079
4756
  // v0.18.0 A6:分级只影响 level 与提示文案;超 warn 仍进 warnings(既有行为),info 不阻断。
4080
- const level = metrics.some(function (m) { return m.level === 'warning'; })
4757
+ const pressureWarning = Object.keys(typePressure).some(function (type) { return typePressure[type].pressure_level === 'warning'; });
4758
+ const level = pressureWarning || metrics.some(function (m) { return m.level === 'warning'; })
4081
4759
  ? 'warning'
4082
4760
  : (metrics.some(function (m) { return m.level === 'info'; }) ? 'info' : 'ok');
4083
4761
  return {
4084
4762
  entries: entries,
4085
4763
  files_per_dir: filesPerDir,
4086
4764
  index_bytes: indexBytes,
4765
+ index_logical_bytes: indexLogicalBytes,
4766
+ index_shards: indexShards,
4767
+ index_format: indexFormat,
4087
4768
  cold_start_ms: coldStartMs,
4088
4769
  thresholds: thresholds,
4089
4770
  metrics: metrics,
4771
+ type_pressure: typePressure,
4090
4772
  level: level,
4091
4773
  warnings: warnings,
4092
4774
  };
@@ -4411,7 +5093,7 @@ function isAllowedBackupPath(rel) {
4411
5093
  if (!normalized) return true;
4412
5094
  const top = normalized.split('/')[0];
4413
5095
  if (top === 'facts' || top === 'private' || top === '.archive') return true;
4414
- if (top === 'agents.json' || top === 'index.json' || /^index-\d{4}\.json$/.test(top)) return true;
5096
+ if (top === 'agents.json' || top === 'index.json' || /^index-\d{4}\.json(?:\.gz)?$/.test(top)) return true;
4415
5097
  if (top === 'keys') {
4416
5098
  return normalized !== 'keys/cache' && normalized.indexOf('keys/cache/') !== 0 &&
4417
5099
  normalized !== 'keys/pending' && normalized.indexOf('keys/pending/') !== 0;
@@ -4475,7 +5157,7 @@ function backupListCore(opts) {
4475
5157
  manifest: manifest,
4476
5158
  });
4477
5159
  }
4478
- backups.sort(function (a, b) { return String(b.id).localeCompare(String(a.id)); });
5160
+ backups.sort(compareBackupCreatedDesc);
4479
5161
  const lines = backups.length
4480
5162
  ? backups.map(function (b) { return '- ' + b.id + ' ' + (b.created || '?') + ' ' + (b.files === null ? 'manifest 缺失' : b.files + ' 文件'); })
4481
5163
  : ['(无备份)'];
@@ -4485,7 +5167,7 @@ function backupDoctorCore(opts) {
4485
5167
  opts = opts || {};
4486
5168
  const listed = backupListCore(opts);
4487
5169
  if (listed.error) return { error: true, ok: false, text: listed.text };
4488
- const id = opts.id || (listed.backups[0] && listed.backups[0].id);
5170
+ const id = opts.id || (selectLatestValidBackup(listed.backups, Date.now()).latestValid || {}).id;
4489
5171
  if (!id) return { error: false, ok: false, text: '没有可检查的备份。' };
4490
5172
  const backup = listed.backups.find(function (item) { return item.id === id; });
4491
5173
  if (!backup) return { error: true, ok: false, text: '未找到备份: ' + id };
@@ -7849,6 +8531,29 @@ function cmdBackupDoctor(opts) {
7849
8531
  console.log(r.text);
7850
8532
  if (r.error || !r.ok) process.exit(2);
7851
8533
  }
8534
+ function cmdIndex(opts) {
8535
+ if (!opts.stats) {
8536
+ console.error('index 只提供只读统计:请使用 yotta-memory index --stats [--json]。');
8537
+ process.exit(2);
8538
+ }
8539
+ const r = indexStatsCore(opts);
8540
+ if (opts.json) console.log(JSON.stringify(r, null, 2));
8541
+ else console.log(r.text);
8542
+ if (r.error) process.exit(2);
8543
+ }
8544
+ function cmdLayout(action, opts) {
8545
+ let r;
8546
+ if (action === 'plan') r = layoutPlanCore(opts);
8547
+ else if (action === 'apply') r = layoutApplyCore(Object.assign({}, opts, { planFile: opts.fromPlan }));
8548
+ else if (action === 'rollback') r = layoutRollbackCore(opts);
8549
+ else {
8550
+ console.error('layout 子命令: plan [--threshold N] [--include-flat] [--out <文件>] / apply --from-plan <文件> --yes / rollback --batch <id> --yes');
8551
+ process.exit(2);
8552
+ }
8553
+ if (opts.json) console.log(JSON.stringify(r, null, 2));
8554
+ else console.log(r.text);
8555
+ if (r.error) process.exit(2);
8556
+ }
7852
8557
  function cmdBackupRestore(id, opts) {
7853
8558
  const r = backupRestoreCore(id, opts);
7854
8559
  console.log(r.text);
@@ -9915,6 +10620,7 @@ const SCAN_SEVERITIES = ['safe', 'low', 'medium', 'high', 'critical'];
9915
10620
  const SCAN_MAX_FINDINGS = 500;
9916
10621
  const SCAN_MAX_FILES = 20000;
9917
10622
  const SCAN_QUARANTINE_DIR = '.memory-scan';
10623
+ const SCAN_POLICY_FILE = 'security/scan-policy.json';
9918
10624
  const MEMORY_SCAN_RULES = [
9919
10625
  // ── 恶意指令(元安 DownloadExec / 元盾危险命令)──
9920
10626
  { id: 'YTM-MAL-001', class: 'malicious-instruction', severity: 'critical', ref: 'DEX-001@yotta-security-audit', title: 'curl 下载内容交给 shell 执行', mask: false, re: /\bcurl\b[^\n|;]{0,120}\|\s*(?:ba)?sh\b/i },
@@ -9966,7 +10672,77 @@ function scanEmptySummary() {
9966
10672
  for (const cls of SCAN_CLASSES) byClass[cls] = 0;
9967
10673
  const bySeverity = {};
9968
10674
  for (const sev of SCAN_SEVERITIES) bySeverity[sev] = 0;
9969
- return { files: 0, encrypted: 0, mislabelled: 0, scanned: 0, hits: 0, byClass: byClass, bySeverity: bySeverity, maxSeverity: 'safe', truncated: false };
10675
+ return {
10676
+ files: 0,
10677
+ encrypted: 0,
10678
+ mislabelled: 0,
10679
+ scanned: 0,
10680
+ hits: 0,
10681
+ suppressed: 0,
10682
+ invalid_policy: 0,
10683
+ expired_policy: 0,
10684
+ byClass: byClass,
10685
+ bySeverity: bySeverity,
10686
+ maxSeverity: 'safe',
10687
+ truncated: false,
10688
+ };
10689
+ }
10690
+ function scanLineHash(line) {
10691
+ return crypto.createHash('sha256').update(String(line || ''), 'utf8').digest('hex');
10692
+ }
10693
+ function loadScanPolicy(root, noPolicy) {
10694
+ const pathname = path.join(root, SCAN_POLICY_FILE);
10695
+ if (noPolicy) {
10696
+ return { path: pathname, entries: [], map: new Map(), invalid: 0, expired: 0, used: new Set(), warnings: [] };
10697
+ }
10698
+ if (!fs.existsSync(pathname)) {
10699
+ return { path: pathname, entries: [], map: new Map(), invalid: 0, expired: 0, used: new Set(), warnings: [] };
10700
+ }
10701
+ let data = null;
10702
+ try { data = JSON.parse(fs.readFileSync(pathname, 'utf8')); } catch (error) {
10703
+ return { path: pathname, entries: [], map: new Map(), invalid: 1, expired: 0, used: new Set(), warnings: ['扫描策略 JSON 无法解析: ' + error.message] };
10704
+ }
10705
+ const rawEntries = data && Array.isArray(data.entries) ? data.entries : [];
10706
+ const entries = [];
10707
+ const map = new Map();
10708
+ let invalid = data && Array.isArray(data.entries) ? 0 : 1;
10709
+ let expired = 0;
10710
+ const warnings = [];
10711
+ if (!data || !Array.isArray(data.entries)) warnings.push('扫描策略结构无效:缺少 entries 数组。');
10712
+ for (const raw of rawEntries) {
10713
+ const rule = String(raw && raw.rule || '').trim();
10714
+ const file = String(raw && raw.file || '').replace(/\\/g, '/');
10715
+ const hash = String(raw && raw.line_sha256 || '').trim().toLowerCase();
10716
+ const reviewBy = String(raw && raw.review_by || '').trim();
10717
+ const reason = String(raw && raw.reason || '').trim();
10718
+ const valid = /^YTM-[A-Z]+-\d{3}$/.test(rule) &&
10719
+ isSafeRelPath(file) && file.indexOf('*') === -1 && MEMORY_FILE_RE.test(file) &&
10720
+ /^[a-f0-9]{64}$/.test(hash) &&
10721
+ /^\d{4}-\d{2}-\d{2}$/.test(reviewBy) && !isNaN(new Date(reviewBy).getTime()) &&
10722
+ reason.length > 0;
10723
+ if (!valid) {
10724
+ invalid++;
10725
+ continue;
10726
+ }
10727
+ if (reviewBy < today()) {
10728
+ expired++;
10729
+ continue;
10730
+ }
10731
+ const entry = {
10732
+ rule: rule,
10733
+ file: file,
10734
+ line_sha256: hash,
10735
+ reason: reason,
10736
+ review_by: reviewBy,
10737
+ added_by: String(raw.added_by || ''),
10738
+ added_at: String(raw.added_at || ''),
10739
+ };
10740
+ entries.push(entry);
10741
+ map.set(rule + '\0' + file + '\0' + hash, entry);
10742
+ }
10743
+ if (invalid) warnings.push('扫描策略有 ' + invalid + ' 条无效条目,已 fail-closed 忽略。');
10744
+ if (expired) warnings.push('扫描策略有 ' + expired + ' 条已过期,已恢复为命中。');
10745
+ return { path: pathname, entries: entries, map: map, invalid: invalid, expired: expired, used: new Set(), warnings: warnings };
9970
10746
  }
9971
10747
  function isSafeRelPath(rel) {
9972
10748
  const value = String(rel || '');
@@ -10108,10 +10884,16 @@ function renderScanText(report) {
10108
10884
  '',
10109
10885
  '- 扫描路径: ' + report.root,
10110
10886
  '- 扫描文件: ' + report.summary.files + '(加密跳过 ' + report.summary.encrypted + (report.summary.mislabelled ? ';.enc 后缀但内容是明文 ' + report.summary.mislabelled + ' 个已按文本扫描' : '') + ')',
10111
- '- 结果: ' + label[report.level] + ' · 命中 ' + report.summary.hits + ' 条' + (report.summary.truncated ? '(已达上限,结果被截断)' : ''),
10887
+ '- 结果: ' + label[report.level] + ' · 命中 ' + report.summary.hits + ' 条'
10888
+ + (report.summary.suppressed ? ' · 已抑制 ' + report.summary.suppressed + ' 条' : '')
10889
+ + (report.summary.truncated ? '(已达上限,结果被截断)' : ''),
10112
10890
  '- 分级: ' + SCAN_SEVERITIES.map(function (s) { return s + '=' + report.summary.bySeverity[s]; }).join(' / '),
10113
10891
  '- 类别: ' + SCAN_CLASSES.map(function (c) { return c + '=' + report.summary.byClass[c]; }).join(' / '),
10114
10892
  ];
10893
+ if (report.policy && (report.policy.invalid || report.policy.expired || report.policy.unused)) {
10894
+ lines.push('- 策略: 无效 ' + report.policy.invalid + ' / 过期 ' + report.policy.expired + ' / 未命中 ' + report.policy.unused);
10895
+ }
10896
+ for (const warning of (report.policy && report.policy.warnings) || []) lines.push('- [策略警告] ' + warning);
10115
10897
  if (report.gate) lines.push('- 门禁: 命中 ' + report.gate + ' 及以上即 exit 1(当前 ' + report.level + ')');
10116
10898
  if (!report.findings.length) lines.push('- 未发现可疑内容。');
10117
10899
  for (const finding of report.findings) {
@@ -10149,6 +10931,8 @@ function scanCore(opts) {
10149
10931
  if (opts.restore) return scanRestoreCore(root, opts);
10150
10932
 
10151
10933
  const findings = [];
10934
+ const suppressed = [];
10935
+ const policy = loadScanPolicy(root, opts.noPolicy);
10152
10936
  const byClass = {};
10153
10937
  for (const cls of SCAN_CLASSES) byClass[cls] = 0;
10154
10938
  const bySeverity = {};
@@ -10182,6 +10966,25 @@ function scanCore(opts) {
10182
10966
  const start = Math.max(0, match.index - 40);
10183
10967
  const end = Math.min(line.length, match.index + match[0].length + 40);
10184
10968
  const snippet = scrubSecrets((start > 0 ? '…' : '') + line.slice(start, end) + (end < line.length ? '…' : '')).slice(0, 200);
10969
+ const lineHash = scanLineHash(line);
10970
+ const policyKey = rule.id + '\0' + rel + '\0' + lineHash;
10971
+ if (policy.map.has(policyKey)) {
10972
+ const exception = policy.map.get(policyKey);
10973
+ policy.used.add(policyKey);
10974
+ suppressed.push({
10975
+ file: rel,
10976
+ line: i + 1,
10977
+ evidence: rel + ':' + (i + 1),
10978
+ rule: rule.id,
10979
+ class: rule.class,
10980
+ severity: rule.severity,
10981
+ title: rule.title,
10982
+ line_sha256: lineHash,
10983
+ reason: exception.reason,
10984
+ review_by: exception.review_by,
10985
+ });
10986
+ continue;
10987
+ }
10185
10988
  findings.push({
10186
10989
  file: rel,
10187
10990
  line: i + 1,
@@ -10191,6 +10994,7 @@ function scanCore(opts) {
10191
10994
  severity: rule.severity,
10192
10995
  title: rule.title,
10193
10996
  ref: rule.ref,
10997
+ line_sha256: lineHash,
10194
10998
  match: scrubSecrets(match[0]).slice(0, 80),
10195
10999
  snippet: snippet,
10196
11000
  });
@@ -10213,6 +11017,9 @@ function scanCore(opts) {
10213
11017
  bySeverity: bySeverity,
10214
11018
  maxSeverity: level,
10215
11019
  truncated: truncated,
11020
+ suppressed: suppressed.length,
11021
+ invalid_policy: policy.invalid,
11022
+ expired_policy: policy.expired,
10216
11023
  };
10217
11024
  const report = {
10218
11025
  schemaVersion: SCAN_SCHEMA_VERSION,
@@ -10223,7 +11030,16 @@ function scanCore(opts) {
10223
11030
  root: root,
10224
11031
  gate: gate,
10225
11032
  findings: findings,
11033
+ suppressed: suppressed,
10226
11034
  summary: summary,
11035
+ policy: {
11036
+ path: policy.path,
11037
+ entries: policy.entries.length,
11038
+ invalid: policy.invalid,
11039
+ expired: policy.expired,
11040
+ unused: Math.max(0, policy.entries.length - policy.used.size),
11041
+ warnings: policy.warnings,
11042
+ },
10227
11043
  quarantine: null,
10228
11044
  restore: null,
10229
11045
  };
@@ -10252,6 +11068,64 @@ function scanCore(opts) {
10252
11068
  }
10253
11069
  return report;
10254
11070
  }
11071
+ function scanPolicyCandidatesCore(opts) {
11072
+ opts = opts || {};
11073
+ const root = path.resolve(opts.path || opts.root || userRoot());
11074
+ const scan = scanCore({
11075
+ root: root,
11076
+ noPolicy: true,
11077
+ quarantine: false,
11078
+ restore: false,
11079
+ gate: '',
11080
+ });
11081
+ if (scan.error) return scan;
11082
+ const seen = new Set();
11083
+ const candidates = [];
11084
+ for (const finding of scan.findings) {
11085
+ const key = finding.rule + '\0' + finding.file + '\0' + finding.line_sha256;
11086
+ if (seen.has(key)) continue;
11087
+ seen.add(key);
11088
+ candidates.push({
11089
+ rule: finding.rule,
11090
+ file: finding.file,
11091
+ line: finding.line,
11092
+ line_sha256: finding.line_sha256,
11093
+ class: finding.class,
11094
+ severity: finding.severity,
11095
+ title: finding.title,
11096
+ reason: '',
11097
+ review_by: '',
11098
+ added_by: '',
11099
+ added_at: '',
11100
+ });
11101
+ }
11102
+ let outPath = '';
11103
+ if (opts.out) {
11104
+ const rel = String(opts.out).replace(/\\/g, '/');
11105
+ if (!layoutInsideRoot(root, rel)) {
11106
+ return {
11107
+ schemaVersion: SCAN_SCHEMA_VERSION,
11108
+ error: true,
11109
+ exitCode: 2,
11110
+ candidates: [],
11111
+ text: '拒绝写出策略候选:--out 必须位于当前记忆库内。',
11112
+ };
11113
+ }
11114
+ outPath = path.resolve(root, rel);
11115
+ fs.mkdirSync(path.dirname(outPath), { recursive: true });
11116
+ atomicWriteFileSync(outPath, JSON.stringify({ version: 1, candidates: candidates }, null, 2), 'utf8');
11117
+ }
11118
+ return {
11119
+ schemaVersion: SCAN_SCHEMA_VERSION,
11120
+ error: false,
11121
+ exitCode: 0,
11122
+ root: root,
11123
+ candidates: candidates,
11124
+ out: outPath,
11125
+ text: '扫描策略候选: ' + candidates.length + ' 条'
11126
+ + (outPath ? '\n已写出: ' + outPath : '\n未写文件;请人工复核后配置 security/scan-policy.json。'),
11127
+ };
11128
+ }
10255
11129
  // 交互确认:非 TTY 一律返回 null(由调用方给 --yes 指引),避免自动化里静默改写记忆。
10256
11130
  function promptYesNo(promptText) {
10257
11131
  return new Promise(function (resolve) {
@@ -10283,6 +11157,12 @@ function promptYesNo(promptText) {
10283
11157
  }
10284
11158
  async function cmdScan(opts) {
10285
11159
  opts = opts || {};
11160
+ if (opts.policyCandidates) {
11161
+ const candidates = scanPolicyCandidatesCore(opts);
11162
+ console.log(opts.json ? JSON.stringify(candidates, null, 2) : candidates.text);
11163
+ if (candidates.error) process.exit(candidates.exitCode || 2);
11164
+ return;
11165
+ }
10286
11166
  if (opts.quarantine && !opts.yes) {
10287
11167
  const preview = scanCore(Object.assign({}, opts, { quarantine: false }));
10288
11168
  if (preview.error) {
@@ -12327,6 +13207,10 @@ async function main() {
12327
13207
  else if (a === '--rules') opts.rules = true;
12328
13208
  else if (a === '--propose') opts.propose = true;
12329
13209
  else if (a === '--audit') opts.audit = true;
13210
+ else if (a === '--stats') opts.stats = true;
13211
+ else if (a === '--include-flat') opts.includeFlat = true;
13212
+ else if (a === '--no-policy') opts.noPolicy = true;
13213
+ else if (a === '--policy-candidates') opts.policyCandidates = true;
12330
13214
  else if (valueOpts.has(a)) {
12331
13215
  const v = args[++i];
12332
13216
  if (a === '--type') opts.type = v;
@@ -12382,6 +13266,8 @@ async function main() {
12382
13266
  else if (a === '--template') opts.template = v;
12383
13267
  else if (a === '--path') opts.path = v;
12384
13268
  else if (a === '--from') opts.from = v;
13269
+ else if (a === '--from-plan') opts.fromPlan = v;
13270
+ else if (a === '--batch') opts.batch = v;
12385
13271
  } else if (a.startsWith('--')) {
12386
13272
  if (a === '--query') {
12387
13273
  console.error('未知选项: --query。recall/search 的关键词是位置参数:yotta-memory recall [关键词]');
@@ -12458,6 +13344,8 @@ async function main() {
12458
13344
  break;
12459
13345
  }
12460
13346
  case 'reindex': cmdReindex(); break;
13347
+ case 'index': cmdIndex(opts); break;
13348
+ case 'layout': cmdLayout(rest[0], opts); break;
12461
13349
  case 'doctor': cmdDoctor(opts); break;
12462
13350
  case 'profile': cmdProfile(opts); break;
12463
13351
  case 'context': cmdContext(opts); break;
@@ -12565,6 +13453,9 @@ module.exports = {
12565
13453
  viewEntriesCore: viewEntriesCore,
12566
13454
  viewOverviewCore: viewOverviewCore,
12567
13455
  migrateLayout: migrateLayout,
13456
+ layoutPlanCore: layoutPlanCore,
13457
+ layoutApplyCore: layoutApplyCore,
13458
+ layoutRollbackCore: layoutRollbackCore,
12568
13459
  loadIndexFor: loadIndexFor,
12569
13460
  indexEntriesFor: indexEntriesFor,
12570
13461
  loadOwnerIndex: loadOwnerIndex,
@@ -12575,6 +13466,10 @@ module.exports = {
12575
13466
  baselineCore: baselineCore,
12576
13467
  loadBaselineTemplate: loadBaselineTemplate,
12577
13468
  scanCore: scanCore,
13469
+ scanPolicyCandidatesCore: scanPolicyCandidatesCore,
13470
+ loadScanPolicy: loadScanPolicy,
13471
+ scanLineHash: scanLineHash,
13472
+ SCAN_POLICY_FILE: SCAN_POLICY_FILE,
12578
13473
  scanQuarantineCore: scanQuarantineCore,
12579
13474
  scanRestoreCore: scanRestoreCore,
12580
13475
  promptYesNo: promptYesNo,
@@ -12702,6 +13597,10 @@ module.exports = {
12702
13597
 
12703
13598
  loadIndex: loadIndex,
12704
13599
  saveIndex: saveIndex,
13600
+ indexStatsCore: indexStatsCore,
13601
+ INDEX_VERSION: INDEX_VERSION,
13602
+ INDEX_SHARD_THRESHOLD: INDEX_SHARD_THRESHOLD,
13603
+ INDEX_COMPRESS_TRIGGER_BYTES: INDEX_COMPRESS_TRIGGER_BYTES,
12705
13604
  buildIndex: buildIndex,
12706
13605
  getIndex: getIndex,
12707
13606
  touchIndex: touchIndex,
@@ -12736,6 +13635,8 @@ module.exports = {
12736
13635
  listBackupVolumesCore: listBackupVolumesCore,
12737
13636
  backupSetupCore: backupSetupCore,
12738
13637
  backupStatusCore: backupStatusCore,
13638
+ selectLatestValidBackup: selectLatestValidBackup,
13639
+ BACKUP_FUTURE_TOLERANCE_MS: BACKUP_FUTURE_TOLERANCE_MS,
12739
13640
  backupEnsureDailyCore: backupEnsureDailyCore,
12740
13641
  backupHealthCore: backupHealthCore,
12741
13642
  doctorCore: doctorCore,