@yottameta/yotta-memory 0.23.3 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +2 -1
- package/README.zh-CN.md +2 -1
- package/SKILL.md +14 -6
- package/USER_GUIDE.md +20 -3
- package/bin/yotta-memory.js +958 -57
- package/package.json +1 -1
- package/references/faq.md +7 -1
- package/references/protocol.md +9 -7
- package/skill-manifest.json +1 -1
package/bin/yotta-memory.js
CHANGED
|
@@ -22,12 +22,13 @@ const os = require('os');
|
|
|
22
22
|
const crypto = require('crypto');
|
|
23
23
|
const http = require('http');
|
|
24
24
|
const net = require('net');
|
|
25
|
+
const zlib = require('zlib');
|
|
25
26
|
const child_process = require('child_process');
|
|
26
27
|
const { AsyncLocalStorage } = require('async_hooks');
|
|
27
28
|
|
|
28
|
-
const VERSION = '0.
|
|
29
|
-
const CLI_VALUE_OPTS = new Set(['--type', '--limit', '--days', '--out', '--owner', '--agent', '--agent-id', '--agent-key', '--agent-key-file', '--plugin-data', '--threshold', '--scope', '--host', '--port', '--dir', '--name', '--user', '--relationship', '--source', '--weight', '--budget', '--password', '--new-password', '--recovery-key', '--recovery-key-out', '--reason', '--merge', '--model', '--subject', '--embedding', '--focus', '--embedding-timeout', '--min-age', '--min-idle', '--max-utility', '--min-group', '--period', '--to', '--id', '--time', '--tools', '--mcp-config', '--skill-dir', '--year', '--evalset', '--k', '--seed', '--bootstrap', '--gate', '--against', '--template', '--path', '--from']);
|
|
30
|
-
const CLI_FLAG_OPTS = new Set(['--project', '--all', '--unsafe', '--no-auth', '--stdio', '--onstart', '--from-current', '--restart', '--force', '--attach', '--allow-same-volume', '--verify', '--no-hint', '--encrypt', '--no-encrypt', '--password-stdin', '--json', '--manual', '--skip-schedule', '--useful', '--useless', '--undo', '--dry-run', '--apply', '--purge', '--dedup', '--batches', '--propose', '--audit', '--capacity', '--rules', '--explain', '--semantic', '--runtime', '--ablate', '--timing', '--baseline', '--probe', '--quarantine', '--restore', '--no-usage', '--yes', '--keep-memories', '--keep-identity']);
|
|
29
|
+
const VERSION = '0.24.0';
|
|
30
|
+
const CLI_VALUE_OPTS = new Set(['--type', '--limit', '--days', '--out', '--owner', '--agent', '--agent-id', '--agent-key', '--agent-key-file', '--plugin-data', '--threshold', '--scope', '--host', '--port', '--dir', '--name', '--user', '--relationship', '--source', '--weight', '--budget', '--password', '--new-password', '--recovery-key', '--recovery-key-out', '--reason', '--merge', '--model', '--subject', '--embedding', '--focus', '--embedding-timeout', '--min-age', '--min-idle', '--max-utility', '--min-group', '--period', '--to', '--id', '--time', '--tools', '--mcp-config', '--skill-dir', '--year', '--evalset', '--k', '--seed', '--bootstrap', '--gate', '--against', '--template', '--path', '--from', '--from-plan', '--batch']);
|
|
31
|
+
const CLI_FLAG_OPTS = new Set(['--project', '--all', '--unsafe', '--no-auth', '--stdio', '--onstart', '--from-current', '--restart', '--force', '--attach', '--allow-same-volume', '--verify', '--no-hint', '--encrypt', '--no-encrypt', '--password-stdin', '--json', '--manual', '--skip-schedule', '--useful', '--useless', '--undo', '--dry-run', '--apply', '--purge', '--dedup', '--batches', '--propose', '--audit', '--capacity', '--rules', '--explain', '--semantic', '--runtime', '--ablate', '--timing', '--baseline', '--probe', '--quarantine', '--restore', '--no-usage', '--yes', '--keep-memories', '--keep-identity', '--stats', '--include-flat', '--no-policy', '--policy-candidates']);
|
|
31
32
|
|
|
32
33
|
function helpOption(flag, arg, what, when, caution) {
|
|
33
34
|
return { flag: flag, arg: arg || '', what: what, when: when || '', caution: caution || '' };
|
|
@@ -119,6 +120,28 @@ const HELP_MODEL = [
|
|
|
119
120
|
helpOption('--template', '<文件>', '用模板声明显式期望(owner / 最小条数 / 类型条数 / 查询)', '有明确验收清单、要按清单核对时', '模板只加严判定,不会放宽或改写探针'),
|
|
120
121
|
helpOption('--json', '', '输出 JSON', '脚本或自动化读取体检结果时', ''),
|
|
121
122
|
] },
|
|
123
|
+
{ name: 'index', usage: 'index --stats [--json]', what: '只读分析公共索引体积、字段占比与 gzip 分片预估', when: '判断是否需要升级索引格式、压缩或分片时', options: [
|
|
124
|
+
helpOption('--stats', '', '输出索引统计报告', '分析索引膨胀根因与预估压缩收益时', '只读:不重建索引,不改任何记忆文件'),
|
|
125
|
+
helpOption('--json', '', '输出 JSON', '脚本读取字段体积、token 与压缩预估时', ''),
|
|
126
|
+
] },
|
|
127
|
+
{ name: 'layout', usage: 'layout <plan|apply|rollback> [选项]', what: '预演、执行或回滚日级目录重平衡', when: '单个年/月目录文件数超过阈值、需要可回滚迁移时', subcommands: [
|
|
128
|
+
helpSub('plan', 'plan [--threshold <数量>] [--include-flat] [--out <文件>] [--json]', '生成布局迁移计划', '先确定会移动哪些文件、目标路径和冲突时', [
|
|
129
|
+
helpOption('--threshold', '<数量>', '目录文件数超过该值时进入计划', '默认阈值不合适时', ''),
|
|
130
|
+
helpOption('--include-flat', '', '把旧平铺根也纳入计划', '确认愿意迁移旧平铺文件时', '默认只处理年/月桶,旧平铺保持原位'),
|
|
131
|
+
helpOption('--out', '<文件>', '把计划写到指定位置', '需要把计划交给维护窗口审阅时', ''),
|
|
132
|
+
helpOption('--json', '', '输出 JSON', '自动化读取候选、操作与冲突时', ''),
|
|
133
|
+
]),
|
|
134
|
+
helpSub('apply', 'apply --from-plan <文件> --yes [--json]', '执行已审阅的布局迁移计划', '维护窗口内确认计划无冲突后', [
|
|
135
|
+
helpOption('--from-plan', '<文件>', '指定布局迁移计划文件', '执行 plan 生成的计划时', ''),
|
|
136
|
+
helpOption('--yes', '', '跳过交互确认', '计划已审阅且维护窗口已开始时', '会移动记忆文件;失败时自动逆序回滚'),
|
|
137
|
+
helpOption('--json', '', '输出 JSON', '自动化执行迁移时', ''),
|
|
138
|
+
]),
|
|
139
|
+
helpSub('rollback', 'rollback --batch <id> --yes [--json]', '按 journal 回滚一次布局迁移', '迁移后发现问题、需要恢复原路径时', [
|
|
140
|
+
helpOption('--batch', '<id>', '指定布局迁移批次', '找回滚 journal 时', ''),
|
|
141
|
+
helpOption('--yes', '', '跳过交互确认', '已确认要恢复原布局时', '会移动记忆文件;按 journal 逆序恢复'),
|
|
142
|
+
helpOption('--json', '', '输出 JSON', '自动化回滚时', ''),
|
|
143
|
+
]),
|
|
144
|
+
] },
|
|
122
145
|
{ name: 'maintain', usage: 'maintain [选项]', what: '记忆自组织:归档、遗忘候选、去重和合并', when: '定期清理低价值记忆、合并重复条目时', options: [
|
|
123
146
|
helpOption('--capacity', '', '查看容量水位与淘汰 / 晋升建议', '库变大,想看哪些条目值得清理或提升时', '只读报告,不会自动归档或改权重'),
|
|
124
147
|
helpOption('--rules', '', '查看重复踩坑 → 建议升级为规则的清单', '同一类问题反复出现、想把它固化成 BOUND 规则时', '只读报告,不自动写规则;阈值走 config maintain_rule_min_hits'),
|
|
@@ -183,6 +206,9 @@ const HELP_MODEL = [
|
|
|
183
206
|
helpOption('--yes', '', '跳过交互确认', '自动化里执行隔离时', '跳过确认后不再二次询问,请先看一遍命中清单'),
|
|
184
207
|
helpOption('--restore', '', '还原最近一个隔离批次', '隔离后发现问题、要恢复原文时', '会覆盖当前文件;批次已还原过则拒绝重复还原'),
|
|
185
208
|
helpOption('--id', '<id>', '指定要还原的隔离批次', '要还原的不是最近一批时', ''),
|
|
209
|
+
helpOption('--no-policy', '', '忽略扫描例外策略做全量复核', '怀疑策略过期或想重新确认所有历史命中时', '会重新显示已抑制命中,不改策略文件'),
|
|
210
|
+
helpOption('--policy-candidates', '', '只生成例外策略候选清单', '历史安全语料需要人工复核后配置例外时', '只读扫描;不自动写入 security/scan-policy.json'),
|
|
211
|
+
helpOption('--out', '<文件>', '把候选清单写到文件', '想把候选交给人工复核或评审时', '仅用于 --policy-candidates;不会写策略文件'),
|
|
186
212
|
] },
|
|
187
213
|
{ name: 'reindex', usage: 'reindex', what: '重建记忆索引', when: '索引损坏、手动改过记忆文件或 doctor 提示索引异常时', options: [] },
|
|
188
214
|
{ name: 'export', usage: 'export --out <文件.json>', what: '导出全部记忆', when: '备份、迁移或做离线检查时', options: [helpOption('--out', '<文件>', '指定导出文件', '导出时必须给出目标 JSON 文件', '')] },
|
|
@@ -332,6 +358,11 @@ const VIEW_HELP_QUICK = [
|
|
|
332
358
|
{ cmd: 'backup restore', example: 'yotta-memory backup restore <id> --to <目录>', danger: true, note: '恢复会写入目标目录;覆盖非空目录需显式 --force,先确认目标和备份' },
|
|
333
359
|
{ cmd: 'backup drill', example: 'yotta-memory backup drill <id>', note: '先演练确认备份可用,再正式恢复' },
|
|
334
360
|
{ cmd: 'doctor', example: 'yotta-memory doctor' },
|
|
361
|
+
{ cmd: 'index', example: 'yotta-memory index --stats --json', note: '只读统计;--stats 不会重建索引' },
|
|
362
|
+
{ cmd: 'layout', example: 'yotta-memory layout <子命令>' },
|
|
363
|
+
{ cmd: 'layout plan', example: 'yotta-memory layout plan --json' },
|
|
364
|
+
{ cmd: 'layout apply', example: 'yotta-memory layout apply --from-plan <文件> --yes', danger: true, note: '会移动记忆文件;先审阅计划,失败时自动逆序回滚' },
|
|
365
|
+
{ cmd: 'layout rollback', example: 'yotta-memory layout rollback --batch <id> --yes', danger: true, note: '按 journal 逆序恢复原路径;先确认批次' },
|
|
335
366
|
{ cmd: 'maintain', example: 'yotta-memory maintain --capacity', note: '只读报告,不会自动归档或改权重' },
|
|
336
367
|
{ cmd: 'consolidate', example: 'yotta-memory consolidate --propose' },
|
|
337
368
|
{ cmd: 'distill', example: 'yotta-memory distill --model <命令>' },
|
|
@@ -1077,12 +1108,13 @@ function typeSubdir(type, owner) {
|
|
|
1077
1108
|
const id = owner || '';
|
|
1078
1109
|
return path.join(PRIVATE_DIR, id, TYPE_DIRS[t]);
|
|
1079
1110
|
}
|
|
1080
|
-
// v0.
|
|
1111
|
+
// v0.24.0 规模:新写入按 <yyyy>/<mm>/<dd> 分层(公共 facts/<yyyy>/<mm>/<dd>/,
|
|
1112
|
+
// 私密 private/<owner>/<type>/<yyyy>/<mm>/<dd>/)。
|
|
1081
1113
|
// 只影响新写入;旧平铺文件保持原位继续可读,不自动迁移。
|
|
1082
1114
|
function dateSubdir(dateStr) {
|
|
1083
1115
|
const d = String(dateStr || '');
|
|
1084
1116
|
if (!/^\d{4}-\d{2}-\d{2}$/.test(d)) return '';
|
|
1085
|
-
return path.join(d.slice(0, 4), d.slice(5, 7));
|
|
1117
|
+
return path.join(d.slice(0, 4), d.slice(5, 7), d.slice(8, 10));
|
|
1086
1118
|
}
|
|
1087
1119
|
function entryWriteDir(root, type, owner, dateStr) {
|
|
1088
1120
|
const base = path.join(root, typeSubdir(type, owner));
|
|
@@ -1717,10 +1749,7 @@ function recentlyWrittenFiles(typeDir, writeDir) {
|
|
|
1717
1749
|
const yearDir = path.dirname(writeDir);
|
|
1718
1750
|
const seen = new Set(out);
|
|
1719
1751
|
if (fs.existsSync(yearDir)) {
|
|
1720
|
-
|
|
1721
|
-
const full = path.join(yearDir, fp);
|
|
1722
|
-
if (fs.statSync(full).isFile() && MEMORY_FILE_RE.test(fp) && !seen.has(full)) out.push(full);
|
|
1723
|
-
}
|
|
1752
|
+
walkEntryFiles(yearDir, out);
|
|
1724
1753
|
}
|
|
1725
1754
|
if (fs.existsSync(typeDir)) {
|
|
1726
1755
|
for (const f of fs.readdirSync(typeDir)) {
|
|
@@ -2287,14 +2316,38 @@ function writeProfileText(root, owner, text) {
|
|
|
2287
2316
|
|
|
2288
2317
|
// ---- 索引(index.json)----
|
|
2289
2318
|
function indexPath(root) { return path.join(root, INDEX_FILE); }
|
|
2290
|
-
const INDEX_VERSION =
|
|
2291
|
-
|
|
2319
|
+
const INDEX_VERSION = 5;
|
|
2320
|
+
const INDEX_MIN_SUPPORTED_VERSION = 4;
|
|
2321
|
+
// v0.24.0:索引按逻辑体积或条目数触发年份分片;大库使用 gzip 分片。
|
|
2292
2322
|
const INDEX_SHARD_THRESHOLD = 5000;
|
|
2293
|
-
|
|
2294
|
-
|
|
2323
|
+
const INDEX_COMPRESS_TRIGGER_BYTES = 4 * 1024 * 1024;
|
|
2324
|
+
// 安全校验:只接受 index-YYYY.json(.gz),防 manifest 篡改导致路径穿越。
|
|
2325
|
+
const SHARD_RE = /^index-\d{4}\.json(?:\.gz)?$/;
|
|
2295
2326
|
function isSafeShardName(name) { return typeof name === 'string' && SHARD_RE.test(name); }
|
|
2296
|
-
function
|
|
2297
|
-
function
|
|
2327
|
+
function isGzipShardName(name) { return typeof name === 'string' && /\.json\.gz$/.test(name); }
|
|
2328
|
+
function indexShardName(year, compressed) { return 'index-' + year + '.json' + (compressed === false ? '' : '.gz'); }
|
|
2329
|
+
function shardYear(name) {
|
|
2330
|
+
const match = String(name || '').match(/^index-(\d{4})\.json(?:\.gz)?$/);
|
|
2331
|
+
return match ? match[1] : '';
|
|
2332
|
+
}
|
|
2333
|
+
function readIndexShard(root, name) {
|
|
2334
|
+
if (!isSafeShardName(name)) return null;
|
|
2335
|
+
const full = path.join(root, name);
|
|
2336
|
+
if (!fs.existsSync(full)) return null;
|
|
2337
|
+
try {
|
|
2338
|
+
const raw = fs.readFileSync(full);
|
|
2339
|
+
const text = isGzipShardName(name) ? zlib.gunzipSync(raw).toString('utf8') : raw.toString('utf8');
|
|
2340
|
+
const parsed = JSON.parse(text);
|
|
2341
|
+
return parsed && Array.isArray(parsed.entries) ? parsed : null;
|
|
2342
|
+
} catch (error) {
|
|
2343
|
+
return null;
|
|
2344
|
+
}
|
|
2345
|
+
}
|
|
2346
|
+
function writeIndexShard(root, name, payload) {
|
|
2347
|
+
const raw = Buffer.from(JSON.stringify(payload), 'utf8');
|
|
2348
|
+
const data = isGzipShardName(name) ? zlib.gzipSync(raw, { level: 9, mtime: 0 }) : raw;
|
|
2349
|
+
atomicWriteFileSync(path.join(root, name), data);
|
|
2350
|
+
}
|
|
2298
2351
|
function entryYear(entry) {
|
|
2299
2352
|
return String((entry && entry.created) || '').slice(0, 4) || String(today()).slice(0, 4);
|
|
2300
2353
|
}
|
|
@@ -2317,7 +2370,7 @@ function loadIndexManifest(root) {
|
|
|
2317
2370
|
if (!fs.existsSync(p)) return null;
|
|
2318
2371
|
try {
|
|
2319
2372
|
const d = JSON.parse(fs.readFileSync(p, 'utf8'));
|
|
2320
|
-
if (d && Array.isArray(d.shards)) return d;
|
|
2373
|
+
if (d && Array.isArray(d.shards) && (!d.version || d.version >= INDEX_MIN_SUPPORTED_VERSION)) return d;
|
|
2321
2374
|
} catch (e) { /* ignore */ }
|
|
2322
2375
|
return null;
|
|
2323
2376
|
}
|
|
@@ -2328,20 +2381,17 @@ function loadIndex(root) {
|
|
|
2328
2381
|
const d = JSON.parse(fs.readFileSync(p, 'utf8'));
|
|
2329
2382
|
if (!d) return null;
|
|
2330
2383
|
if (Array.isArray(d.entries)) {
|
|
2331
|
-
//
|
|
2332
|
-
if (d.version && d.version >=
|
|
2384
|
+
// v4 平铺索引与 v5 平铺索引都继续读取;更旧版本返回 null 触发重建。
|
|
2385
|
+
if (d.version && d.version >= INDEX_MIN_SUPPORTED_VERSION) return d.entries;
|
|
2333
2386
|
return null;
|
|
2334
2387
|
}
|
|
2335
2388
|
if (Array.isArray(d.shards)) {
|
|
2336
|
-
|
|
2337
|
-
if (d.version && d.version < INDEX_VERSION) return null;
|
|
2389
|
+
if (d.version && d.version < INDEX_MIN_SUPPORTED_VERSION) return null;
|
|
2338
2390
|
const out = [];
|
|
2339
2391
|
for (const sh of d.shards) {
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
const sd = JSON.parse(fs.readFileSync(sp, 'utf8'));
|
|
2344
|
-
if (sd && Array.isArray(sd.entries)) out.push.apply(out, sd.entries);
|
|
2392
|
+
const sd = readIndexShard(root, sh);
|
|
2393
|
+
if (!sd) return null;
|
|
2394
|
+
out.push.apply(out, sd.entries);
|
|
2345
2395
|
}
|
|
2346
2396
|
return out;
|
|
2347
2397
|
}
|
|
@@ -2361,21 +2411,20 @@ function loadIndexFor(root, options) {
|
|
|
2361
2411
|
const d = JSON.parse(fs.readFileSync(p, 'utf8'));
|
|
2362
2412
|
if (!d) return null;
|
|
2363
2413
|
if (Array.isArray(d.entries)) {
|
|
2364
|
-
if (d.version && d.version <
|
|
2414
|
+
if (d.version && d.version < INDEX_MIN_SUPPORTED_VERSION) return null;
|
|
2365
2415
|
const want = new Set(years);
|
|
2366
2416
|
return d.entries.filter(function (e) { return want.has(entryYear(e)); });
|
|
2367
2417
|
}
|
|
2368
2418
|
if (Array.isArray(d.shards)) {
|
|
2369
|
-
if (d.version && d.version <
|
|
2419
|
+
if (d.version && d.version < INDEX_MIN_SUPPORTED_VERSION) return null;
|
|
2370
2420
|
const want = new Set(years);
|
|
2371
2421
|
const out = [];
|
|
2372
2422
|
for (const sh of d.shards) {
|
|
2373
2423
|
if (!isSafeShardName(sh)) continue;
|
|
2374
2424
|
if (!want.has(shardYear(sh))) continue; // 只读取命中年份的分片文件
|
|
2375
|
-
const
|
|
2376
|
-
if (!
|
|
2377
|
-
|
|
2378
|
-
if (sd && Array.isArray(sd.entries)) out.push.apply(out, sd.entries);
|
|
2425
|
+
const sd = readIndexShard(root, sh);
|
|
2426
|
+
if (!sd) return null;
|
|
2427
|
+
out.push.apply(out, sd.entries);
|
|
2379
2428
|
}
|
|
2380
2429
|
return out;
|
|
2381
2430
|
}
|
|
@@ -2395,10 +2444,15 @@ function indexEntriesFor(root, options) {
|
|
|
2395
2444
|
return { entries: all.filter(function (e) { return want.has(entryYear(e)); }), error: '' };
|
|
2396
2445
|
}
|
|
2397
2446
|
function getIndex(root) { return loadIndex(root) || []; }
|
|
2398
|
-
function saveIndex(root, entries) {
|
|
2447
|
+
function saveIndex(root, entries, options) {
|
|
2448
|
+
options = options || {};
|
|
2399
2449
|
const clean = entries.map(function (e) { const c = Object.assign({}, e); delete c.meta; return c; });
|
|
2400
2450
|
const old = loadIndexManifest(root);
|
|
2401
|
-
|
|
2451
|
+
const shardThreshold = options.shardThresholdEntries === undefined ? INDEX_SHARD_THRESHOLD : Number(options.shardThresholdEntries);
|
|
2452
|
+
const byteThreshold = options.compressThresholdBytes === undefined ? INDEX_COMPRESS_TRIGGER_BYTES : Number(options.compressThresholdBytes);
|
|
2453
|
+
const compress = options.compress !== false;
|
|
2454
|
+
const logicalBytes = jsonByteLength(clean);
|
|
2455
|
+
if (clean.length > shardThreshold || logicalBytes > byteThreshold) {
|
|
2402
2456
|
const byYear = {};
|
|
2403
2457
|
for (const e of clean) {
|
|
2404
2458
|
const y = entryYear(e);
|
|
@@ -2406,21 +2460,206 @@ function saveIndex(root, entries) {
|
|
|
2406
2460
|
}
|
|
2407
2461
|
const newShards = [];
|
|
2408
2462
|
for (const y of Object.keys(byYear).sort()) {
|
|
2409
|
-
const name = indexShardName(y);
|
|
2410
|
-
|
|
2463
|
+
const name = indexShardName(y, compress);
|
|
2464
|
+
writeIndexShard(root, name, { version: INDEX_VERSION, year: (parseInt(y, 10) || 0), entries: byYear[y] });
|
|
2411
2465
|
newShards.push(name);
|
|
2412
2466
|
}
|
|
2413
|
-
atomicWriteFileSync(indexPath(root), JSON.stringify({
|
|
2467
|
+
atomicWriteFileSync(indexPath(root), JSON.stringify({
|
|
2468
|
+
version: INDEX_VERSION,
|
|
2469
|
+
updated: today(),
|
|
2470
|
+
format: compress ? 'gzip-year' : 'json-year',
|
|
2471
|
+
shards: newShards,
|
|
2472
|
+
count: clean.length,
|
|
2473
|
+
}, null, 2), 'utf8');
|
|
2414
2474
|
if (old && Array.isArray(old.shards)) {
|
|
2415
2475
|
for (const sh of old.shards) if (newShards.indexOf(sh) === -1 && isSafeShardName(sh)) { try { fs.unlinkSync(path.join(root, sh)); } catch (e) {} }
|
|
2416
2476
|
}
|
|
2417
2477
|
} else {
|
|
2478
|
+
atomicWriteFileSync(indexPath(root), JSON.stringify({ version: INDEX_VERSION, updated: today(), entries: clean }, null, 2), 'utf8');
|
|
2418
2479
|
if (old && Array.isArray(old.shards)) {
|
|
2419
2480
|
for (const sh of old.shards) { try { fs.unlinkSync(path.join(root, sh)); } catch (e) {} }
|
|
2420
2481
|
}
|
|
2421
|
-
atomicWriteFileSync(indexPath(root), JSON.stringify({ version: INDEX_VERSION, updated: today(), entries: clean }, null, 2), 'utf8');
|
|
2422
2482
|
}
|
|
2423
2483
|
}
|
|
2484
|
+
function jsonByteLength(value) {
|
|
2485
|
+
return Buffer.byteLength(JSON.stringify(value === undefined ? null : value), 'utf8');
|
|
2486
|
+
}
|
|
2487
|
+
function indexFieldBytes(entries) {
|
|
2488
|
+
const groups = {
|
|
2489
|
+
tokens: ['tokens'],
|
|
2490
|
+
statement: ['statement'],
|
|
2491
|
+
subject: ['subject'],
|
|
2492
|
+
tags: ['tags'],
|
|
2493
|
+
usage: ['access_count', 'last_accessed', 'feedback_net', 'hit_days', 'hit_queries'],
|
|
2494
|
+
};
|
|
2495
|
+
const grouped = new Set();
|
|
2496
|
+
for (const names of Object.values(groups)) for (const name of names) grouped.add(name);
|
|
2497
|
+
const out = { tokens: 0, statement: 0, subject: 0, tags: 0, usage: 0, other: 0, total: 0, overhead: 0 };
|
|
2498
|
+
for (const entry of entries) {
|
|
2499
|
+
out.total += jsonByteLength(entry);
|
|
2500
|
+
for (const group of Object.keys(groups)) {
|
|
2501
|
+
for (const name of groups[group]) out[group] += jsonByteLength(entry[name]);
|
|
2502
|
+
}
|
|
2503
|
+
for (const name of Object.keys(entry)) {
|
|
2504
|
+
if (!grouped.has(name)) out.other += jsonByteLength(entry[name]);
|
|
2505
|
+
}
|
|
2506
|
+
}
|
|
2507
|
+
out.overhead = Math.max(0, out.total - (out.tokens + out.statement + out.subject + out.tags + out.usage + out.other));
|
|
2508
|
+
return out;
|
|
2509
|
+
}
|
|
2510
|
+
function indexTokenStats(entries) {
|
|
2511
|
+
let unique = 0;
|
|
2512
|
+
let weighted = 0;
|
|
2513
|
+
let pinyin = 0;
|
|
2514
|
+
for (const entry of entries) {
|
|
2515
|
+
const tokens = entry && entry.tokens && typeof entry.tokens === 'object' ? entry.tokens : {};
|
|
2516
|
+
for (const key of Object.keys(tokens)) {
|
|
2517
|
+
unique += 1;
|
|
2518
|
+
weighted += Number(tokens[key]) || 0;
|
|
2519
|
+
if (key.indexOf('py:') === 0 || key.indexOf('pyi:') === 0) pinyin += 1;
|
|
2520
|
+
}
|
|
2521
|
+
}
|
|
2522
|
+
return {
|
|
2523
|
+
unique: unique,
|
|
2524
|
+
weighted: Math.round(weighted * 1000) / 1000,
|
|
2525
|
+
pinyin: pinyin,
|
|
2526
|
+
averageUniquePerEntry: entries.length ? Math.round((unique / entries.length) * 1000) / 1000 : 0,
|
|
2527
|
+
};
|
|
2528
|
+
}
|
|
2529
|
+
function indexGzipProjection(entries) {
|
|
2530
|
+
const byYear = {};
|
|
2531
|
+
for (const entry of entries) {
|
|
2532
|
+
const clean = Object.assign({}, entry);
|
|
2533
|
+
delete clean.meta;
|
|
2534
|
+
const year = entryYear(clean);
|
|
2535
|
+
(byYear[year] = byYear[year] || []).push(clean);
|
|
2536
|
+
}
|
|
2537
|
+
const shards = [];
|
|
2538
|
+
let gzipBytes = 0;
|
|
2539
|
+
for (const year of Object.keys(byYear).sort()) {
|
|
2540
|
+
const raw = Buffer.from(JSON.stringify({ version: 5, year: parseInt(year, 10) || 0, entries: byYear[year] }), 'utf8');
|
|
2541
|
+
const compressed = zlib.gzipSync(raw, { level: 9, mtime: 0 });
|
|
2542
|
+
gzipBytes += compressed.length;
|
|
2543
|
+
shards.push({
|
|
2544
|
+
year: year,
|
|
2545
|
+
entries: byYear[year].length,
|
|
2546
|
+
jsonBytes: raw.length,
|
|
2547
|
+
gzipBytes: compressed.length,
|
|
2548
|
+
});
|
|
2549
|
+
}
|
|
2550
|
+
const manifest = Buffer.from(JSON.stringify({
|
|
2551
|
+
version: 5,
|
|
2552
|
+
updated: '',
|
|
2553
|
+
shards: shards.map(function (item) { return 'index-' + item.year + '.json.gz'; }),
|
|
2554
|
+
count: entries.length,
|
|
2555
|
+
}), 'utf8');
|
|
2556
|
+
const storageBytes = gzipBytes + manifest.length;
|
|
2557
|
+
return {
|
|
2558
|
+
format: 'index-v5-gzip-year',
|
|
2559
|
+
storageBytes: storageBytes,
|
|
2560
|
+
manifestBytes: manifest.length,
|
|
2561
|
+
shards: shards,
|
|
2562
|
+
};
|
|
2563
|
+
}
|
|
2564
|
+
function indexStatsCore(opts) {
|
|
2565
|
+
opts = opts || {};
|
|
2566
|
+
const root = path.resolve(opts.root || userRoot());
|
|
2567
|
+
const idx = indexPath(root);
|
|
2568
|
+
if (!fs.existsSync(idx)) {
|
|
2569
|
+
return { error: true, format: 'missing', entryCount: 0, storageBytes: 0, text: '索引不存在: ' + idx + '。请先运行 yotta-memory reindex。' };
|
|
2570
|
+
}
|
|
2571
|
+
let manifestBytes = 0;
|
|
2572
|
+
let data = null;
|
|
2573
|
+
try {
|
|
2574
|
+
manifestBytes = fs.statSync(idx).size;
|
|
2575
|
+
data = JSON.parse(fs.readFileSync(idx, 'utf8'));
|
|
2576
|
+
} catch (error) {
|
|
2577
|
+
return { error: true, format: 'invalid', entryCount: 0, storageBytes: manifestBytes, text: '索引无法解析: ' + error.message };
|
|
2578
|
+
}
|
|
2579
|
+
if (!data || (!Array.isArray(data.entries) && !Array.isArray(data.shards))) {
|
|
2580
|
+
return { error: true, format: 'invalid', entryCount: 0, storageBytes: manifestBytes, text: '索引结构无效:缺少 entries 或 shards 数组。' };
|
|
2581
|
+
}
|
|
2582
|
+
const warnings = [];
|
|
2583
|
+
const entries = [];
|
|
2584
|
+
let shardBytes = 0;
|
|
2585
|
+
const shards = [];
|
|
2586
|
+
if (Array.isArray(data.entries)) {
|
|
2587
|
+
entries.push.apply(entries, data.entries);
|
|
2588
|
+
} else {
|
|
2589
|
+
for (const shard of data.shards) {
|
|
2590
|
+
if (!isSafeShardName(shard)) {
|
|
2591
|
+
warnings.push('忽略不安全分片名: ' + String(shard));
|
|
2592
|
+
continue;
|
|
2593
|
+
}
|
|
2594
|
+
const full = path.join(root, shard);
|
|
2595
|
+
if (!fs.existsSync(full)) {
|
|
2596
|
+
warnings.push('分片缺失: ' + shard);
|
|
2597
|
+
continue;
|
|
2598
|
+
}
|
|
2599
|
+
try {
|
|
2600
|
+
const bytes = fs.statSync(full).size;
|
|
2601
|
+
const parsed = readIndexShard(root, shard);
|
|
2602
|
+
shardBytes += bytes;
|
|
2603
|
+
if (parsed) {
|
|
2604
|
+
entries.push.apply(entries, parsed.entries);
|
|
2605
|
+
shards.push({ name: shard, bytes: bytes, entries: parsed.entries.length });
|
|
2606
|
+
} else {
|
|
2607
|
+
warnings.push('分片结构无效: ' + shard);
|
|
2608
|
+
}
|
|
2609
|
+
} catch (error) {
|
|
2610
|
+
warnings.push('分片无法解析: ' + shard + '(' + error.message + ')');
|
|
2611
|
+
}
|
|
2612
|
+
}
|
|
2613
|
+
}
|
|
2614
|
+
const storageBytes = manifestBytes + shardBytes;
|
|
2615
|
+
const logicalBytes = jsonByteLength(entries);
|
|
2616
|
+
const fieldBytes = indexFieldBytes(entries);
|
|
2617
|
+
const tokens = indexTokenStats(entries);
|
|
2618
|
+
const gzipProjection = indexGzipProjection(entries);
|
|
2619
|
+
const warnBytes = scaleThreshold(loadConfig(), 'scale_warn_index_bytes');
|
|
2620
|
+
gzipProjection.underWarningThreshold = gzipProjection.storageBytes <= warnBytes;
|
|
2621
|
+
gzipProjection.warningThresholdBytes = warnBytes;
|
|
2622
|
+
gzipProjection.compressionRatio = storageBytes ? Math.round((gzipProjection.storageBytes / storageBytes) * 1000) / 1000 : 0;
|
|
2623
|
+
const triggerReasons = [];
|
|
2624
|
+
if (entries.length > INDEX_SHARD_THRESHOLD) triggerReasons.push('entries');
|
|
2625
|
+
if (logicalBytes > INDEX_COMPRESS_TRIGGER_BYTES) triggerReasons.push('logical_bytes');
|
|
2626
|
+
const lines = [
|
|
2627
|
+
'索引统计(只读)',
|
|
2628
|
+
'- 格式: ' + (Array.isArray(data.entries) ? 'inline' : 'sharded'),
|
|
2629
|
+
'- 条目: ' + entries.length,
|
|
2630
|
+
'- 索引存储: ' + storageBytes + ' 字节(manifest ' + manifestBytes + ' / 分片 ' + shardBytes + ')',
|
|
2631
|
+
'- 逻辑原始体积: ' + logicalBytes + ' 字节',
|
|
2632
|
+
'- token: ' + tokens.unique + ' 个唯一 token(加权 ' + tokens.weighted + ' / 拼音 ' + tokens.pinyin + ' / 平均 ' + tokens.averageUniquePerEntry + ' 每条例)',
|
|
2633
|
+
'- 字段体积: tokens ' + fieldBytes.tokens + ' / statement ' + fieldBytes.statement + ' / subject ' + fieldBytes.subject + ' / tags ' + fieldBytes.tags + ' / usage ' + fieldBytes.usage + ' / other ' + fieldBytes.other,
|
|
2634
|
+
'- gzip 年份分片预估: ' + gzipProjection.storageBytes + ' 字节(压缩比 ' + gzipProjection.compressionRatio + ' / 低于告警线 ' + (gzipProjection.underWarningThreshold ? '是' : '否') + ')',
|
|
2635
|
+
'- 触发建议: ' + (triggerReasons.length ? triggerReasons.join(' + ') : '当前无需按现有阈值分片'),
|
|
2636
|
+
];
|
|
2637
|
+
if (warnings.length) lines.push('- 告警: ' + warnings.join(';'));
|
|
2638
|
+
return {
|
|
2639
|
+
error: false,
|
|
2640
|
+
root: root,
|
|
2641
|
+
format: Array.isArray(data.entries) ? 'inline' : 'sharded',
|
|
2642
|
+
version: Number(data.version || 0),
|
|
2643
|
+
updated: data.updated || '',
|
|
2644
|
+
entryCount: entries.length,
|
|
2645
|
+
shardCount: shards.length,
|
|
2646
|
+
shards: shards,
|
|
2647
|
+
manifestBytes: manifestBytes,
|
|
2648
|
+
shardBytes: shardBytes,
|
|
2649
|
+
storageBytes: storageBytes,
|
|
2650
|
+
logicalBytes: logicalBytes,
|
|
2651
|
+
fieldBytes: fieldBytes,
|
|
2652
|
+
tokens: tokens,
|
|
2653
|
+
gzipProjection: gzipProjection,
|
|
2654
|
+
trigger: {
|
|
2655
|
+
entryThreshold: INDEX_SHARD_THRESHOLD,
|
|
2656
|
+
logicalByteThreshold: INDEX_COMPRESS_TRIGGER_BYTES,
|
|
2657
|
+
reasons: triggerReasons,
|
|
2658
|
+
},
|
|
2659
|
+
warnings: warnings,
|
|
2660
|
+
text: lines.join('\n'),
|
|
2661
|
+
};
|
|
2662
|
+
}
|
|
2424
2663
|
function collectEntryFiles(root) {
|
|
2425
2664
|
const dirs = [];
|
|
2426
2665
|
const facts = path.join(root, PUBLIC_DIR);
|
|
@@ -2534,6 +2773,318 @@ function layoutCollisionReport(root) {
|
|
|
2534
2773
|
report.ok = report.conflicts === 0 && report.unverified === 0;
|
|
2535
2774
|
return report;
|
|
2536
2775
|
}
|
|
2776
|
+
const LAYOUT_PLAN_DIR = path.join('.maintenance', 'layout-plans');
|
|
2777
|
+
const LAYOUT_JOURNAL_DIR = path.join('.maintenance', 'layout-journals');
|
|
2778
|
+
function safeLayoutRel(value) {
|
|
2779
|
+
const rel = String(value || '').replace(/\\/g, '/');
|
|
2780
|
+
if (!rel || path.isAbsolute(rel) || rel.split('/').includes('..')) return false;
|
|
2781
|
+
if (!MEMORY_FILE_RE.test(path.posix.basename(rel))) return false;
|
|
2782
|
+
return rel.indexOf(PUBLIC_DIR + '/') === 0 || rel.indexOf(PRIVATE_DIR + '/') === 0;
|
|
2783
|
+
}
|
|
2784
|
+
function layoutInsideRoot(root, rel) {
|
|
2785
|
+
const base = path.resolve(root);
|
|
2786
|
+
const full = path.resolve(root, String(rel || ''));
|
|
2787
|
+
return full === base || full.indexOf(base + path.sep) === 0;
|
|
2788
|
+
}
|
|
2789
|
+
function layoutBaseInfo(rel) {
|
|
2790
|
+
const parts = String(rel || '').split('/');
|
|
2791
|
+
if (parts[0] === PUBLIC_DIR) return { kind: 'public', baseDepth: 1, owner: '', type: 'facts' };
|
|
2792
|
+
if (parts[0] === PRIVATE_DIR && parts.length >= 3) {
|
|
2793
|
+
return { kind: 'private', baseDepth: 3, owner: parts[1], type: parts[2] };
|
|
2794
|
+
}
|
|
2795
|
+
return null;
|
|
2796
|
+
}
|
|
2797
|
+
function isLayoutMonthBucket(rel, includeFlat) {
|
|
2798
|
+
const parts = String(rel || '').split('/');
|
|
2799
|
+
const info = layoutBaseInfo(rel);
|
|
2800
|
+
if (!info) return false;
|
|
2801
|
+
if (parts.length === info.baseDepth) return !!includeFlat;
|
|
2802
|
+
if (parts.length !== info.baseDepth + 2) return false;
|
|
2803
|
+
return /^\d{4}$/.test(parts[info.baseDepth]) && /^\d{2}$/.test(parts[info.baseDepth + 1]);
|
|
2804
|
+
}
|
|
2805
|
+
function layoutTargetFor(rel) {
|
|
2806
|
+
const parts = String(rel || '').split('/');
|
|
2807
|
+
const info = layoutBaseInfo(rel);
|
|
2808
|
+
const base = path.basename(rel);
|
|
2809
|
+
const match = base.match(/^(\d{4})-(\d{2})-(\d{2})-\d{4}\.md(?:\.enc)?$/);
|
|
2810
|
+
if (!info || !match) return '';
|
|
2811
|
+
return parts.slice(0, info.baseDepth).concat([match[1], match[2], match[3], base]).join('/');
|
|
2812
|
+
}
|
|
2813
|
+
function layoutCandidateDirs(root, includeFlat, threshold) {
|
|
2814
|
+
const seeds = [];
|
|
2815
|
+
const facts = path.join(root, PUBLIC_DIR);
|
|
2816
|
+
if (fs.existsSync(facts)) seeds.push({ dir: facts, baseDepth: 1, owner: '' });
|
|
2817
|
+
const pdir = path.join(root, PRIVATE_DIR);
|
|
2818
|
+
if (fs.existsSync(pdir)) {
|
|
2819
|
+
for (const owner of fs.readdirSync(pdir)) {
|
|
2820
|
+
const ownerDir = path.join(pdir, owner);
|
|
2821
|
+
if (!fs.statSync(ownerDir).isDirectory()) continue;
|
|
2822
|
+
for (const type of PRIVATE_LEAF) {
|
|
2823
|
+
const typeDir = path.join(ownerDir, type);
|
|
2824
|
+
if (fs.existsSync(typeDir)) seeds.push({ dir: typeDir, baseDepth: 3, owner: owner });
|
|
2825
|
+
}
|
|
2826
|
+
}
|
|
2827
|
+
}
|
|
2828
|
+
const found = [];
|
|
2829
|
+
const visit = function (dir) {
|
|
2830
|
+
const rel = path.relative(root, dir).replace(/\\/g, '/');
|
|
2831
|
+
if (isLayoutMonthBucket(rel, includeFlat)) {
|
|
2832
|
+
const files = fs.readdirSync(dir, { withFileTypes: true })
|
|
2833
|
+
.filter(function (entry) { return entry.isFile() && MEMORY_FILE_RE.test(entry.name); })
|
|
2834
|
+
.map(function (entry) { return entry.name; });
|
|
2835
|
+
if (files.length > threshold) {
|
|
2836
|
+
const info = layoutBaseInfo(rel);
|
|
2837
|
+
if (info && info.kind === 'private' && isEncrypted(root) && !getOwnerKeyFor(root, info.owner)) {
|
|
2838
|
+
found.push({ path: rel, files: files.length, skipped: true, reason: 'encrypted_owner_without_key' });
|
|
2839
|
+
} else {
|
|
2840
|
+
found.push({ path: rel, files: files.length, skipped: false, reason: '' });
|
|
2841
|
+
}
|
|
2842
|
+
}
|
|
2843
|
+
}
|
|
2844
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
2845
|
+
if (entry.isDirectory()) visit(path.join(dir, entry.name));
|
|
2846
|
+
}
|
|
2847
|
+
};
|
|
2848
|
+
for (const seed of seeds) visit(seed.dir);
|
|
2849
|
+
return found.sort(function (a, b) { return a.path.localeCompare(b.path); });
|
|
2850
|
+
}
|
|
2851
|
+
function layoutBatchId(now) {
|
|
2852
|
+
return 'layout-' + now.toISOString().replace(/[:.]/g, '-') + '-' + crypto.randomBytes(3).toString('hex');
|
|
2853
|
+
}
|
|
2854
|
+
function layoutPlanCore(opts) {
|
|
2855
|
+
opts = opts || {};
|
|
2856
|
+
const root = path.resolve(opts.root || userRoot());
|
|
2857
|
+
const cfg = loadConfig();
|
|
2858
|
+
const threshold = opts.threshold === undefined ? scaleThreshold(cfg, 'scale_warn_files_per_dir') : Number(opts.threshold);
|
|
2859
|
+
const includeFlat = !!opts.includeFlat;
|
|
2860
|
+
const now = opts.now ? new Date(opts.now) : new Date();
|
|
2861
|
+
const batch = layoutBatchId(now);
|
|
2862
|
+
const candidates = layoutCandidateDirs(root, includeFlat, threshold);
|
|
2863
|
+
const operations = [];
|
|
2864
|
+
const conflicts = [];
|
|
2865
|
+
const warnings = [];
|
|
2866
|
+
for (const candidate of candidates) {
|
|
2867
|
+
if (candidate.skipped) {
|
|
2868
|
+
warnings.push('跳过 ' + candidate.path + ': 加密 owner 无可用密钥');
|
|
2869
|
+
continue;
|
|
2870
|
+
}
|
|
2871
|
+
const dir = path.join(root, candidate.path);
|
|
2872
|
+
for (const name of fs.readdirSync(dir)) {
|
|
2873
|
+
if (!MEMORY_FILE_RE.test(name)) continue;
|
|
2874
|
+
const from = path.posix.join(candidate.path, name);
|
|
2875
|
+
const to = layoutTargetFor(from);
|
|
2876
|
+
if (!to || !safeLayoutRel(to)) {
|
|
2877
|
+
conflicts.push({ from: from, to: to, reason: 'target_invalid' });
|
|
2878
|
+
continue;
|
|
2879
|
+
}
|
|
2880
|
+
if (fs.existsSync(path.join(root, to))) {
|
|
2881
|
+
conflicts.push({ from: from, to: to, reason: 'target_exists' });
|
|
2882
|
+
continue;
|
|
2883
|
+
}
|
|
2884
|
+
operations.push({
|
|
2885
|
+
from: from,
|
|
2886
|
+
to: to,
|
|
2887
|
+
sha256: normalizedLayoutDigest(fs.readFileSync(path.join(root, from))),
|
|
2888
|
+
status: 'pending',
|
|
2889
|
+
});
|
|
2890
|
+
}
|
|
2891
|
+
}
|
|
2892
|
+
operations.sort(function (a, b) { return a.from.localeCompare(b.from); });
|
|
2893
|
+
const plan = {
|
|
2894
|
+
batch: batch,
|
|
2895
|
+
createdAt: now.toISOString(),
|
|
2896
|
+
root: root,
|
|
2897
|
+
threshold: threshold,
|
|
2898
|
+
includeFlat: includeFlat,
|
|
2899
|
+
candidates: candidates,
|
|
2900
|
+
operations: operations,
|
|
2901
|
+
conflicts: conflicts,
|
|
2902
|
+
warnings: warnings,
|
|
2903
|
+
planFile: '',
|
|
2904
|
+
};
|
|
2905
|
+
if (conflicts.length) {
|
|
2906
|
+
plan.error = true;
|
|
2907
|
+
plan.text = '布局迁移计划存在 ' + conflicts.length + ' 个冲突,拒绝生成可执行计划。\n'
|
|
2908
|
+
+ conflicts.map(function (item) { return '- ' + item.from + ' -> ' + item.to + '(' + item.reason + ')'; }).join('\n');
|
|
2909
|
+
return plan;
|
|
2910
|
+
}
|
|
2911
|
+
const planFile = opts.out
|
|
2912
|
+
? path.resolve(String(opts.out))
|
|
2913
|
+
: path.join(root, LAYOUT_PLAN_DIR, batch + '.json');
|
|
2914
|
+
fs.mkdirSync(path.dirname(planFile), { recursive: true });
|
|
2915
|
+
plan.planFile = planFile;
|
|
2916
|
+
atomicWriteFileSync(planFile, JSON.stringify(plan, null, 2), 'utf8');
|
|
2917
|
+
plan.error = false;
|
|
2918
|
+
plan.text = '布局迁移计划: ' + batch + '\n- 目录: ' + candidates.length + ' 个候选'
|
|
2919
|
+
+ '(可迁移 ' + candidates.filter(function (item) { return !item.skipped; }).length + ')\n'
|
|
2920
|
+
+ '- 文件: ' + operations.length + ' 个\n- 计划: ' + planFile
|
|
2921
|
+
+ (warnings.length ? '\n- 告警: ' + warnings.join(';') : '');
|
|
2922
|
+
return plan;
|
|
2923
|
+
}
|
|
2924
|
+
function validateLayoutPlan(root, plan) {
|
|
2925
|
+
if (!plan || plan.root !== root) return '计划 root 与当前记忆库不一致,拒绝执行。';
|
|
2926
|
+
if (!/^layout-[A-Za-z0-9-]+$/.test(String(plan.batch || ''))) return '计划 batch 不安全。';
|
|
2927
|
+
if (!Array.isArray(plan.operations) || !plan.operations.length) return '计划没有可执行文件。';
|
|
2928
|
+
if (Array.isArray(plan.conflicts) && plan.conflicts.length) return '计划包含冲突,拒绝执行。';
|
|
2929
|
+
for (const op of plan.operations) {
|
|
2930
|
+
if (!safeLayoutRel(op.from) || !safeLayoutRel(op.to) || !layoutInsideRoot(root, op.from) || !layoutInsideRoot(root, op.to)) {
|
|
2931
|
+
return '计划包含不安全路径: ' + String(op.from) + ' -> ' + String(op.to);
|
|
2932
|
+
}
|
|
2933
|
+
}
|
|
2934
|
+
return '';
|
|
2935
|
+
}
|
|
2936
|
+
function writeLayoutJournal(pathname, journal) {
|
|
2937
|
+
atomicWriteFileSync(pathname, JSON.stringify(journal, null, 2), 'utf8');
|
|
2938
|
+
}
|
|
2939
|
+
function restoreLayoutDone(root, journal, pathname) {
|
|
2940
|
+
let ok = true;
|
|
2941
|
+
for (let i = journal.operations.length - 1; i >= 0; i--) {
|
|
2942
|
+
const op = journal.operations[i];
|
|
2943
|
+
if (op.status !== 'done') continue;
|
|
2944
|
+
const from = path.join(root, op.from);
|
|
2945
|
+
const to = path.join(root, op.to);
|
|
2946
|
+
if (!fs.existsSync(to) || fs.existsSync(from)) {
|
|
2947
|
+
op.status = 'rollback_conflict';
|
|
2948
|
+
ok = false;
|
|
2949
|
+
continue;
|
|
2950
|
+
}
|
|
2951
|
+
fs.mkdirSync(path.dirname(from), { recursive: true });
|
|
2952
|
+
fs.renameSync(to, from);
|
|
2953
|
+
op.status = 'rolled_back';
|
|
2954
|
+
writeLayoutJournal(pathname, journal);
|
|
2955
|
+
}
|
|
2956
|
+
return ok;
|
|
2957
|
+
}
|
|
2958
|
+
function withLayoutLock(root, fn) {
|
|
2959
|
+
const lock = path.join(root, '.maintenance', 'layout.lock');
|
|
2960
|
+
fs.mkdirSync(path.dirname(lock), { recursive: true });
|
|
2961
|
+
try {
|
|
2962
|
+
fs.writeFileSync(lock, JSON.stringify({ pid: process.pid, created: new Date().toISOString() }), { encoding: 'utf8', flag: 'wx' });
|
|
2963
|
+
} catch (error) {
|
|
2964
|
+
return { error: true, text: '已有布局迁移在运行或上次未清理 lock: ' + lock };
|
|
2965
|
+
}
|
|
2966
|
+
try {
|
|
2967
|
+
return fn();
|
|
2968
|
+
} finally {
|
|
2969
|
+
try { fs.unlinkSync(lock); } catch (error) { /* ignore */ }
|
|
2970
|
+
}
|
|
2971
|
+
}
|
|
2972
|
+
function layoutApplyCore(opts) {
|
|
2973
|
+
opts = opts || {};
|
|
2974
|
+
const root = path.resolve(opts.root || userRoot());
|
|
2975
|
+
if (!opts.yes) return { error: true, text: '拒绝执行布局迁移:请复核计划后显式加 --yes。' };
|
|
2976
|
+
if (!opts.planFile) return { error: true, text: '请用 --from-plan <文件> 指定布局迁移计划。' };
|
|
2977
|
+
const planFile = path.resolve(String(opts.planFile));
|
|
2978
|
+
let plan = null;
|
|
2979
|
+
try { plan = JSON.parse(fs.readFileSync(planFile, 'utf8')); } catch (error) {
|
|
2980
|
+
return { error: true, text: '无法读取布局迁移计划: ' + error.message };
|
|
2981
|
+
}
|
|
2982
|
+
const invalid = validateLayoutPlan(root, plan);
|
|
2983
|
+
if (invalid) return { error: true, text: invalid };
|
|
2984
|
+
return withLayoutLock(root, function () {
|
|
2985
|
+
const journalFile = path.join(root, LAYOUT_JOURNAL_DIR, plan.batch + '.json');
|
|
2986
|
+
if (fs.existsSync(journalFile)) return { error: true, text: '该计划已有 journal,拒绝重复执行: ' + journalFile };
|
|
2987
|
+
const journal = {
|
|
2988
|
+
batch: plan.batch,
|
|
2989
|
+
createdAt: plan.createdAt,
|
|
2990
|
+
startedAt: new Date().toISOString(),
|
|
2991
|
+
root: root,
|
|
2992
|
+
status: 'applying',
|
|
2993
|
+
operations: plan.operations.map(function (op) {
|
|
2994
|
+
return { from: op.from, to: op.to, sha256: op.sha256, status: 'pending' };
|
|
2995
|
+
}),
|
|
2996
|
+
};
|
|
2997
|
+
fs.mkdirSync(path.dirname(journalFile), { recursive: true });
|
|
2998
|
+
writeLayoutJournal(journalFile, journal);
|
|
2999
|
+
let moved = 0;
|
|
3000
|
+
try {
|
|
3001
|
+
for (const op of journal.operations) {
|
|
3002
|
+
const from = path.join(root, op.from);
|
|
3003
|
+
const to = path.join(root, op.to);
|
|
3004
|
+
if (!fs.existsSync(from)) throw new Error('源文件缺失: ' + op.from);
|
|
3005
|
+
const digest = normalizedLayoutDigest(fs.readFileSync(from));
|
|
3006
|
+
if (digest !== op.sha256) throw new Error('源文件哈希变化: ' + op.from);
|
|
3007
|
+
if (fs.existsSync(to)) throw new Error('目标已存在: ' + op.to);
|
|
3008
|
+
fs.mkdirSync(path.dirname(to), { recursive: true });
|
|
3009
|
+
fs.renameSync(from, to);
|
|
3010
|
+
op.status = 'done';
|
|
3011
|
+
moved += 1;
|
|
3012
|
+
writeLayoutJournal(journalFile, journal);
|
|
3013
|
+
}
|
|
3014
|
+
buildIndex(root);
|
|
3015
|
+
journal.status = 'applied';
|
|
3016
|
+
journal.completedAt = new Date().toISOString();
|
|
3017
|
+
writeLayoutJournal(journalFile, journal);
|
|
3018
|
+
return {
|
|
3019
|
+
error: false,
|
|
3020
|
+
batch: plan.batch,
|
|
3021
|
+
moved: moved,
|
|
3022
|
+
rolledBack: false,
|
|
3023
|
+
planFile: planFile,
|
|
3024
|
+
journalFile: journalFile,
|
|
3025
|
+
text: '布局迁移完成: ' + plan.batch + '(' + moved + ' 文件)\n计划: ' + planFile + '\nJournal: ' + journalFile,
|
|
3026
|
+
};
|
|
3027
|
+
} catch (error) {
|
|
3028
|
+
const rolledBack = restoreLayoutDone(root, journal, journalFile);
|
|
3029
|
+
journal.status = rolledBack ? 'rolled_back' : 'rollback_failed';
|
|
3030
|
+
journal.error = error.message;
|
|
3031
|
+
journal.completedAt = new Date().toISOString();
|
|
3032
|
+
writeLayoutJournal(journalFile, journal);
|
|
3033
|
+
try { buildIndex(root); } catch (e) { /* ignore */ }
|
|
3034
|
+
return {
|
|
3035
|
+
error: true,
|
|
3036
|
+
batch: plan.batch,
|
|
3037
|
+
moved: moved,
|
|
3038
|
+
rolledBack: rolledBack,
|
|
3039
|
+
planFile: planFile,
|
|
3040
|
+
journalFile: journalFile,
|
|
3041
|
+
text: '布局迁移失败并' + (rolledBack ? '已回滚' : '回滚不完整') + ': ' + error.message + '\nJournal: ' + journalFile,
|
|
3042
|
+
};
|
|
3043
|
+
}
|
|
3044
|
+
});
|
|
3045
|
+
}
|
|
3046
|
+
function layoutRollbackCore(opts) {
|
|
3047
|
+
opts = opts || {};
|
|
3048
|
+
const root = path.resolve(opts.root || userRoot());
|
|
3049
|
+
if (!opts.yes) return { error: true, text: '拒绝回滚布局迁移:请显式加 --yes。' };
|
|
3050
|
+
if (!/^layout-[A-Za-z0-9-]+$/.test(String(opts.batch || ''))) return { error: true, text: '请用 --batch <id> 指定安全的布局迁移批次。' };
|
|
3051
|
+
const journalFile = path.join(root, LAYOUT_JOURNAL_DIR, opts.batch + '.json');
|
|
3052
|
+
let journal = null;
|
|
3053
|
+
try { journal = JSON.parse(fs.readFileSync(journalFile, 'utf8')); } catch (error) {
|
|
3054
|
+
return { error: true, text: '无法读取布局迁移 journal: ' + error.message };
|
|
3055
|
+
}
|
|
3056
|
+
if (journal.root !== root) return { error: true, text: 'journal root 与当前记忆库不一致。' };
|
|
3057
|
+
if (journal.status === 'rolled_back') return { error: false, batch: opts.batch, rolledBack: 0, journalFile: journalFile, text: '布局迁移批次已回滚,无需重复操作。' };
|
|
3058
|
+
if (journal.status !== 'applied') return { error: true, text: '批次状态不是 applied,不能常规回滚: ' + journal.status };
|
|
3059
|
+
return withLayoutLock(root, function () {
|
|
3060
|
+
let restored = 0;
|
|
3061
|
+
try {
|
|
3062
|
+
for (let i = journal.operations.length - 1; i >= 0; i--) {
|
|
3063
|
+
const op = journal.operations[i];
|
|
3064
|
+
if (op.status !== 'done') continue;
|
|
3065
|
+
const from = path.join(root, op.from);
|
|
3066
|
+
const to = path.join(root, op.to);
|
|
3067
|
+
if (!fs.existsSync(to)) throw new Error('迁移目标缺失: ' + op.to);
|
|
3068
|
+
if (fs.existsSync(from)) throw new Error('回滚目标已存在: ' + op.from);
|
|
3069
|
+
fs.mkdirSync(path.dirname(from), { recursive: true });
|
|
3070
|
+
fs.renameSync(to, from);
|
|
3071
|
+
op.status = 'rolled_back';
|
|
3072
|
+
restored += 1;
|
|
3073
|
+
writeLayoutJournal(journalFile, journal);
|
|
3074
|
+
}
|
|
3075
|
+
buildIndex(root);
|
|
3076
|
+
journal.status = 'rolled_back';
|
|
3077
|
+
journal.rolledBackAt = new Date().toISOString();
|
|
3078
|
+
writeLayoutJournal(journalFile, journal);
|
|
3079
|
+
return { error: false, batch: opts.batch, restored: restored, journalFile: journalFile, text: '布局迁移已回滚: ' + opts.batch + '(' + restored + ' 文件)' };
|
|
3080
|
+
} catch (error) {
|
|
3081
|
+
journal.status = 'rollback_failed';
|
|
3082
|
+
journal.rollbackError = error.message;
|
|
3083
|
+
writeLayoutJournal(journalFile, journal);
|
|
3084
|
+
return { error: true, batch: opts.batch, restored: restored, journalFile: journalFile, text: '布局回滚失败: ' + error.message };
|
|
3085
|
+
}
|
|
3086
|
+
});
|
|
3087
|
+
}
|
|
2537
3088
|
// 迁移:把根下旧平铺 prefs|bounds|commits/*.md 按 frontmatter owner 迁入 private/<owner>/<type>/
|
|
2538
3089
|
function migrateLayout(root) {
|
|
2539
3090
|
let moved = 0;
|
|
@@ -2804,15 +3355,13 @@ function touchIndex(root, relFiles, options) {
|
|
|
2804
3355
|
for (const sh of manifest.shards) {
|
|
2805
3356
|
if (!isSafeShardName(sh)) continue;
|
|
2806
3357
|
if (yearSet && !yearSet.has(shardYear(sh))) continue;
|
|
2807
|
-
const
|
|
2808
|
-
if (!
|
|
2809
|
-
let sd = null; try { sd = JSON.parse(fs.readFileSync(sp, 'utf8')); } catch (e) { continue; }
|
|
2810
|
-
if (!sd || !Array.isArray(sd.entries)) continue;
|
|
3358
|
+
const sd = readIndexShard(root, sh);
|
|
3359
|
+
if (!sd) continue;
|
|
2811
3360
|
let dirty = false;
|
|
2812
3361
|
for (const e of sd.entries) {
|
|
2813
3362
|
if (set.has(e.file)) { bumpEntry(e); dirty = true; }
|
|
2814
3363
|
}
|
|
2815
|
-
if (dirty)
|
|
3364
|
+
if (dirty) writeIndexShard(root, sh, sd);
|
|
2816
3365
|
}
|
|
2817
3366
|
});
|
|
2818
3367
|
return;
|
|
@@ -3429,6 +3978,53 @@ function backupSetupCore(opts) {
|
|
|
3429
3978
|
+ (schedule && schedule.error ? '\n[警告] ' + schedule.text + ';serve 补跑仍会在启动后检查。' : ''),
|
|
3430
3979
|
};
|
|
3431
3980
|
}
|
|
3981
|
+
const BACKUP_FUTURE_TOLERANCE_MS = 15 * 60 * 1000;
|
|
3982
|
+
function backupCreatedTs(value) {
|
|
3983
|
+
const ts = Date.parse(String(value || ''));
|
|
3984
|
+
return Number.isFinite(ts) ? ts : null;
|
|
3985
|
+
}
|
|
3986
|
+
function compareBackupCreatedDesc(a, b) {
|
|
3987
|
+
const at = backupCreatedTs(a && a.created);
|
|
3988
|
+
const bt = backupCreatedTs(b && b.created);
|
|
3989
|
+
if (at !== null && bt !== null && at !== bt) return bt - at;
|
|
3990
|
+
if (at !== null && bt === null) return -1;
|
|
3991
|
+
if (at === null && bt !== null) return 1;
|
|
3992
|
+
return String((b && b.id) || '').localeCompare(String((a && a.id) || ''));
|
|
3993
|
+
}
|
|
3994
|
+
function selectLatestValidBackup(backups, nowMs) {
|
|
3995
|
+
const now = Number.isFinite(nowMs) ? nowMs : Date.now();
|
|
3996
|
+
const valid = [];
|
|
3997
|
+
const anomalies = [];
|
|
3998
|
+
for (const backup of backups || []) {
|
|
3999
|
+
if (!backup || !backup.manifest) {
|
|
4000
|
+
anomalies.push({ id: backup && backup.id || '', type: 'manifest_missing', created: backup && backup.created || '' });
|
|
4001
|
+
continue;
|
|
4002
|
+
}
|
|
4003
|
+
if (!Array.isArray(backup.manifest.files)) {
|
|
4004
|
+
anomalies.push({ id: backup.id, type: 'manifest_incomplete', created: backup.created || '' });
|
|
4005
|
+
continue;
|
|
4006
|
+
}
|
|
4007
|
+
const createdTs = backupCreatedTs(backup.created);
|
|
4008
|
+
if (createdTs === null) {
|
|
4009
|
+
anomalies.push({ id: backup.id, type: 'invalid_created', created: backup.created || '' });
|
|
4010
|
+
continue;
|
|
4011
|
+
}
|
|
4012
|
+
if (createdTs > now + BACKUP_FUTURE_TOLERANCE_MS) {
|
|
4013
|
+
anomalies.push({ id: backup.id, type: 'future_timestamp', created: backup.created || '' });
|
|
4014
|
+
continue;
|
|
4015
|
+
}
|
|
4016
|
+
valid.push(Object.assign({}, backup, { createdTs: createdTs }));
|
|
4017
|
+
}
|
|
4018
|
+
valid.sort(compareBackupCreatedDesc);
|
|
4019
|
+
return { latestValid: valid[0] || null, anomalies: anomalies };
|
|
4020
|
+
}
|
|
4021
|
+
function backupAnomalyLabel(type) {
|
|
4022
|
+
if (type === 'future_timestamp') return '未来时间';
|
|
4023
|
+
if (type === 'manifest_missing') return 'manifest 缺失';
|
|
4024
|
+
if (type === 'manifest_incomplete') return 'manifest 不完整';
|
|
4025
|
+
if (type === 'invalid_created') return 'created 无效';
|
|
4026
|
+
return type;
|
|
4027
|
+
}
|
|
3432
4028
|
function backupStatusCore(opts) {
|
|
3433
4029
|
opts = opts || {};
|
|
3434
4030
|
const root = path.resolve(opts.root || userRoot());
|
|
@@ -3450,14 +4046,20 @@ function backupStatusCore(opts) {
|
|
|
3450
4046
|
};
|
|
3451
4047
|
}
|
|
3452
4048
|
const listed = backupListCore({ dir: dir });
|
|
3453
|
-
const latest = listed.backups[0] || null;
|
|
3454
4049
|
const now = opts.now ? new Date(opts.now).getTime() : Date.now();
|
|
3455
|
-
const
|
|
3456
|
-
const
|
|
4050
|
+
const selection = selectLatestValidBackup(listed.backups, now);
|
|
4051
|
+
const latest = selection.latestValid;
|
|
4052
|
+
const ageHours = latest ? (now - latest.createdTs) / 3600000 : null;
|
|
3457
4053
|
const fresh = ageHours !== null && ageHours <= maxAge;
|
|
3458
4054
|
const same = opts.sameVolumeFn ? !!opts.sameVolumeFn(root, dir) : sameVolume(root, dir);
|
|
3459
4055
|
const independent = !same;
|
|
3460
4056
|
const healthy = !!latest && fresh && independent && !cfg.backup_last_error && !cfg.backup_scheduler_error;
|
|
4057
|
+
const configLastSuccess = cfg.backup_last_success || null;
|
|
4058
|
+
const effectiveLastSuccess = latest ? latest.created : configLastSuccess;
|
|
4059
|
+
const source = latest ? 'latest_valid' : (configLastSuccess ? 'config' : 'none');
|
|
4060
|
+
const anomalyLines = selection.anomalies.map(function (item) {
|
|
4061
|
+
return item.id + '(' + backupAnomalyLabel(item.type) + (item.created ? ' / ' + item.created : '') + ')';
|
|
4062
|
+
});
|
|
3461
4063
|
const lines = [
|
|
3462
4064
|
'备份状态: ' + (cfg.backup_enabled ? '已启用' : '已关闭'),
|
|
3463
4065
|
'- 备份目录: ' + dir,
|
|
@@ -3465,8 +4067,12 @@ function backupStatusCore(opts) {
|
|
|
3465
4067
|
'- 计划: ' + (cfg.backup_schedule || 'manual') + (cfg.backup_time ? ' ' + cfg.backup_time : ''),
|
|
3466
4068
|
'- 调度器: ' + (cfg.backup_scheduler || '(未注册)'),
|
|
3467
4069
|
'- 调度异常: ' + (cfg.backup_scheduler_error || '(无)'),
|
|
3468
|
-
'-
|
|
4070
|
+
'- 配置缓存上次成功: ' + (configLastSuccess || '(无)'),
|
|
4071
|
+
'- 有效备份: ' + (latest ? latest.id + ' / ' + latest.created : '(无)'),
|
|
4072
|
+
'- 有效上次成功: ' + (effectiveLastSuccess || '(无)'),
|
|
4073
|
+
'- 来源: ' + source,
|
|
3469
4074
|
'- 备份年龄: ' + (ageHours === null ? '(无可用备份)' : ageHours.toFixed(1) + ' 小时'),
|
|
4075
|
+
'- 异常备份: ' + (anomalyLines.length ? anomalyLines.join(';') : '(无)'),
|
|
3470
4076
|
'- 最近失败: ' + (cfg.backup_last_error || '(无)'),
|
|
3471
4077
|
'- 健康: ' + (healthy ? '正常' : '异常'),
|
|
3472
4078
|
];
|
|
@@ -3477,6 +4083,11 @@ function backupStatusCore(opts) {
|
|
|
3477
4083
|
healthy: healthy,
|
|
3478
4084
|
independent: independent,
|
|
3479
4085
|
latest: latest,
|
|
4086
|
+
latest_valid: latest,
|
|
4087
|
+
config_last_success: configLastSuccess,
|
|
4088
|
+
effective_last_success: effectiveLastSuccess,
|
|
4089
|
+
source: source,
|
|
4090
|
+
anomalies: selection.anomalies,
|
|
3480
4091
|
ageHours: ageHours,
|
|
3481
4092
|
text: lines.join('\n'),
|
|
3482
4093
|
};
|
|
@@ -3487,9 +4098,7 @@ function localDateKey(value) {
|
|
|
3487
4098
|
return d.getFullYear() + '-' + p(d.getMonth() + 1) + '-' + p(d.getDate());
|
|
3488
4099
|
}
|
|
3489
4100
|
function latestBackupByCreated(backups) {
|
|
3490
|
-
return (backups || []).slice().sort(
|
|
3491
|
-
return String(b.created || '').localeCompare(String(a.created || ''));
|
|
3492
|
-
})[0] || null;
|
|
4101
|
+
return (backups || []).slice().sort(compareBackupCreatedDesc)[0] || null;
|
|
3493
4102
|
}
|
|
3494
4103
|
function acquireBackupLock(lockPath, now, staleHours) {
|
|
3495
4104
|
const nowMs = now instanceof Date ? now.getTime() : new Date(now).getTime();
|
|
@@ -3560,7 +4169,7 @@ function backupEnsureDailyCore(opts) {
|
|
|
3560
4169
|
const listed = backupListCore({ dir: dir });
|
|
3561
4170
|
if (listed.error) throw new Error(listed.text);
|
|
3562
4171
|
const todayKey = localDateKey(now);
|
|
3563
|
-
const latest =
|
|
4172
|
+
const latest = selectLatestValidBackup(listed.backups, now.getTime()).latestValid;
|
|
3564
4173
|
if (latest && latest.created && localDateKey(latest.created) === todayKey) {
|
|
3565
4174
|
cfg.backup_last_check = nowIso;
|
|
3566
4175
|
cfg.backup_last_success = latest.created;
|
|
@@ -3609,7 +4218,14 @@ function backupHealthCore(opts) {
|
|
|
3609
4218
|
const maxAge = Number(cfg.backup_max_age_hours) || 36;
|
|
3610
4219
|
if (cfg.backup_last_error) return { level: 'warning', needsSetup: false, text: '最近一次备份失败: ' + cfg.backup_last_error };
|
|
3611
4220
|
if (cfg.backup_scheduler_error) return { level: 'warning', needsSetup: false, text: '每日备份调度注册异常: ' + cfg.backup_scheduler_error };
|
|
3612
|
-
if (!status.
|
|
4221
|
+
if (!status.latest_valid) {
|
|
4222
|
+
const anomalies = (status.anomalies || []).map(function (item) { return item.id + '(' + item.type + ')'; });
|
|
4223
|
+
return {
|
|
4224
|
+
level: 'warning',
|
|
4225
|
+
needsSetup: true,
|
|
4226
|
+
text: '自动备份已启用,但没有可用备份。' + (anomalies.length ? '检测到异常备份: ' + anomalies.join(';') + '。' : '请检查 backup 目录与调度器。'),
|
|
4227
|
+
};
|
|
4228
|
+
}
|
|
3613
4229
|
if (status.ageHours !== null && status.ageHours > maxAge) {
|
|
3614
4230
|
return { level: 'warning', needsSetup: false, text: '最近备份已过期(' + status.ageHours.toFixed(1) + ' 小时,超过 ' + maxAge + ' 小时);请检查备份盘与每日调度。' };
|
|
3615
4231
|
}
|
|
@@ -4024,6 +4640,56 @@ function scaleMetric(key, value, unit, info, warn) {
|
|
|
4024
4640
|
const level = value > warn ? 'warning' : (value > info ? 'info' : 'ok');
|
|
4025
4641
|
return { key: key, value: value, unit: unit, info: info, warn: warn, level: level };
|
|
4026
4642
|
}
|
|
4643
|
+
function typePressureReport(entries, cfg) {
|
|
4644
|
+
const archiveUtility = parseFloat(cfg && cfg.maintain_archived_utility || ARCHIVE_UTILITY_DEFAULT);
|
|
4645
|
+
const archiveAge = parseInt(cfg && cfg.maintain_archived_age || '180', 10) || 180;
|
|
4646
|
+
const report = {};
|
|
4647
|
+
for (const entry of entries) {
|
|
4648
|
+
const type = String(entry.type || 'FACT').toUpperCase();
|
|
4649
|
+
if (!report[type]) {
|
|
4650
|
+
report[type] = {
|
|
4651
|
+
total: 0,
|
|
4652
|
+
stale_180d: 0,
|
|
4653
|
+
archive_candidates: 0,
|
|
4654
|
+
growth_30d: 0,
|
|
4655
|
+
stale_ratio: 0,
|
|
4656
|
+
growth_ratio_30d: 0,
|
|
4657
|
+
pressure_level: 'ok',
|
|
4658
|
+
observation_days: 7,
|
|
4659
|
+
final_thresholds_deferred: true,
|
|
4660
|
+
warnings: [],
|
|
4661
|
+
};
|
|
4662
|
+
}
|
|
4663
|
+
const row = report[type];
|
|
4664
|
+
row.total += 1;
|
|
4665
|
+
const last = entry.last_accessed || entry.updated || entry.created || '';
|
|
4666
|
+
if (last && daysBetween(last, today()) > 180) row.stale_180d += 1;
|
|
4667
|
+
if (entry.created && daysBetween(entry.created, today()) <= 30) row.growth_30d += 1;
|
|
4668
|
+
const createdAge = entry.created ? daysBetween(entry.created, today()) : 0;
|
|
4669
|
+
if (type !== 'BOUND' && !entry.immutable && createdAge > archiveAge && utilityScore(entry) < archiveUtility) {
|
|
4670
|
+
row.archive_candidates += 1;
|
|
4671
|
+
}
|
|
4672
|
+
}
|
|
4673
|
+
for (const type of Object.keys(report)) {
|
|
4674
|
+
const row = report[type];
|
|
4675
|
+
row.stale_ratio = row.total ? Math.round((row.stale_180d / row.total) * 1000) / 1000 : 0;
|
|
4676
|
+
const prior = Math.max(0, row.total - row.growth_30d);
|
|
4677
|
+
row.growth_ratio_30d = prior ? Math.round((row.growth_30d / prior) * 1000) / 1000 : (row.growth_30d ? 1 : 0);
|
|
4678
|
+
if (row.total > 5000) {
|
|
4679
|
+
row.pressure_level = 'warning';
|
|
4680
|
+
row.warnings.push('类型压力: ' + type + ' 总量 ' + row.total + ' 超过 5000,需要分类型瘦身或归档。');
|
|
4681
|
+
} else if (row.total > 500 && row.stale_ratio >= 0.3 && row.archive_candidates > 0) {
|
|
4682
|
+
row.pressure_level = 'warning';
|
|
4683
|
+
row.warnings.push('类型压力: ' + type + ' ' + row.total + ' 条中 ' + row.stale_180d
|
|
4684
|
+
+ ' 条超过 180 天未活跃,且有 ' + row.archive_candidates + ' 条归档候选。');
|
|
4685
|
+
} else if (row.total > 500 || row.growth_ratio_30d >= 0.2) {
|
|
4686
|
+
row.pressure_level = 'info';
|
|
4687
|
+
} else {
|
|
4688
|
+
row.pressure_level = 'observe';
|
|
4689
|
+
}
|
|
4690
|
+
}
|
|
4691
|
+
return report;
|
|
4692
|
+
}
|
|
4027
4693
|
function scaleReport(root, cfg) {
|
|
4028
4694
|
const thresholds = {};
|
|
4029
4695
|
for (const key of Object.keys(SCALE_THRESHOLD_DEFAULTS)) thresholds[key] = scaleThreshold(cfg, key);
|
|
@@ -4032,6 +4698,13 @@ function scaleReport(root, cfg) {
|
|
|
4032
4698
|
const indexBytes = indexSizeBytes(root);
|
|
4033
4699
|
// v0.23.0 D3:超长 statement(> 2000 字)与单类型堆积(读索引,不逐条解密读文件)
|
|
4034
4700
|
const scaleEntries = loadIndex(root) || [];
|
|
4701
|
+
let indexData = null;
|
|
4702
|
+
try { indexData = JSON.parse(fs.readFileSync(indexPath(root), 'utf8')); } catch (e) { indexData = null; }
|
|
4703
|
+
const indexShards = indexData && Array.isArray(indexData.shards) ? indexData.shards.length : 0;
|
|
4704
|
+
const indexFormat = indexData
|
|
4705
|
+
? String(indexData.format || (Array.isArray(indexData.entries) ? 'inline' : (indexShards && indexData.shards.every(isGzipShardName) ? 'gzip-year' : 'json-year')))
|
|
4706
|
+
: 'missing';
|
|
4707
|
+
const indexLogicalBytes = jsonByteLength(scaleEntries);
|
|
4035
4708
|
let longStatements = 0;
|
|
4036
4709
|
const typeCounts = { FACT: 0, PREF: 0, BOUND: 0, COMMIT: 0 };
|
|
4037
4710
|
for (const e of scaleEntries) {
|
|
@@ -4049,6 +4722,7 @@ function scaleReport(root, cfg) {
|
|
|
4049
4722
|
for (let i = 0; i < 3; i++) samples.push(measureIndexColdStart(root));
|
|
4050
4723
|
samples.sort(function (a, b) { return a - b; });
|
|
4051
4724
|
const coldStartMs = samples[Math.floor(samples.length / 2)];
|
|
4725
|
+
const typePressure = typePressureReport(scaleEntries, cfg);
|
|
4052
4726
|
const metrics = [
|
|
4053
4727
|
scaleMetric('entries', entries, '条', thresholds.scale_info_entries, thresholds.scale_warn_entries),
|
|
4054
4728
|
scaleMetric('files_per_dir', filesPerDir, '个文件', thresholds.scale_info_files_per_dir, thresholds.scale_warn_files_per_dir),
|
|
@@ -4076,17 +4750,25 @@ function scaleReport(root, cfg) {
|
|
|
4076
4750
|
if (metrics[5].level === 'warning') {
|
|
4077
4751
|
warnings.push('规模: 单类型堆积(' + typeMaxName + ' ' + typeMax + ' 条)超过阈值 ' + thresholds.scale_warn_type_count + '(考虑归档 / 合并 / 精简;COMMIT 收工锚点建议短写)');
|
|
4078
4752
|
}
|
|
4753
|
+
for (const type of Object.keys(typePressure)) {
|
|
4754
|
+
for (const message of typePressure[type].warnings) warnings.push(message);
|
|
4755
|
+
}
|
|
4079
4756
|
// v0.18.0 A6:分级只影响 level 与提示文案;超 warn 仍进 warnings(既有行为),info 不阻断。
|
|
4080
|
-
const
|
|
4757
|
+
const pressureWarning = Object.keys(typePressure).some(function (type) { return typePressure[type].pressure_level === 'warning'; });
|
|
4758
|
+
const level = pressureWarning || metrics.some(function (m) { return m.level === 'warning'; })
|
|
4081
4759
|
? 'warning'
|
|
4082
4760
|
: (metrics.some(function (m) { return m.level === 'info'; }) ? 'info' : 'ok');
|
|
4083
4761
|
return {
|
|
4084
4762
|
entries: entries,
|
|
4085
4763
|
files_per_dir: filesPerDir,
|
|
4086
4764
|
index_bytes: indexBytes,
|
|
4765
|
+
index_logical_bytes: indexLogicalBytes,
|
|
4766
|
+
index_shards: indexShards,
|
|
4767
|
+
index_format: indexFormat,
|
|
4087
4768
|
cold_start_ms: coldStartMs,
|
|
4088
4769
|
thresholds: thresholds,
|
|
4089
4770
|
metrics: metrics,
|
|
4771
|
+
type_pressure: typePressure,
|
|
4090
4772
|
level: level,
|
|
4091
4773
|
warnings: warnings,
|
|
4092
4774
|
};
|
|
@@ -4411,7 +5093,7 @@ function isAllowedBackupPath(rel) {
|
|
|
4411
5093
|
if (!normalized) return true;
|
|
4412
5094
|
const top = normalized.split('/')[0];
|
|
4413
5095
|
if (top === 'facts' || top === 'private' || top === '.archive') return true;
|
|
4414
|
-
if (top === 'agents.json' || top === 'index.json' || /^index-\d{4}\.json
|
|
5096
|
+
if (top === 'agents.json' || top === 'index.json' || /^index-\d{4}\.json(?:\.gz)?$/.test(top)) return true;
|
|
4415
5097
|
if (top === 'keys') {
|
|
4416
5098
|
return normalized !== 'keys/cache' && normalized.indexOf('keys/cache/') !== 0 &&
|
|
4417
5099
|
normalized !== 'keys/pending' && normalized.indexOf('keys/pending/') !== 0;
|
|
@@ -4475,7 +5157,7 @@ function backupListCore(opts) {
|
|
|
4475
5157
|
manifest: manifest,
|
|
4476
5158
|
});
|
|
4477
5159
|
}
|
|
4478
|
-
backups.sort(
|
|
5160
|
+
backups.sort(compareBackupCreatedDesc);
|
|
4479
5161
|
const lines = backups.length
|
|
4480
5162
|
? backups.map(function (b) { return '- ' + b.id + ' ' + (b.created || '?') + ' ' + (b.files === null ? 'manifest 缺失' : b.files + ' 文件'); })
|
|
4481
5163
|
: ['(无备份)'];
|
|
@@ -4485,7 +5167,7 @@ function backupDoctorCore(opts) {
|
|
|
4485
5167
|
opts = opts || {};
|
|
4486
5168
|
const listed = backupListCore(opts);
|
|
4487
5169
|
if (listed.error) return { error: true, ok: false, text: listed.text };
|
|
4488
|
-
const id = opts.id || (listed.backups
|
|
5170
|
+
const id = opts.id || (selectLatestValidBackup(listed.backups, Date.now()).latestValid || {}).id;
|
|
4489
5171
|
if (!id) return { error: false, ok: false, text: '没有可检查的备份。' };
|
|
4490
5172
|
const backup = listed.backups.find(function (item) { return item.id === id; });
|
|
4491
5173
|
if (!backup) return { error: true, ok: false, text: '未找到备份: ' + id };
|
|
@@ -7849,6 +8531,29 @@ function cmdBackupDoctor(opts) {
|
|
|
7849
8531
|
console.log(r.text);
|
|
7850
8532
|
if (r.error || !r.ok) process.exit(2);
|
|
7851
8533
|
}
|
|
8534
|
+
function cmdIndex(opts) {
|
|
8535
|
+
if (!opts.stats) {
|
|
8536
|
+
console.error('index 只提供只读统计:请使用 yotta-memory index --stats [--json]。');
|
|
8537
|
+
process.exit(2);
|
|
8538
|
+
}
|
|
8539
|
+
const r = indexStatsCore(opts);
|
|
8540
|
+
if (opts.json) console.log(JSON.stringify(r, null, 2));
|
|
8541
|
+
else console.log(r.text);
|
|
8542
|
+
if (r.error) process.exit(2);
|
|
8543
|
+
}
|
|
8544
|
+
function cmdLayout(action, opts) {
|
|
8545
|
+
let r;
|
|
8546
|
+
if (action === 'plan') r = layoutPlanCore(opts);
|
|
8547
|
+
else if (action === 'apply') r = layoutApplyCore(Object.assign({}, opts, { planFile: opts.fromPlan }));
|
|
8548
|
+
else if (action === 'rollback') r = layoutRollbackCore(opts);
|
|
8549
|
+
else {
|
|
8550
|
+
console.error('layout 子命令: plan [--threshold N] [--include-flat] [--out <文件>] / apply --from-plan <文件> --yes / rollback --batch <id> --yes');
|
|
8551
|
+
process.exit(2);
|
|
8552
|
+
}
|
|
8553
|
+
if (opts.json) console.log(JSON.stringify(r, null, 2));
|
|
8554
|
+
else console.log(r.text);
|
|
8555
|
+
if (r.error) process.exit(2);
|
|
8556
|
+
}
|
|
7852
8557
|
function cmdBackupRestore(id, opts) {
|
|
7853
8558
|
const r = backupRestoreCore(id, opts);
|
|
7854
8559
|
console.log(r.text);
|
|
@@ -9915,6 +10620,7 @@ const SCAN_SEVERITIES = ['safe', 'low', 'medium', 'high', 'critical'];
|
|
|
9915
10620
|
const SCAN_MAX_FINDINGS = 500;
|
|
9916
10621
|
const SCAN_MAX_FILES = 20000;
|
|
9917
10622
|
const SCAN_QUARANTINE_DIR = '.memory-scan';
|
|
10623
|
+
const SCAN_POLICY_FILE = 'security/scan-policy.json';
|
|
9918
10624
|
const MEMORY_SCAN_RULES = [
|
|
9919
10625
|
// ── 恶意指令(元安 DownloadExec / 元盾危险命令)──
|
|
9920
10626
|
{ id: 'YTM-MAL-001', class: 'malicious-instruction', severity: 'critical', ref: 'DEX-001@yotta-security-audit', title: 'curl 下载内容交给 shell 执行', mask: false, re: /\bcurl\b[^\n|;]{0,120}\|\s*(?:ba)?sh\b/i },
|
|
@@ -9966,7 +10672,77 @@ function scanEmptySummary() {
|
|
|
9966
10672
|
for (const cls of SCAN_CLASSES) byClass[cls] = 0;
|
|
9967
10673
|
const bySeverity = {};
|
|
9968
10674
|
for (const sev of SCAN_SEVERITIES) bySeverity[sev] = 0;
|
|
9969
|
-
return {
|
|
10675
|
+
return {
|
|
10676
|
+
files: 0,
|
|
10677
|
+
encrypted: 0,
|
|
10678
|
+
mislabelled: 0,
|
|
10679
|
+
scanned: 0,
|
|
10680
|
+
hits: 0,
|
|
10681
|
+
suppressed: 0,
|
|
10682
|
+
invalid_policy: 0,
|
|
10683
|
+
expired_policy: 0,
|
|
10684
|
+
byClass: byClass,
|
|
10685
|
+
bySeverity: bySeverity,
|
|
10686
|
+
maxSeverity: 'safe',
|
|
10687
|
+
truncated: false,
|
|
10688
|
+
};
|
|
10689
|
+
}
|
|
10690
|
+
function scanLineHash(line) {
|
|
10691
|
+
return crypto.createHash('sha256').update(String(line || ''), 'utf8').digest('hex');
|
|
10692
|
+
}
|
|
10693
|
+
function loadScanPolicy(root, noPolicy) {
|
|
10694
|
+
const pathname = path.join(root, SCAN_POLICY_FILE);
|
|
10695
|
+
if (noPolicy) {
|
|
10696
|
+
return { path: pathname, entries: [], map: new Map(), invalid: 0, expired: 0, used: new Set(), warnings: [] };
|
|
10697
|
+
}
|
|
10698
|
+
if (!fs.existsSync(pathname)) {
|
|
10699
|
+
return { path: pathname, entries: [], map: new Map(), invalid: 0, expired: 0, used: new Set(), warnings: [] };
|
|
10700
|
+
}
|
|
10701
|
+
let data = null;
|
|
10702
|
+
try { data = JSON.parse(fs.readFileSync(pathname, 'utf8')); } catch (error) {
|
|
10703
|
+
return { path: pathname, entries: [], map: new Map(), invalid: 1, expired: 0, used: new Set(), warnings: ['扫描策略 JSON 无法解析: ' + error.message] };
|
|
10704
|
+
}
|
|
10705
|
+
const rawEntries = data && Array.isArray(data.entries) ? data.entries : [];
|
|
10706
|
+
const entries = [];
|
|
10707
|
+
const map = new Map();
|
|
10708
|
+
let invalid = data && Array.isArray(data.entries) ? 0 : 1;
|
|
10709
|
+
let expired = 0;
|
|
10710
|
+
const warnings = [];
|
|
10711
|
+
if (!data || !Array.isArray(data.entries)) warnings.push('扫描策略结构无效:缺少 entries 数组。');
|
|
10712
|
+
for (const raw of rawEntries) {
|
|
10713
|
+
const rule = String(raw && raw.rule || '').trim();
|
|
10714
|
+
const file = String(raw && raw.file || '').replace(/\\/g, '/');
|
|
10715
|
+
const hash = String(raw && raw.line_sha256 || '').trim().toLowerCase();
|
|
10716
|
+
const reviewBy = String(raw && raw.review_by || '').trim();
|
|
10717
|
+
const reason = String(raw && raw.reason || '').trim();
|
|
10718
|
+
const valid = /^YTM-[A-Z]+-\d{3}$/.test(rule) &&
|
|
10719
|
+
isSafeRelPath(file) && file.indexOf('*') === -1 && MEMORY_FILE_RE.test(file) &&
|
|
10720
|
+
/^[a-f0-9]{64}$/.test(hash) &&
|
|
10721
|
+
/^\d{4}-\d{2}-\d{2}$/.test(reviewBy) && !isNaN(new Date(reviewBy).getTime()) &&
|
|
10722
|
+
reason.length > 0;
|
|
10723
|
+
if (!valid) {
|
|
10724
|
+
invalid++;
|
|
10725
|
+
continue;
|
|
10726
|
+
}
|
|
10727
|
+
if (reviewBy < today()) {
|
|
10728
|
+
expired++;
|
|
10729
|
+
continue;
|
|
10730
|
+
}
|
|
10731
|
+
const entry = {
|
|
10732
|
+
rule: rule,
|
|
10733
|
+
file: file,
|
|
10734
|
+
line_sha256: hash,
|
|
10735
|
+
reason: reason,
|
|
10736
|
+
review_by: reviewBy,
|
|
10737
|
+
added_by: String(raw.added_by || ''),
|
|
10738
|
+
added_at: String(raw.added_at || ''),
|
|
10739
|
+
};
|
|
10740
|
+
entries.push(entry);
|
|
10741
|
+
map.set(rule + '\0' + file + '\0' + hash, entry);
|
|
10742
|
+
}
|
|
10743
|
+
if (invalid) warnings.push('扫描策略有 ' + invalid + ' 条无效条目,已 fail-closed 忽略。');
|
|
10744
|
+
if (expired) warnings.push('扫描策略有 ' + expired + ' 条已过期,已恢复为命中。');
|
|
10745
|
+
return { path: pathname, entries: entries, map: map, invalid: invalid, expired: expired, used: new Set(), warnings: warnings };
|
|
9970
10746
|
}
|
|
9971
10747
|
function isSafeRelPath(rel) {
|
|
9972
10748
|
const value = String(rel || '');
|
|
@@ -10108,10 +10884,16 @@ function renderScanText(report) {
|
|
|
10108
10884
|
'',
|
|
10109
10885
|
'- 扫描路径: ' + report.root,
|
|
10110
10886
|
'- 扫描文件: ' + report.summary.files + '(加密跳过 ' + report.summary.encrypted + (report.summary.mislabelled ? ';.enc 后缀但内容是明文 ' + report.summary.mislabelled + ' 个已按文本扫描' : '') + ')',
|
|
10111
|
-
'- 结果: ' + label[report.level] + ' · 命中 ' + report.summary.hits + ' 条'
|
|
10887
|
+
'- 结果: ' + label[report.level] + ' · 命中 ' + report.summary.hits + ' 条'
|
|
10888
|
+
+ (report.summary.suppressed ? ' · 已抑制 ' + report.summary.suppressed + ' 条' : '')
|
|
10889
|
+
+ (report.summary.truncated ? '(已达上限,结果被截断)' : ''),
|
|
10112
10890
|
'- 分级: ' + SCAN_SEVERITIES.map(function (s) { return s + '=' + report.summary.bySeverity[s]; }).join(' / '),
|
|
10113
10891
|
'- 类别: ' + SCAN_CLASSES.map(function (c) { return c + '=' + report.summary.byClass[c]; }).join(' / '),
|
|
10114
10892
|
];
|
|
10893
|
+
if (report.policy && (report.policy.invalid || report.policy.expired || report.policy.unused)) {
|
|
10894
|
+
lines.push('- 策略: 无效 ' + report.policy.invalid + ' / 过期 ' + report.policy.expired + ' / 未命中 ' + report.policy.unused);
|
|
10895
|
+
}
|
|
10896
|
+
for (const warning of (report.policy && report.policy.warnings) || []) lines.push('- [策略警告] ' + warning);
|
|
10115
10897
|
if (report.gate) lines.push('- 门禁: 命中 ' + report.gate + ' 及以上即 exit 1(当前 ' + report.level + ')');
|
|
10116
10898
|
if (!report.findings.length) lines.push('- 未发现可疑内容。');
|
|
10117
10899
|
for (const finding of report.findings) {
|
|
@@ -10149,6 +10931,8 @@ function scanCore(opts) {
|
|
|
10149
10931
|
if (opts.restore) return scanRestoreCore(root, opts);
|
|
10150
10932
|
|
|
10151
10933
|
const findings = [];
|
|
10934
|
+
const suppressed = [];
|
|
10935
|
+
const policy = loadScanPolicy(root, opts.noPolicy);
|
|
10152
10936
|
const byClass = {};
|
|
10153
10937
|
for (const cls of SCAN_CLASSES) byClass[cls] = 0;
|
|
10154
10938
|
const bySeverity = {};
|
|
@@ -10182,6 +10966,25 @@ function scanCore(opts) {
|
|
|
10182
10966
|
const start = Math.max(0, match.index - 40);
|
|
10183
10967
|
const end = Math.min(line.length, match.index + match[0].length + 40);
|
|
10184
10968
|
const snippet = scrubSecrets((start > 0 ? '…' : '') + line.slice(start, end) + (end < line.length ? '…' : '')).slice(0, 200);
|
|
10969
|
+
const lineHash = scanLineHash(line);
|
|
10970
|
+
const policyKey = rule.id + '\0' + rel + '\0' + lineHash;
|
|
10971
|
+
if (policy.map.has(policyKey)) {
|
|
10972
|
+
const exception = policy.map.get(policyKey);
|
|
10973
|
+
policy.used.add(policyKey);
|
|
10974
|
+
suppressed.push({
|
|
10975
|
+
file: rel,
|
|
10976
|
+
line: i + 1,
|
|
10977
|
+
evidence: rel + ':' + (i + 1),
|
|
10978
|
+
rule: rule.id,
|
|
10979
|
+
class: rule.class,
|
|
10980
|
+
severity: rule.severity,
|
|
10981
|
+
title: rule.title,
|
|
10982
|
+
line_sha256: lineHash,
|
|
10983
|
+
reason: exception.reason,
|
|
10984
|
+
review_by: exception.review_by,
|
|
10985
|
+
});
|
|
10986
|
+
continue;
|
|
10987
|
+
}
|
|
10185
10988
|
findings.push({
|
|
10186
10989
|
file: rel,
|
|
10187
10990
|
line: i + 1,
|
|
@@ -10191,6 +10994,7 @@ function scanCore(opts) {
|
|
|
10191
10994
|
severity: rule.severity,
|
|
10192
10995
|
title: rule.title,
|
|
10193
10996
|
ref: rule.ref,
|
|
10997
|
+
line_sha256: lineHash,
|
|
10194
10998
|
match: scrubSecrets(match[0]).slice(0, 80),
|
|
10195
10999
|
snippet: snippet,
|
|
10196
11000
|
});
|
|
@@ -10213,6 +11017,9 @@ function scanCore(opts) {
|
|
|
10213
11017
|
bySeverity: bySeverity,
|
|
10214
11018
|
maxSeverity: level,
|
|
10215
11019
|
truncated: truncated,
|
|
11020
|
+
suppressed: suppressed.length,
|
|
11021
|
+
invalid_policy: policy.invalid,
|
|
11022
|
+
expired_policy: policy.expired,
|
|
10216
11023
|
};
|
|
10217
11024
|
const report = {
|
|
10218
11025
|
schemaVersion: SCAN_SCHEMA_VERSION,
|
|
@@ -10223,7 +11030,16 @@ function scanCore(opts) {
|
|
|
10223
11030
|
root: root,
|
|
10224
11031
|
gate: gate,
|
|
10225
11032
|
findings: findings,
|
|
11033
|
+
suppressed: suppressed,
|
|
10226
11034
|
summary: summary,
|
|
11035
|
+
policy: {
|
|
11036
|
+
path: policy.path,
|
|
11037
|
+
entries: policy.entries.length,
|
|
11038
|
+
invalid: policy.invalid,
|
|
11039
|
+
expired: policy.expired,
|
|
11040
|
+
unused: Math.max(0, policy.entries.length - policy.used.size),
|
|
11041
|
+
warnings: policy.warnings,
|
|
11042
|
+
},
|
|
10227
11043
|
quarantine: null,
|
|
10228
11044
|
restore: null,
|
|
10229
11045
|
};
|
|
@@ -10252,6 +11068,64 @@ function scanCore(opts) {
|
|
|
10252
11068
|
}
|
|
10253
11069
|
return report;
|
|
10254
11070
|
}
|
|
11071
|
+
function scanPolicyCandidatesCore(opts) {
|
|
11072
|
+
opts = opts || {};
|
|
11073
|
+
const root = path.resolve(opts.path || opts.root || userRoot());
|
|
11074
|
+
const scan = scanCore({
|
|
11075
|
+
root: root,
|
|
11076
|
+
noPolicy: true,
|
|
11077
|
+
quarantine: false,
|
|
11078
|
+
restore: false,
|
|
11079
|
+
gate: '',
|
|
11080
|
+
});
|
|
11081
|
+
if (scan.error) return scan;
|
|
11082
|
+
const seen = new Set();
|
|
11083
|
+
const candidates = [];
|
|
11084
|
+
for (const finding of scan.findings) {
|
|
11085
|
+
const key = finding.rule + '\0' + finding.file + '\0' + finding.line_sha256;
|
|
11086
|
+
if (seen.has(key)) continue;
|
|
11087
|
+
seen.add(key);
|
|
11088
|
+
candidates.push({
|
|
11089
|
+
rule: finding.rule,
|
|
11090
|
+
file: finding.file,
|
|
11091
|
+
line: finding.line,
|
|
11092
|
+
line_sha256: finding.line_sha256,
|
|
11093
|
+
class: finding.class,
|
|
11094
|
+
severity: finding.severity,
|
|
11095
|
+
title: finding.title,
|
|
11096
|
+
reason: '',
|
|
11097
|
+
review_by: '',
|
|
11098
|
+
added_by: '',
|
|
11099
|
+
added_at: '',
|
|
11100
|
+
});
|
|
11101
|
+
}
|
|
11102
|
+
let outPath = '';
|
|
11103
|
+
if (opts.out) {
|
|
11104
|
+
const rel = String(opts.out).replace(/\\/g, '/');
|
|
11105
|
+
if (!layoutInsideRoot(root, rel)) {
|
|
11106
|
+
return {
|
|
11107
|
+
schemaVersion: SCAN_SCHEMA_VERSION,
|
|
11108
|
+
error: true,
|
|
11109
|
+
exitCode: 2,
|
|
11110
|
+
candidates: [],
|
|
11111
|
+
text: '拒绝写出策略候选:--out 必须位于当前记忆库内。',
|
|
11112
|
+
};
|
|
11113
|
+
}
|
|
11114
|
+
outPath = path.resolve(root, rel);
|
|
11115
|
+
fs.mkdirSync(path.dirname(outPath), { recursive: true });
|
|
11116
|
+
atomicWriteFileSync(outPath, JSON.stringify({ version: 1, candidates: candidates }, null, 2), 'utf8');
|
|
11117
|
+
}
|
|
11118
|
+
return {
|
|
11119
|
+
schemaVersion: SCAN_SCHEMA_VERSION,
|
|
11120
|
+
error: false,
|
|
11121
|
+
exitCode: 0,
|
|
11122
|
+
root: root,
|
|
11123
|
+
candidates: candidates,
|
|
11124
|
+
out: outPath,
|
|
11125
|
+
text: '扫描策略候选: ' + candidates.length + ' 条'
|
|
11126
|
+
+ (outPath ? '\n已写出: ' + outPath : '\n未写文件;请人工复核后配置 security/scan-policy.json。'),
|
|
11127
|
+
};
|
|
11128
|
+
}
|
|
10255
11129
|
// 交互确认:非 TTY 一律返回 null(由调用方给 --yes 指引),避免自动化里静默改写记忆。
|
|
10256
11130
|
function promptYesNo(promptText) {
|
|
10257
11131
|
return new Promise(function (resolve) {
|
|
@@ -10283,6 +11157,12 @@ function promptYesNo(promptText) {
|
|
|
10283
11157
|
}
|
|
10284
11158
|
async function cmdScan(opts) {
|
|
10285
11159
|
opts = opts || {};
|
|
11160
|
+
if (opts.policyCandidates) {
|
|
11161
|
+
const candidates = scanPolicyCandidatesCore(opts);
|
|
11162
|
+
console.log(opts.json ? JSON.stringify(candidates, null, 2) : candidates.text);
|
|
11163
|
+
if (candidates.error) process.exit(candidates.exitCode || 2);
|
|
11164
|
+
return;
|
|
11165
|
+
}
|
|
10286
11166
|
if (opts.quarantine && !opts.yes) {
|
|
10287
11167
|
const preview = scanCore(Object.assign({}, opts, { quarantine: false }));
|
|
10288
11168
|
if (preview.error) {
|
|
@@ -12327,6 +13207,10 @@ async function main() {
|
|
|
12327
13207
|
else if (a === '--rules') opts.rules = true;
|
|
12328
13208
|
else if (a === '--propose') opts.propose = true;
|
|
12329
13209
|
else if (a === '--audit') opts.audit = true;
|
|
13210
|
+
else if (a === '--stats') opts.stats = true;
|
|
13211
|
+
else if (a === '--include-flat') opts.includeFlat = true;
|
|
13212
|
+
else if (a === '--no-policy') opts.noPolicy = true;
|
|
13213
|
+
else if (a === '--policy-candidates') opts.policyCandidates = true;
|
|
12330
13214
|
else if (valueOpts.has(a)) {
|
|
12331
13215
|
const v = args[++i];
|
|
12332
13216
|
if (a === '--type') opts.type = v;
|
|
@@ -12382,6 +13266,8 @@ async function main() {
|
|
|
12382
13266
|
else if (a === '--template') opts.template = v;
|
|
12383
13267
|
else if (a === '--path') opts.path = v;
|
|
12384
13268
|
else if (a === '--from') opts.from = v;
|
|
13269
|
+
else if (a === '--from-plan') opts.fromPlan = v;
|
|
13270
|
+
else if (a === '--batch') opts.batch = v;
|
|
12385
13271
|
} else if (a.startsWith('--')) {
|
|
12386
13272
|
if (a === '--query') {
|
|
12387
13273
|
console.error('未知选项: --query。recall/search 的关键词是位置参数:yotta-memory recall [关键词]');
|
|
@@ -12458,6 +13344,8 @@ async function main() {
|
|
|
12458
13344
|
break;
|
|
12459
13345
|
}
|
|
12460
13346
|
case 'reindex': cmdReindex(); break;
|
|
13347
|
+
case 'index': cmdIndex(opts); break;
|
|
13348
|
+
case 'layout': cmdLayout(rest[0], opts); break;
|
|
12461
13349
|
case 'doctor': cmdDoctor(opts); break;
|
|
12462
13350
|
case 'profile': cmdProfile(opts); break;
|
|
12463
13351
|
case 'context': cmdContext(opts); break;
|
|
@@ -12565,6 +13453,9 @@ module.exports = {
|
|
|
12565
13453
|
viewEntriesCore: viewEntriesCore,
|
|
12566
13454
|
viewOverviewCore: viewOverviewCore,
|
|
12567
13455
|
migrateLayout: migrateLayout,
|
|
13456
|
+
layoutPlanCore: layoutPlanCore,
|
|
13457
|
+
layoutApplyCore: layoutApplyCore,
|
|
13458
|
+
layoutRollbackCore: layoutRollbackCore,
|
|
12568
13459
|
loadIndexFor: loadIndexFor,
|
|
12569
13460
|
indexEntriesFor: indexEntriesFor,
|
|
12570
13461
|
loadOwnerIndex: loadOwnerIndex,
|
|
@@ -12575,6 +13466,10 @@ module.exports = {
|
|
|
12575
13466
|
baselineCore: baselineCore,
|
|
12576
13467
|
loadBaselineTemplate: loadBaselineTemplate,
|
|
12577
13468
|
scanCore: scanCore,
|
|
13469
|
+
scanPolicyCandidatesCore: scanPolicyCandidatesCore,
|
|
13470
|
+
loadScanPolicy: loadScanPolicy,
|
|
13471
|
+
scanLineHash: scanLineHash,
|
|
13472
|
+
SCAN_POLICY_FILE: SCAN_POLICY_FILE,
|
|
12578
13473
|
scanQuarantineCore: scanQuarantineCore,
|
|
12579
13474
|
scanRestoreCore: scanRestoreCore,
|
|
12580
13475
|
promptYesNo: promptYesNo,
|
|
@@ -12702,6 +13597,10 @@ module.exports = {
|
|
|
12702
13597
|
|
|
12703
13598
|
loadIndex: loadIndex,
|
|
12704
13599
|
saveIndex: saveIndex,
|
|
13600
|
+
indexStatsCore: indexStatsCore,
|
|
13601
|
+
INDEX_VERSION: INDEX_VERSION,
|
|
13602
|
+
INDEX_SHARD_THRESHOLD: INDEX_SHARD_THRESHOLD,
|
|
13603
|
+
INDEX_COMPRESS_TRIGGER_BYTES: INDEX_COMPRESS_TRIGGER_BYTES,
|
|
12705
13604
|
buildIndex: buildIndex,
|
|
12706
13605
|
getIndex: getIndex,
|
|
12707
13606
|
touchIndex: touchIndex,
|
|
@@ -12736,6 +13635,8 @@ module.exports = {
|
|
|
12736
13635
|
listBackupVolumesCore: listBackupVolumesCore,
|
|
12737
13636
|
backupSetupCore: backupSetupCore,
|
|
12738
13637
|
backupStatusCore: backupStatusCore,
|
|
13638
|
+
selectLatestValidBackup: selectLatestValidBackup,
|
|
13639
|
+
BACKUP_FUTURE_TOLERANCE_MS: BACKUP_FUTURE_TOLERANCE_MS,
|
|
12739
13640
|
backupEnsureDailyCore: backupEnsureDailyCore,
|
|
12740
13641
|
backupHealthCore: backupHealthCore,
|
|
12741
13642
|
doctorCore: doctorCore,
|