@joshuaswarren/openclaw-engram 8.3.32 → 8.3.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +517 -22
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -16350,8 +16350,8 @@ mistakes: ${res.mistakesCount} patterns`
|
|
|
16350
16350
|
}
|
|
16351
16351
|
|
|
16352
16352
|
// src/cli.ts
|
|
16353
|
-
import
|
|
16354
|
-
import { access as access2, readFile as
|
|
16353
|
+
import path38 from "path";
|
|
16354
|
+
import { access as access2, readFile as readFile27, readdir as readdir16, unlink as unlink5 } from "fs/promises";
|
|
16355
16355
|
|
|
16356
16356
|
// src/transfer/export-json.ts
|
|
16357
16357
|
import path26 from "path";
|
|
@@ -17230,8 +17230,8 @@ function gatherCandidates(input, warnings) {
|
|
|
17230
17230
|
const record = rec;
|
|
17231
17231
|
const content = typeof record.content === "string" ? record.content : null;
|
|
17232
17232
|
if (!content) continue;
|
|
17233
|
-
const
|
|
17234
|
-
if (!
|
|
17233
|
+
const path40 = typeof record.path === "string" ? record.path : "";
|
|
17234
|
+
if (!path40.startsWith("transcripts/") && !path40.includes("/transcripts/")) continue;
|
|
17235
17235
|
rows.push(...parseJsonl(content, warnings));
|
|
17236
17236
|
}
|
|
17237
17237
|
return rows;
|
|
@@ -17286,6 +17286,419 @@ var openclawReplayNormalizer = {
|
|
|
17286
17286
|
}
|
|
17287
17287
|
};
|
|
17288
17288
|
|
|
17289
|
+
// src/maintenance/archive-observations.ts
|
|
17290
|
+
import path34 from "path";
|
|
17291
|
+
import { mkdir as mkdir25, readdir as readdir13, readFile as readFile23, unlink as unlink4, writeFile as writeFile22 } from "fs/promises";
|
|
17292
|
+
var DATE_FILE_PATTERN = /^(\d{4})-(\d{2})-(\d{2})\.(jsonl|md)$/;
|
|
17293
|
+
function normalizeRetentionDays(value) {
|
|
17294
|
+
if (!Number.isFinite(value)) return 30;
|
|
17295
|
+
return Math.max(0, Math.floor(value));
|
|
17296
|
+
}
|
|
17297
|
+
function extractDateFromFilename(name) {
|
|
17298
|
+
const match = DATE_FILE_PATTERN.exec(name);
|
|
17299
|
+
if (!match) return null;
|
|
17300
|
+
const iso = `${match[1]}-${match[2]}-${match[3]}T00:00:00.000Z`;
|
|
17301
|
+
const parsed = new Date(iso);
|
|
17302
|
+
if (Number.isNaN(parsed.getTime())) return null;
|
|
17303
|
+
return parsed;
|
|
17304
|
+
}
|
|
17305
|
+
function startOfUtcDay(value) {
|
|
17306
|
+
return Date.UTC(value.getUTCFullYear(), value.getUTCMonth(), value.getUTCDate());
|
|
17307
|
+
}
|
|
17308
|
+
async function listFilesRecursive2(root, relPrefix = "") {
|
|
17309
|
+
const out = [];
|
|
17310
|
+
let entries;
|
|
17311
|
+
try {
|
|
17312
|
+
entries = await readdir13(root, { withFileTypes: true });
|
|
17313
|
+
} catch {
|
|
17314
|
+
return out;
|
|
17315
|
+
}
|
|
17316
|
+
for (const entry of entries) {
|
|
17317
|
+
const rel = relPrefix ? path34.join(relPrefix, entry.name) : entry.name;
|
|
17318
|
+
const full = path34.join(root, entry.name);
|
|
17319
|
+
if (entry.isDirectory()) {
|
|
17320
|
+
out.push(...await listFilesRecursive2(full, rel));
|
|
17321
|
+
continue;
|
|
17322
|
+
}
|
|
17323
|
+
if (entry.isFile()) out.push(rel);
|
|
17324
|
+
}
|
|
17325
|
+
return out;
|
|
17326
|
+
}
|
|
17327
|
+
async function collectArchiveCandidates(memoryDir, cutoffTimeMs) {
|
|
17328
|
+
const roots = ["transcripts", path34.join("state", "tool-usage"), path34.join("summaries", "hourly")];
|
|
17329
|
+
const out = [];
|
|
17330
|
+
for (const relRoot of roots) {
|
|
17331
|
+
const absRoot = path34.join(memoryDir, relRoot);
|
|
17332
|
+
const files = await listFilesRecursive2(absRoot);
|
|
17333
|
+
for (const fileRel of files) {
|
|
17334
|
+
const filename = path34.basename(fileRel);
|
|
17335
|
+
const parsedDate = extractDateFromFilename(filename);
|
|
17336
|
+
if (!parsedDate) continue;
|
|
17337
|
+
if (parsedDate.getTime() >= cutoffTimeMs) continue;
|
|
17338
|
+
out.push({
|
|
17339
|
+
absolutePath: path34.join(absRoot, fileRel),
|
|
17340
|
+
relativePath: path34.join(relRoot, fileRel)
|
|
17341
|
+
});
|
|
17342
|
+
}
|
|
17343
|
+
}
|
|
17344
|
+
out.sort((a, b) => a.relativePath.localeCompare(b.relativePath));
|
|
17345
|
+
return out;
|
|
17346
|
+
}
|
|
17347
|
+
async function archiveObservations(options) {
|
|
17348
|
+
const retentionDays = normalizeRetentionDays(options.retentionDays);
|
|
17349
|
+
const dryRun = options.dryRun !== false;
|
|
17350
|
+
const now = options.now ?? /* @__PURE__ */ new Date();
|
|
17351
|
+
const cutoffDayStartUtc = startOfUtcDay(
|
|
17352
|
+
new Date(now.getTime() - retentionDays * 24 * 60 * 60 * 1e3)
|
|
17353
|
+
);
|
|
17354
|
+
const stamp = now.toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z");
|
|
17355
|
+
const archiveRoot = path34.join(options.memoryDir, "archive", "observations", stamp);
|
|
17356
|
+
const candidates = retentionDays === 0 ? [] : await collectArchiveCandidates(
|
|
17357
|
+
options.memoryDir,
|
|
17358
|
+
cutoffDayStartUtc
|
|
17359
|
+
);
|
|
17360
|
+
let archivedFiles = 0;
|
|
17361
|
+
let archivedBytes = 0;
|
|
17362
|
+
const archivedRelativePaths = [];
|
|
17363
|
+
if (!dryRun && candidates.length > 0) {
|
|
17364
|
+
await mkdir25(archiveRoot, { recursive: true });
|
|
17365
|
+
for (const candidate of candidates) {
|
|
17366
|
+
const archivePath = path34.join(archiveRoot, candidate.relativePath);
|
|
17367
|
+
const archiveDir = path34.dirname(archivePath);
|
|
17368
|
+
await mkdir25(archiveDir, { recursive: true });
|
|
17369
|
+
const raw = await readFile23(candidate.absolutePath);
|
|
17370
|
+
await writeFile22(archivePath, raw);
|
|
17371
|
+
await unlink4(candidate.absolutePath);
|
|
17372
|
+
archivedFiles += 1;
|
|
17373
|
+
archivedBytes += raw.byteLength;
|
|
17374
|
+
archivedRelativePaths.push(candidate.relativePath);
|
|
17375
|
+
}
|
|
17376
|
+
} else {
|
|
17377
|
+
archivedRelativePaths.push(...candidates.map((c) => c.relativePath));
|
|
17378
|
+
}
|
|
17379
|
+
return {
|
|
17380
|
+
dryRun,
|
|
17381
|
+
retentionDays,
|
|
17382
|
+
scannedFiles: candidates.length,
|
|
17383
|
+
archivedFiles,
|
|
17384
|
+
archivedBytes,
|
|
17385
|
+
archiveRoot,
|
|
17386
|
+
archivedRelativePaths
|
|
17387
|
+
};
|
|
17388
|
+
}
|
|
17389
|
+
|
|
17390
|
+
// src/maintenance/rebuild-observations.ts
|
|
17391
|
+
import path36 from "path";
|
|
17392
|
+
import { readdir as readdir14, readFile as readFile25 } from "fs/promises";
|
|
17393
|
+
|
|
17394
|
+
// src/maintenance/observation-ledger-utils.ts
|
|
17395
|
+
import path35 from "path";
|
|
17396
|
+
import { mkdir as mkdir26, readFile as readFile24, writeFile as writeFile23 } from "fs/promises";
|
|
17397
|
+
function toHourBucketIso(timestamp) {
|
|
17398
|
+
const normalized = /(?:Z|[+-]\d{2}:\d{2})$/u.test(timestamp) ? timestamp : `${timestamp}Z`;
|
|
17399
|
+
const ms = Date.parse(normalized);
|
|
17400
|
+
if (!Number.isFinite(ms)) return null;
|
|
17401
|
+
const d = new Date(ms);
|
|
17402
|
+
d.setUTCMinutes(0, 0, 0);
|
|
17403
|
+
return d.toISOString();
|
|
17404
|
+
}
|
|
17405
|
+
async function backupAndWriteRebuiltObservations(options) {
|
|
17406
|
+
const stamp = options.now.toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z");
|
|
17407
|
+
const archiveRoot = path35.join(options.memoryDir, "archive", "observations", stamp);
|
|
17408
|
+
let backupPath = path35.join(
|
|
17409
|
+
archiveRoot,
|
|
17410
|
+
"state",
|
|
17411
|
+
"observation-ledger",
|
|
17412
|
+
"rebuilt-observations.jsonl"
|
|
17413
|
+
);
|
|
17414
|
+
try {
|
|
17415
|
+
const existing = await readFile24(options.outputPath, "utf-8");
|
|
17416
|
+
await mkdir26(path35.dirname(backupPath), { recursive: true });
|
|
17417
|
+
await writeFile23(backupPath, existing, "utf-8");
|
|
17418
|
+
} catch (err) {
|
|
17419
|
+
const code = err.code;
|
|
17420
|
+
if (code && code === "ENOENT") {
|
|
17421
|
+
backupPath = void 0;
|
|
17422
|
+
} else {
|
|
17423
|
+
throw err;
|
|
17424
|
+
}
|
|
17425
|
+
}
|
|
17426
|
+
const rebuiltAt = options.now.toISOString();
|
|
17427
|
+
const lines = options.rows.map(
|
|
17428
|
+
(row) => JSON.stringify({
|
|
17429
|
+
...row,
|
|
17430
|
+
rebuiltAt
|
|
17431
|
+
})
|
|
17432
|
+
);
|
|
17433
|
+
await mkdir26(path35.dirname(options.outputPath), { recursive: true });
|
|
17434
|
+
await writeFile23(options.outputPath, lines.length > 0 ? `${lines.join("\n")}
|
|
17435
|
+
` : "", "utf-8");
|
|
17436
|
+
return backupPath;
|
|
17437
|
+
}
|
|
17438
|
+
|
|
17439
|
+
// src/maintenance/rebuild-observations.ts
|
|
17440
|
+
function toSortableKey(sessionKey, hour) {
|
|
17441
|
+
return `${sessionKey}\0${hour}`;
|
|
17442
|
+
}
|
|
17443
|
+
async function listTranscriptFiles(root) {
|
|
17444
|
+
const out = [];
|
|
17445
|
+
let entries;
|
|
17446
|
+
try {
|
|
17447
|
+
entries = await readdir14(root, { withFileTypes: true });
|
|
17448
|
+
} catch (err) {
|
|
17449
|
+
const code = err.code;
|
|
17450
|
+
if (code && code === "ENOENT") return out;
|
|
17451
|
+
throw err;
|
|
17452
|
+
}
|
|
17453
|
+
for (const entry of entries) {
|
|
17454
|
+
if (entry.name === "." || entry.name === "..") continue;
|
|
17455
|
+
if (entry.isSymbolicLink()) continue;
|
|
17456
|
+
const full = path36.join(root, entry.name);
|
|
17457
|
+
if (entry.isDirectory()) {
|
|
17458
|
+
out.push(...await listTranscriptFiles(full));
|
|
17459
|
+
continue;
|
|
17460
|
+
}
|
|
17461
|
+
if (entry.isFile() && entry.name.endsWith(".jsonl")) {
|
|
17462
|
+
out.push(full);
|
|
17463
|
+
}
|
|
17464
|
+
}
|
|
17465
|
+
out.sort((a, b) => a.localeCompare(b));
|
|
17466
|
+
return out;
|
|
17467
|
+
}
|
|
17468
|
+
function buildLedgerRows(linesByFile) {
|
|
17469
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
17470
|
+
let parsedTurns = 0;
|
|
17471
|
+
let malformedLines = 0;
|
|
17472
|
+
for (const rawFile of linesByFile) {
|
|
17473
|
+
for (const line of rawFile.split("\n")) {
|
|
17474
|
+
if (!line.trim()) continue;
|
|
17475
|
+
let parsed;
|
|
17476
|
+
try {
|
|
17477
|
+
const candidate = JSON.parse(line);
|
|
17478
|
+
if (candidate == null || typeof candidate !== "object" || Array.isArray(candidate)) {
|
|
17479
|
+
malformedLines += 1;
|
|
17480
|
+
continue;
|
|
17481
|
+
}
|
|
17482
|
+
parsed = candidate;
|
|
17483
|
+
} catch {
|
|
17484
|
+
malformedLines += 1;
|
|
17485
|
+
continue;
|
|
17486
|
+
}
|
|
17487
|
+
if (typeof parsed.sessionKey !== "string" || parsed.sessionKey.length === 0) continue;
|
|
17488
|
+
if (parsed.role !== "user" && parsed.role !== "assistant") continue;
|
|
17489
|
+
if (typeof parsed.timestamp !== "string") continue;
|
|
17490
|
+
const hour = toHourBucketIso(parsed.timestamp);
|
|
17491
|
+
if (!hour) continue;
|
|
17492
|
+
const key = toSortableKey(parsed.sessionKey, hour);
|
|
17493
|
+
const existing = byKey.get(key) ?? {
|
|
17494
|
+
sessionKey: parsed.sessionKey,
|
|
17495
|
+
hour,
|
|
17496
|
+
turnCount: 0,
|
|
17497
|
+
userTurns: 0,
|
|
17498
|
+
assistantTurns: 0
|
|
17499
|
+
};
|
|
17500
|
+
existing.turnCount += 1;
|
|
17501
|
+
if (parsed.role === "user") existing.userTurns += 1;
|
|
17502
|
+
if (parsed.role === "assistant") existing.assistantTurns += 1;
|
|
17503
|
+
byKey.set(key, existing);
|
|
17504
|
+
parsedTurns += 1;
|
|
17505
|
+
}
|
|
17506
|
+
}
|
|
17507
|
+
const aggregates = Array.from(byKey.values()).sort((a, b) => {
|
|
17508
|
+
if (a.sessionKey !== b.sessionKey) return a.sessionKey.localeCompare(b.sessionKey);
|
|
17509
|
+
return a.hour.localeCompare(b.hour);
|
|
17510
|
+
});
|
|
17511
|
+
return { aggregates, parsedTurns, malformedLines };
|
|
17512
|
+
}
|
|
17513
|
+
async function rebuildObservations(options) {
|
|
17514
|
+
const dryRun = options.dryRun !== false;
|
|
17515
|
+
const now = options.now ?? /* @__PURE__ */ new Date();
|
|
17516
|
+
const transcriptsRoot = path36.join(options.memoryDir, "transcripts");
|
|
17517
|
+
const outputPath = path36.join(
|
|
17518
|
+
options.memoryDir,
|
|
17519
|
+
"state",
|
|
17520
|
+
"observation-ledger",
|
|
17521
|
+
"rebuilt-observations.jsonl"
|
|
17522
|
+
);
|
|
17523
|
+
const transcriptFiles = await listTranscriptFiles(transcriptsRoot);
|
|
17524
|
+
const contents = [];
|
|
17525
|
+
for (const file of transcriptFiles) {
|
|
17526
|
+
try {
|
|
17527
|
+
contents.push(await readFile25(file, "utf-8"));
|
|
17528
|
+
} catch {
|
|
17529
|
+
}
|
|
17530
|
+
}
|
|
17531
|
+
const { aggregates, parsedTurns, malformedLines } = buildLedgerRows(contents);
|
|
17532
|
+
let backupPath;
|
|
17533
|
+
if (!dryRun) {
|
|
17534
|
+
backupPath = await backupAndWriteRebuiltObservations({
|
|
17535
|
+
memoryDir: options.memoryDir,
|
|
17536
|
+
outputPath,
|
|
17537
|
+
rows: aggregates,
|
|
17538
|
+
now
|
|
17539
|
+
});
|
|
17540
|
+
}
|
|
17541
|
+
return {
|
|
17542
|
+
dryRun,
|
|
17543
|
+
scannedFiles: transcriptFiles.length,
|
|
17544
|
+
parsedTurns,
|
|
17545
|
+
malformedLines,
|
|
17546
|
+
rebuiltRows: aggregates.length,
|
|
17547
|
+
outputPath,
|
|
17548
|
+
backupPath
|
|
17549
|
+
};
|
|
17550
|
+
}
|
|
17551
|
+
|
|
17552
|
+
// src/maintenance/migrate-observations.ts
|
|
17553
|
+
import path37 from "path";
|
|
17554
|
+
import { readdir as readdir15, readFile as readFile26 } from "fs/promises";
|
|
17555
|
+
function toNonNegativeInt(value) {
|
|
17556
|
+
if (typeof value !== "number" || !Number.isFinite(value)) return null;
|
|
17557
|
+
const normalized = Math.floor(value);
|
|
17558
|
+
if (normalized < 0) return null;
|
|
17559
|
+
return normalized;
|
|
17560
|
+
}
|
|
17561
|
+
function toHourIso(value) {
|
|
17562
|
+
if (typeof value !== "string" || value.length === 0) return null;
|
|
17563
|
+
return toHourBucketIso(value);
|
|
17564
|
+
}
|
|
17565
|
+
function toSessionKey(row) {
|
|
17566
|
+
const candidates = [row.sessionKey, row.session, row.session_id, row.sessionId];
|
|
17567
|
+
for (const candidate of candidates) {
|
|
17568
|
+
if (typeof candidate === "string" && candidate.length > 0) return candidate;
|
|
17569
|
+
}
|
|
17570
|
+
return null;
|
|
17571
|
+
}
|
|
17572
|
+
function toLegacyHour(row) {
|
|
17573
|
+
const candidates = [row.hour, row.hourStart, row.timestamp, row.time];
|
|
17574
|
+
for (const candidate of candidates) {
|
|
17575
|
+
const hour = toHourIso(candidate);
|
|
17576
|
+
if (hour) return hour;
|
|
17577
|
+
}
|
|
17578
|
+
return null;
|
|
17579
|
+
}
|
|
17580
|
+
function toCounts(row) {
|
|
17581
|
+
const role = row.role === "user" || row.role === "assistant" ? row.role : null;
|
|
17582
|
+
const explicitTurnCount = toNonNegativeInt(row.turnCount) ?? toNonNegativeInt(row.turns) ?? toNonNegativeInt(row.totalTurns);
|
|
17583
|
+
const explicitUserTurns = toNonNegativeInt(row.userTurns) ?? toNonNegativeInt(row.user) ?? toNonNegativeInt(row.userCount);
|
|
17584
|
+
const explicitAssistantTurns = toNonNegativeInt(row.assistantTurns) ?? toNonNegativeInt(row.assistant) ?? toNonNegativeInt(row.assistantCount);
|
|
17585
|
+
let turnCount = explicitTurnCount ?? 0;
|
|
17586
|
+
let userTurns = explicitUserTurns ?? 0;
|
|
17587
|
+
let assistantTurns = explicitAssistantTurns ?? 0;
|
|
17588
|
+
if (turnCount === 0 && userTurns === 0 && assistantTurns === 0 && role) {
|
|
17589
|
+
turnCount = 1;
|
|
17590
|
+
if (role === "user") userTurns = 1;
|
|
17591
|
+
if (role === "assistant") assistantTurns = 1;
|
|
17592
|
+
}
|
|
17593
|
+
if (turnCount === 0 && (userTurns > 0 || assistantTurns > 0)) {
|
|
17594
|
+
turnCount = userTurns + assistantTurns;
|
|
17595
|
+
}
|
|
17596
|
+
if (turnCount < userTurns + assistantTurns) {
|
|
17597
|
+
turnCount = userTurns + assistantTurns;
|
|
17598
|
+
}
|
|
17599
|
+
if (turnCount === 0) return null;
|
|
17600
|
+
return { turnCount, userTurns, assistantTurns };
|
|
17601
|
+
}
|
|
17602
|
+
async function listLegacyObservationFiles(root) {
|
|
17603
|
+
let entries;
|
|
17604
|
+
try {
|
|
17605
|
+
entries = await readdir15(root, { withFileTypes: true });
|
|
17606
|
+
} catch (err) {
|
|
17607
|
+
const code = err.code;
|
|
17608
|
+
if (code && code === "ENOENT") return [];
|
|
17609
|
+
throw err;
|
|
17610
|
+
}
|
|
17611
|
+
const files = entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => entry.name).filter((name) => name !== "rebuilt-observations.jsonl").sort((a, b) => a.localeCompare(b));
|
|
17612
|
+
return files;
|
|
17613
|
+
}
|
|
17614
|
+
async function migrateObservations(options) {
|
|
17615
|
+
const dryRun = options.dryRun !== false;
|
|
17616
|
+
const now = options.now ?? /* @__PURE__ */ new Date();
|
|
17617
|
+
const ledgerRoot = path37.join(options.memoryDir, "state", "observation-ledger");
|
|
17618
|
+
const outputPath = path37.join(ledgerRoot, "rebuilt-observations.jsonl");
|
|
17619
|
+
const legacyFiles = await listLegacyObservationFiles(ledgerRoot);
|
|
17620
|
+
const sourceRelativePaths = legacyFiles.map((name) => path37.join("state", "observation-ledger", name));
|
|
17621
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
17622
|
+
let parsedRows = 0;
|
|
17623
|
+
let malformedLines = 0;
|
|
17624
|
+
for (const file of legacyFiles) {
|
|
17625
|
+
const full = path37.join(ledgerRoot, file);
|
|
17626
|
+
const raw = await readFile26(full, "utf-8");
|
|
17627
|
+
for (const line of raw.split("\n")) {
|
|
17628
|
+
if (!line.trim()) continue;
|
|
17629
|
+
let parsed;
|
|
17630
|
+
try {
|
|
17631
|
+
const candidate = JSON.parse(line);
|
|
17632
|
+
if (candidate == null || typeof candidate !== "object" || Array.isArray(candidate)) {
|
|
17633
|
+
malformedLines += 1;
|
|
17634
|
+
continue;
|
|
17635
|
+
}
|
|
17636
|
+
parsed = candidate;
|
|
17637
|
+
} catch {
|
|
17638
|
+
malformedLines += 1;
|
|
17639
|
+
continue;
|
|
17640
|
+
}
|
|
17641
|
+
const sessionKey = toSessionKey(parsed);
|
|
17642
|
+
const hour = toLegacyHour(parsed);
|
|
17643
|
+
const counts = toCounts(parsed);
|
|
17644
|
+
if (!sessionKey || !hour || !counts) {
|
|
17645
|
+
malformedLines += 1;
|
|
17646
|
+
continue;
|
|
17647
|
+
}
|
|
17648
|
+
const key = `${sessionKey}\0${hour}`;
|
|
17649
|
+
const existing = byKey.get(key) ?? {
|
|
17650
|
+
sessionKey,
|
|
17651
|
+
hour,
|
|
17652
|
+
turnCount: 0,
|
|
17653
|
+
userTurns: 0,
|
|
17654
|
+
assistantTurns: 0
|
|
17655
|
+
};
|
|
17656
|
+
existing.turnCount += counts.turnCount;
|
|
17657
|
+
existing.userTurns += counts.userTurns;
|
|
17658
|
+
existing.assistantTurns += counts.assistantTurns;
|
|
17659
|
+
if (existing.turnCount < existing.userTurns + existing.assistantTurns) {
|
|
17660
|
+
existing.turnCount = existing.userTurns + existing.assistantTurns;
|
|
17661
|
+
}
|
|
17662
|
+
byKey.set(key, existing);
|
|
17663
|
+
parsedRows += 1;
|
|
17664
|
+
}
|
|
17665
|
+
}
|
|
17666
|
+
const aggregates = Array.from(byKey.values()).sort((a, b) => {
|
|
17667
|
+
if (a.sessionKey !== b.sessionKey) return a.sessionKey.localeCompare(b.sessionKey);
|
|
17668
|
+
return a.hour.localeCompare(b.hour);
|
|
17669
|
+
});
|
|
17670
|
+
if (!dryRun && legacyFiles.length === 0) {
|
|
17671
|
+
return {
|
|
17672
|
+
dryRun,
|
|
17673
|
+
scannedFiles: legacyFiles.length,
|
|
17674
|
+
parsedRows,
|
|
17675
|
+
malformedLines,
|
|
17676
|
+
migratedRows: aggregates.length,
|
|
17677
|
+
outputPath,
|
|
17678
|
+
sourceRelativePaths
|
|
17679
|
+
};
|
|
17680
|
+
}
|
|
17681
|
+
let backupPath;
|
|
17682
|
+
if (!dryRun) {
|
|
17683
|
+
backupPath = await backupAndWriteRebuiltObservations({
|
|
17684
|
+
memoryDir: options.memoryDir,
|
|
17685
|
+
outputPath,
|
|
17686
|
+
rows: aggregates,
|
|
17687
|
+
now
|
|
17688
|
+
});
|
|
17689
|
+
}
|
|
17690
|
+
return {
|
|
17691
|
+
dryRun,
|
|
17692
|
+
scannedFiles: legacyFiles.length,
|
|
17693
|
+
parsedRows,
|
|
17694
|
+
malformedLines,
|
|
17695
|
+
migratedRows: aggregates.length,
|
|
17696
|
+
outputPath,
|
|
17697
|
+
backupPath,
|
|
17698
|
+
sourceRelativePaths
|
|
17699
|
+
};
|
|
17700
|
+
}
|
|
17701
|
+
|
|
17289
17702
|
// src/cli.ts
|
|
17290
17703
|
function rankCandidateForKeep(a, b) {
|
|
17291
17704
|
const aConfidence = typeof a.frontmatter.confidence === "number" ? a.frontmatter.confidence : 0;
|
|
@@ -17335,6 +17748,28 @@ function planExactDuplicateDeletions(memories) {
|
|
|
17335
17748
|
function planAggressiveDuplicateDeletions(memories) {
|
|
17336
17749
|
return buildDedupePlan(memories, (memory) => normalizeAggressiveBody(memory.content));
|
|
17337
17750
|
}
|
|
17751
|
+
async function runArchiveObservationsCliCommand(options) {
|
|
17752
|
+
return archiveObservations({
|
|
17753
|
+
memoryDir: options.memoryDir,
|
|
17754
|
+
retentionDays: options.retentionDays,
|
|
17755
|
+
dryRun: options.write !== true,
|
|
17756
|
+
now: options.now
|
|
17757
|
+
});
|
|
17758
|
+
}
|
|
17759
|
+
async function runRebuildObservationsCliCommand(options) {
|
|
17760
|
+
return rebuildObservations({
|
|
17761
|
+
memoryDir: options.memoryDir,
|
|
17762
|
+
dryRun: options.write !== true,
|
|
17763
|
+
now: options.now
|
|
17764
|
+
});
|
|
17765
|
+
}
|
|
17766
|
+
async function runMigrateObservationsCliCommand(options) {
|
|
17767
|
+
return migrateObservations({
|
|
17768
|
+
memoryDir: options.memoryDir,
|
|
17769
|
+
dryRun: options.write !== true,
|
|
17770
|
+
now: options.now
|
|
17771
|
+
});
|
|
17772
|
+
}
|
|
17338
17773
|
async function withTimeout(promise, timeoutMs, timeoutMessage) {
|
|
17339
17774
|
let timer;
|
|
17340
17775
|
try {
|
|
@@ -17350,7 +17785,7 @@ async function withTimeout(promise, timeoutMs, timeoutMessage) {
|
|
|
17350
17785
|
}
|
|
17351
17786
|
async function runReplayCliCommand(orchestrator, options) {
|
|
17352
17787
|
const extractionIdleTimeoutMs = Number.isFinite(options.extractionIdleTimeoutMs) ? Math.max(1e3, Math.floor(options.extractionIdleTimeoutMs)) : 15 * 6e4;
|
|
17353
|
-
const inputRaw = await
|
|
17788
|
+
const inputRaw = await readFile27(options.inputPath, "utf-8");
|
|
17354
17789
|
const registry = buildReplayNormalizerRegistry([
|
|
17355
17790
|
openclawReplayNormalizer,
|
|
17356
17791
|
claudeReplayNormalizer,
|
|
@@ -17415,7 +17850,7 @@ async function runReplayCliCommand(orchestrator, options) {
|
|
|
17415
17850
|
async function getPluginVersion() {
|
|
17416
17851
|
try {
|
|
17417
17852
|
const pkgPath = new URL("../package.json", import.meta.url);
|
|
17418
|
-
const raw = await
|
|
17853
|
+
const raw = await readFile27(pkgPath, "utf-8");
|
|
17419
17854
|
const parsed = JSON.parse(raw);
|
|
17420
17855
|
return parsed.version ?? "unknown";
|
|
17421
17856
|
} catch {
|
|
@@ -17434,32 +17869,32 @@ async function resolveMemoryDirForNamespace(orchestrator, namespace) {
|
|
|
17434
17869
|
const ns = (namespace ?? "").trim();
|
|
17435
17870
|
if (!ns) return orchestrator.config.memoryDir;
|
|
17436
17871
|
if (!orchestrator.config.namespacesEnabled) return orchestrator.config.memoryDir;
|
|
17437
|
-
const candidate =
|
|
17872
|
+
const candidate = path38.join(orchestrator.config.memoryDir, "namespaces", ns);
|
|
17438
17873
|
if (ns === orchestrator.config.defaultNamespace) {
|
|
17439
17874
|
return await exists2(candidate) ? candidate : orchestrator.config.memoryDir;
|
|
17440
17875
|
}
|
|
17441
17876
|
return candidate;
|
|
17442
17877
|
}
|
|
17443
17878
|
async function readAllMemoryFiles(memoryDir) {
|
|
17444
|
-
const roots = [
|
|
17879
|
+
const roots = [path38.join(memoryDir, "facts"), path38.join(memoryDir, "corrections")];
|
|
17445
17880
|
const out = [];
|
|
17446
17881
|
const walk = async (dir) => {
|
|
17447
17882
|
let entries;
|
|
17448
17883
|
try {
|
|
17449
|
-
entries = await
|
|
17884
|
+
entries = await readdir16(dir, { withFileTypes: true });
|
|
17450
17885
|
} catch {
|
|
17451
17886
|
return;
|
|
17452
17887
|
}
|
|
17453
17888
|
for (const entry of entries) {
|
|
17454
17889
|
const entryName = typeof entry.name === "string" ? entry.name : entry.name.toString("utf-8");
|
|
17455
|
-
const fullPath =
|
|
17890
|
+
const fullPath = path38.join(dir, entryName);
|
|
17456
17891
|
if (entry.isDirectory()) {
|
|
17457
17892
|
await walk(fullPath);
|
|
17458
17893
|
continue;
|
|
17459
17894
|
}
|
|
17460
17895
|
if (!entry.isFile() || !entryName.endsWith(".md")) continue;
|
|
17461
17896
|
try {
|
|
17462
|
-
const raw = await
|
|
17897
|
+
const raw = await readFile27(fullPath, "utf-8");
|
|
17463
17898
|
const parsed = raw.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
|
|
17464
17899
|
if (!parsed) continue;
|
|
17465
17900
|
const fmRaw = parsed[1];
|
|
@@ -17696,6 +18131,66 @@ function registerCli(api, orchestrator) {
|
|
|
17696
18131
|
}
|
|
17697
18132
|
console.log("OK");
|
|
17698
18133
|
});
|
|
18134
|
+
cmd.command("archive-observations").description("Archive aged observation artifacts (dry-run by default)").option("--retention-days <n>", "Archive files older than N days", "30").option("--write", "Apply archive mutations (default: dry-run)").action(async (...args) => {
|
|
18135
|
+
const options = args[0] ?? {};
|
|
18136
|
+
const retentionDays = parseInt(String(options.retentionDays ?? "30"), 10);
|
|
18137
|
+
const result = await runArchiveObservationsCliCommand({
|
|
18138
|
+
memoryDir: orchestrator.config.memoryDir,
|
|
18139
|
+
retentionDays: Number.isFinite(retentionDays) ? retentionDays : 30,
|
|
18140
|
+
write: options.write === true
|
|
18141
|
+
});
|
|
18142
|
+
console.log(`Dry run: ${result.dryRun ? "yes" : "no"}`);
|
|
18143
|
+
console.log(`Retention days: ${result.retentionDays}`);
|
|
18144
|
+
console.log(`Scanned files: ${result.scannedFiles}`);
|
|
18145
|
+
console.log(`Archived files: ${result.archivedFiles}`);
|
|
18146
|
+
console.log(`Archived bytes: ${result.archivedBytes}`);
|
|
18147
|
+
if (result.archivedRelativePaths.length > 0) {
|
|
18148
|
+
console.log("Archived paths:");
|
|
18149
|
+
for (const relPath of result.archivedRelativePaths.slice(0, 20)) {
|
|
18150
|
+
console.log(` - ${relPath}`);
|
|
18151
|
+
}
|
|
18152
|
+
if (result.archivedRelativePaths.length > 20) {
|
|
18153
|
+
console.log(` ... and ${result.archivedRelativePaths.length - 20} more`);
|
|
18154
|
+
}
|
|
18155
|
+
}
|
|
18156
|
+
console.log("OK");
|
|
18157
|
+
});
|
|
18158
|
+
cmd.command("rebuild-observations").description("Rebuild observation ledger from transcript history (dry-run by default)").option("--write", "Write rebuilt ledger (default: dry-run)").action(async (...args) => {
|
|
18159
|
+
const options = args[0] ?? {};
|
|
18160
|
+
const result = await runRebuildObservationsCliCommand({
|
|
18161
|
+
memoryDir: orchestrator.config.memoryDir,
|
|
18162
|
+
write: options.write === true
|
|
18163
|
+
});
|
|
18164
|
+
console.log(`Dry run: ${result.dryRun ? "yes" : "no"}`);
|
|
18165
|
+
console.log(`Scanned transcript files: ${result.scannedFiles}`);
|
|
18166
|
+
console.log(`Parsed turns: ${result.parsedTurns}`);
|
|
18167
|
+
console.log(`Malformed lines: ${result.malformedLines}`);
|
|
18168
|
+
console.log(`Rebuilt rows: ${result.rebuiltRows}`);
|
|
18169
|
+
console.log(`Output path: ${result.outputPath}`);
|
|
18170
|
+
if (result.backupPath) console.log(`Backup path: ${result.backupPath}`);
|
|
18171
|
+
console.log("OK");
|
|
18172
|
+
});
|
|
18173
|
+
cmd.command("migrate-observations").description("Migrate legacy observation ledgers into rebuilt format (dry-run by default)").option("--write", "Write migrated ledger (default: dry-run)").action(async (...args) => {
|
|
18174
|
+
const options = args[0] ?? {};
|
|
18175
|
+
const result = await runMigrateObservationsCliCommand({
|
|
18176
|
+
memoryDir: orchestrator.config.memoryDir,
|
|
18177
|
+
write: options.write === true
|
|
18178
|
+
});
|
|
18179
|
+
console.log(`Dry run: ${result.dryRun ? "yes" : "no"}`);
|
|
18180
|
+
console.log(`Scanned legacy files: ${result.scannedFiles}`);
|
|
18181
|
+
console.log(`Parsed rows: ${result.parsedRows}`);
|
|
18182
|
+
console.log(`Malformed lines: ${result.malformedLines}`);
|
|
18183
|
+
console.log(`Migrated rows: ${result.migratedRows}`);
|
|
18184
|
+
if (result.sourceRelativePaths.length > 0) {
|
|
18185
|
+
console.log("Source files:");
|
|
18186
|
+
for (const relPath of result.sourceRelativePaths) {
|
|
18187
|
+
console.log(` - ${relPath}`);
|
|
18188
|
+
}
|
|
18189
|
+
}
|
|
18190
|
+
console.log(`Output path: ${result.outputPath}`);
|
|
18191
|
+
if (result.backupPath) console.log(`Backup path: ${result.backupPath}`);
|
|
18192
|
+
console.log("OK");
|
|
18193
|
+
});
|
|
17699
18194
|
cmd.command("dedupe-exact").description("Delete exact duplicate memory entries (same body text), keeping highest-confidence/newest copy").option("--dry-run", "Show what would be deleted without deleting files").option("--namespace <ns>", "Namespace to dedupe (v3.0+, default: config defaultNamespace)", "").option("--qmd-sync", "Run QMD update/embed after deletions (default: off)").action(async (...args) => {
|
|
17700
18195
|
const options = args[0] ?? {};
|
|
17701
18196
|
const dryRun = options.dryRun === true;
|
|
@@ -17724,7 +18219,7 @@ function registerCli(api, orchestrator) {
|
|
|
17724
18219
|
let deleted = 0;
|
|
17725
18220
|
for (const filePath of plan.deletePaths) {
|
|
17726
18221
|
try {
|
|
17727
|
-
await
|
|
18222
|
+
await unlink5(filePath);
|
|
17728
18223
|
deleted += 1;
|
|
17729
18224
|
} catch (err) {
|
|
17730
18225
|
console.log(` failed to delete ${filePath}: ${String(err)}`);
|
|
@@ -17772,7 +18267,7 @@ function registerCli(api, orchestrator) {
|
|
|
17772
18267
|
let deleted = 0;
|
|
17773
18268
|
for (const filePath of plan.deletePaths) {
|
|
17774
18269
|
try {
|
|
17775
|
-
await
|
|
18270
|
+
await unlink5(filePath);
|
|
17776
18271
|
deleted += 1;
|
|
17777
18272
|
} catch (err) {
|
|
17778
18273
|
console.log(` failed to delete ${filePath}: ${String(err)}`);
|
|
@@ -17936,7 +18431,7 @@ function registerCli(api, orchestrator) {
|
|
|
17936
18431
|
}
|
|
17937
18432
|
});
|
|
17938
18433
|
cmd.command("identity").description("Show agent identity reflections").action(async () => {
|
|
17939
|
-
const workspaceDir =
|
|
18434
|
+
const workspaceDir = path38.join(process.env.HOME ?? "~", ".openclaw", "workspace");
|
|
17940
18435
|
const identity = await orchestrator.storage.readIdentity(workspaceDir);
|
|
17941
18436
|
if (!identity) {
|
|
17942
18437
|
console.log("No identity file found.");
|
|
@@ -18159,8 +18654,8 @@ function registerCli(api, orchestrator) {
|
|
|
18159
18654
|
const options = args[0] ?? {};
|
|
18160
18655
|
const threadId = options.thread;
|
|
18161
18656
|
const top = parseInt(options.top ?? "10", 10);
|
|
18162
|
-
const memoryDir =
|
|
18163
|
-
const threading = new ThreadingManager(
|
|
18657
|
+
const memoryDir = path38.join(process.env.HOME ?? "~", ".openclaw", "workspace", "memory", "local");
|
|
18658
|
+
const threading = new ThreadingManager(path38.join(memoryDir, "threads"));
|
|
18164
18659
|
if (threadId) {
|
|
18165
18660
|
const thread = await threading.loadThread(threadId);
|
|
18166
18661
|
if (!thread) {
|
|
@@ -18333,16 +18828,16 @@ function parseDuration(duration) {
|
|
|
18333
18828
|
}
|
|
18334
18829
|
|
|
18335
18830
|
// src/index.ts
|
|
18336
|
-
import { readFile as
|
|
18831
|
+
import { readFile as readFile28, writeFile as writeFile24 } from "fs/promises";
|
|
18337
18832
|
import { readFileSync as readFileSync4 } from "fs";
|
|
18338
|
-
import
|
|
18833
|
+
import path39 from "path";
|
|
18339
18834
|
import os5 from "os";
|
|
18340
18835
|
var ENGRAM_REGISTERED_GUARD = "__openclawEngramRegistered";
|
|
18341
18836
|
function loadPluginConfigFromFile() {
|
|
18342
18837
|
try {
|
|
18343
18838
|
const explicitConfigPath = process.env.OPENCLAW_ENGRAM_CONFIG_PATH || process.env.OPENCLAW_CONFIG_PATH;
|
|
18344
18839
|
const homeDir = process.env.HOME ?? os5.homedir();
|
|
18345
|
-
const configPath = explicitConfigPath && explicitConfigPath.length > 0 ? explicitConfigPath :
|
|
18840
|
+
const configPath = explicitConfigPath && explicitConfigPath.length > 0 ? explicitConfigPath : path39.join(homeDir, ".openclaw", "openclaw.json");
|
|
18346
18841
|
const content = readFileSync4(configPath, "utf-8");
|
|
18347
18842
|
const config = JSON.parse(content);
|
|
18348
18843
|
const pluginEntry = config?.plugins?.entries?.["openclaw-engram"];
|
|
@@ -18550,11 +19045,11 @@ Use this context naturally when relevant. Never quote or expose this memory cont
|
|
|
18550
19045
|
);
|
|
18551
19046
|
async function ensureHourlySummaryCron(api2) {
|
|
18552
19047
|
const jobId = "engram-hourly-summary";
|
|
18553
|
-
const cronFilePath =
|
|
19048
|
+
const cronFilePath = path39.join(os5.homedir(), ".openclaw", "cron", "jobs.json");
|
|
18554
19049
|
try {
|
|
18555
19050
|
let jobsData = { version: 1, jobs: [] };
|
|
18556
19051
|
try {
|
|
18557
|
-
const content = await
|
|
19052
|
+
const content = await readFile28(cronFilePath, "utf-8");
|
|
18558
19053
|
jobsData = JSON.parse(content);
|
|
18559
19054
|
} catch {
|
|
18560
19055
|
}
|
|
@@ -18591,7 +19086,7 @@ Use this context naturally when relevant. Never quote or expose this memory cont
|
|
|
18591
19086
|
state: {}
|
|
18592
19087
|
};
|
|
18593
19088
|
jobsData.jobs.push(newJob);
|
|
18594
|
-
await
|
|
19089
|
+
await writeFile24(cronFilePath, JSON.stringify(jobsData, null, 2), "utf-8");
|
|
18595
19090
|
log.info("auto-registered hourly summary cron job");
|
|
18596
19091
|
} catch (err) {
|
|
18597
19092
|
log.error("failed to auto-register hourly summary cron job:", err);
|