@remnic/core 9.3.762 → 9.3.764

package/dist/purge-6ATBGT77.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/maintenance/purge.ts"],"sourcesContent":["/**\n * Operator-facing bulk hard-delete (issue #686 retention-completion).\n *\n * `purgeMemories(opts)` hard-deletes memories that are:\n *   - older than `olderThanMs` (measured against `updated ?? created`), AND\n *   - in the specified tier (`\"cold\"` or `\"all\"`), AND\n *   - optionally in `status: \"forgotten\"` only (when `forgottenOnly` is true)\n *\n * Hard-delete means:\n *   1. Remove the file from disk.\n *   2. Trigger a QMD collection update so the index no longer contains it.\n *   3. Append a purge audit entry to the observation ledger.\n *\n * `dryRun: true` (the default) reports what *would* be deleted without\n * actually removing anything — callers must opt into mutations by\n * setting `dryRun: false`.\n *\n * The CLI surface enforces an explicit `--confirm yes` guard and\n * defaults to `--dry-run` to make accidental data loss impossible.\n */\n\nimport path from \"node:path\";\nimport { appendFile, mkdir, unlink } from \"node:fs/promises\";\nimport type { StorageManager } from \"../storage.js\";\nimport type { MemoryFile } from \"../types.js\";\nimport type { SearchBackend } from \"../search/port.js\";\n\nexport type PurgeTierFilter = \"cold\" | \"all\";\n\nexport interface PurgeMemoriesOptions {\n  storage: StorageManager;\n  /**\n   * Only memories whose `updated ?? created` timestamp is older than this\n   * value (in milliseconds, Unix epoch) will be candidates.\n   */\n  olderThanMs: number;\n  /**\n   * `\"cold\"` — only memories in the cold tier (files under `.../cold/`).\n   * `\"all\"` — memories in any tier (hot, cold, or archived).\n   * Default: `\"cold\"`.\n   */\n  tier?: PurgeTierFilter;\n  /**\n   * When `true`, only memories with `status === \"forgotten\"` are candidates.\n   * Default: `false` (purge by age + tier regardless of status).\n   */\n  forgottenOnly?: boolean;\n  /** When `true` (default), report candidates but do not delete. */\n  dryRun?: boolean;\n  /**\n   * Optional QMD backend. When supplied and `dryRun === false`, a\n   * `updateCollection` call is issued after each deletion so the index stays\n   * consistent. Pass `undefined` or omit to skip index maintenance (e.g. in\n   * tests that don't have a live QMD instance).\n   */\n  qmd?: SearchBackend;\n  /** Hot-tier QMD collection name (default: `\"openclaw-engram\"`). */\n  hotCollection?: string;\n  /** Cold-tier QMD collection name (default: `\"openclaw-engram-cold\"`). */\n  coldCollection?: string;\n  /** Optional hook for long-lived hosts to clear their live dedupe mirror after purge hash cleanup. */\n  afterFactHashRemoval?: () => void | Promise<void>;\n  /** Optional shutdown signal forwarded to QMD collection refreshes. */\n  signal?: AbortSignal;\n  /** Override clock for tests. */\n  now?: () => Date;\n}\n\nexport interface PurgeCandidate {\n  id: string;\n  path: string;\n  tier: \"hot\" | \"cold\" | \"archive\";\n  status: string;\n  updatedOrCreated: string;\n  ageMs: number;\n}\n\nexport interface PurgeMemoriesResult {\n  dryRun: boolean;\n  tier: PurgeTierFilter;\n  olderThanMs: number;\n  candidates: PurgeCandidate[];\n  purgedCount: number;\n  alreadyAbsentCount: number;\n  errorCount: number;\n  errors: Array<{ id: string; path: string; error: string }>;\n}\n\nfunction hasErrorCode(err: unknown, code: string): boolean {\n  return typeof err === \"object\" && err !== null && \"code\" in err && (err as { code?: unknown }).code === code;\n}\n\nfunction resolveTimestamp(memory: MemoryFile): string {\n  const fm = memory.frontmatter as unknown as Record<string, unknown>;\n  const updated = typeof fm.updated === \"string\" ? (fm.updated as string) : \"\";\n  const created = typeof fm.created === \"string\" ? (fm.created as string) : \"\";\n  return updated.length > 0 ? updated : created;\n}\n\nasync function logPurgeAudit(\n  storage: StorageManager,\n  candidates: PurgeCandidate[],\n  now: Date,\n  event: \"PURGE_DELETE_INTENT\" | \"PURGE_HARD_DELETE\" | \"PURGE_ALREADY_ABSENT\" = \"PURGE_HARD_DELETE\",\n): Promise<void> {\n  if (candidates.length === 0) return;\n  const ledgerDir = path.join(storage.dir, \"state\", \"observation-ledger\");\n  await mkdir(ledgerDir, { recursive: true });\n  const ledgerPath = path.join(ledgerDir, \"purge-audit.jsonl\");\n  const entries = candidates.map((c) =>\n    JSON.stringify({\n      event,\n      timestamp: now.toISOString(),\n      memoryId: c.id,\n      path: c.path,\n      tier: c.tier,\n      status: c.status,\n      updatedOrCreated: c.updatedOrCreated,\n    }),\n  );\n  if (entries.length > 0) {\n    await appendFile(ledgerPath, `${entries.join(\"\\n\")}\\n`, \"utf-8\");\n  }\n}\n\nexport async function purgeMemories(\n  options: PurgeMemoriesOptions,\n): Promise<PurgeMemoriesResult> {\n  const {\n    storage,\n    olderThanMs,\n    tier = \"cold\",\n    forgottenOnly = false,\n    dryRun = true,\n    qmd,\n    hotCollection = \"openclaw-engram\",\n    coldCollection = \"openclaw-engram-cold\",\n  } = options;\n\n  if (!Number.isFinite(olderThanMs) || olderThanMs <= 0) {\n    throw new Error(\"olderThanMs must be a finite positive number\");\n  }\n\n  const now = (options.now ?? (() => new Date()))();\n  const nowMs = now.getTime();\n\n  // Collect all memories from applicable tiers\n  const [hotMemories, coldMemories, archivedMemories] = await Promise.all([\n    tier === \"all\" ? storage.readAllMemories() : Promise.resolve([]),\n    storage.readAllColdMemories(),\n    tier === \"all\" ? storage.readArchivedMemories() : Promise.resolve([]),\n  ]);\n\n  const poolEntries: Array<{ memory: MemoryFile; resolvedTier: \"hot\" | \"cold\" | \"archive\" }> = [];\n\n  if (tier === \"all\") {\n    for (const m of hotMemories) {\n      poolEntries.push({ memory: m, resolvedTier: \"hot\" });\n    }\n    for (const m of archivedMemories) {\n      poolEntries.push({ memory: m, resolvedTier: \"archive\" });\n    }\n  }\n  for (const m of coldMemories) {\n    poolEntries.push({ memory: m, resolvedTier: \"cold\" });\n  }\n\n  // Build candidate list\n  const candidates: PurgeCandidate[] = [];\n  const candidateMemoriesById = new Map<string, MemoryFile>();\n  for (const { memory, resolvedTier } of poolEntries) {\n    const ts = resolveTimestamp(memory);\n    if (ts.length === 0) continue;\n\n    const tsMs = Date.parse(ts);\n    if (!Number.isFinite(tsMs)) continue;\n\n    const ageMs = nowMs - tsMs;\n    if (ageMs < olderThanMs) continue;\n\n    const fm = memory.frontmatter as unknown as Record<string, unknown>;\n    const status: string =\n      typeof fm.status === \"string\" ? (fm.status as string) : \"active\";\n\n    if (forgottenOnly && status !== \"forgotten\") continue;\n\n    candidates.push({\n      id: memory.frontmatter.id,\n      path: memory.path,\n      tier: resolvedTier,\n      status,\n      updatedOrCreated: ts,\n      ageMs,\n    });\n    candidateMemoriesById.set(memory.frontmatter.id, memory);\n  }\n\n  if (dryRun) {\n    return {\n      dryRun: true,\n      tier,\n      olderThanMs,\n      candidates,\n      purgedCount: 0,\n      alreadyAbsentCount: 0,\n      errorCount: 0,\n      errors: [],\n    };\n  }\n\n  // Hard-delete phase\n  const errors: Array<{ id: string; path: string; error: string }> = [];\n  try {\n    await logPurgeAudit(storage, candidates, now, \"PURGE_DELETE_INTENT\");\n  } catch (auditErr) {\n    errors.push({\n      id: \"(purge-audit)\",\n      path: path.join(storage.dir, \"state\", \"observation-ledger\", \"purge-audit.jsonl\"),\n      error: auditErr instanceof Error ? auditErr.message : String(auditErr),\n    });\n  }\n\n  const actuallyPurged: PurgeCandidate[] = [];\n  const alreadyAbsent: PurgeCandidate[] = [];\n  const collectionsToUpdate = new Set<string>();\n  const addCollectionForCandidate = (candidate: PurgeCandidate) => {\n    if (candidate.tier === \"cold\") {\n      collectionsToUpdate.add(coldCollection);\n    } else if (candidate.tier === \"hot\") {\n      collectionsToUpdate.add(hotCollection);\n    }\n  };\n\n  for (const candidate of candidates) {\n    try {\n      await unlink(candidate.path);\n      actuallyPurged.push(candidate);\n      addCollectionForCandidate(candidate);\n    } catch (err) {\n      const message = err instanceof Error ? err.message : String(err);\n      // ENOENT is fine — already gone\n      if (hasErrorCode(err, \"ENOENT\")) {\n        alreadyAbsent.push(candidate);\n        addCollectionForCandidate(candidate);\n      } else {\n        errors.push({ id: candidate.id, path: candidate.path, error: message });\n      }\n    }\n  }\n\n  const recordPostDeleteAudit = async (\n    purgedCandidates: PurgeCandidate[],\n    event: \"PURGE_HARD_DELETE\" | \"PURGE_ALREADY_ABSENT\",\n  ) => {\n    try {\n      await logPurgeAudit(storage, purgedCandidates, now, event);\n    } catch (auditErr) {\n      errors.push({\n        id: \"(purge-audit)\",\n        path: path.join(storage.dir, \"state\", \"observation-ledger\", \"purge-audit.jsonl\"),\n        error: auditErr instanceof Error ? auditErr.message : String(auditErr),\n      });\n    }\n  };\n  await recordPostDeleteAudit(actuallyPurged, \"PURGE_HARD_DELETE\");\n  await recordPostDeleteAudit(alreadyAbsent, \"PURGE_ALREADY_ABSENT\");\n\n  const resolvedPurges = [...actuallyPurged, ...alreadyAbsent];\n\n  const invalidateTierCaches = (\n    storage as unknown as {\n      invalidateMemoryCachesForTiers?: (tiers: Iterable<\"hot\" | \"cold\" | \"archive\">) => void;\n    }\n  ).invalidateMemoryCachesForTiers;\n  if (typeof invalidateTierCaches === \"function\") {\n    invalidateTierCaches.call(storage, new Set(resolvedPurges.map((candidate) => candidate.tier)));\n  } else if (typeof (storage as unknown as { invalidateAllMemoriesCacheForDir?: () => void }).invalidateAllMemoriesCacheForDir === \"function\") {\n    (storage as unknown as { invalidateAllMemoriesCacheForDir: () => void }).invalidateAllMemoriesCacheForDir();\n  }\n\n  // Update QMD index for affected collections\n  if (qmd) {\n    for (const collection of collectionsToUpdate) {\n      try {\n        if (typeof qmd.updateCollectionStrict === \"function\") {\n          await qmd.updateCollectionStrict(collection, { signal: options.signal });\n        } else {\n          await qmd.updateCollection(collection, { signal: options.signal });\n        }\n      } catch (indexErr) {\n        // Non-fatal — operator can re-index manually\n        errors.push({\n          id: \"(qmd-update)\",\n          path: collection,\n          error: indexErr instanceof Error ? indexErr.message : String(indexErr),\n        });\n      }\n    }\n  }\n\n  const purgedFactMemories = resolvedPurges\n    .map((candidate) => candidateMemoriesById.get(candidate.id))\n    .filter((memory): memory is MemoryFile => memory?.frontmatter.category === \"fact\");\n  if (purgedFactMemories.length > 0) {\n    const removeFactHashes = (\n      storage as unknown as {\n        removeFactContentHashesForMemories?: (memories: MemoryFile[]) => Promise<void>;\n      }\n    ).removeFactContentHashesForMemories;\n    if (typeof removeFactHashes === \"function\") {\n      try {\n        await removeFactHashes.call(storage, purgedFactMemories);\n        await options.afterFactHashRemoval?.();\n      } catch (hashErr) {\n        errors.push({\n          id: \"(fact-hash-index)\",\n          path: path.join(storage.dir, \"state\", \"fact-hashes.txt\"),\n          error: hashErr instanceof Error ? hashErr.message : String(hashErr),\n        });\n      }\n    }\n  }\n\n  return {\n    dryRun: false,\n    tier,\n    olderThanMs,\n    candidates,\n    purgedCount: actuallyPurged.length,\n    alreadyAbsentCount: alreadyAbsent.length,\n    errorCount: errors.length,\n    errors,\n  };\n}\n"],"mappings":";;;AAqBA,OAAO,UAAU;AACjB,SAAS,YAAY,OAAO,cAAc;AAkE1C,SAAS,aAAa,KAAc,MAAuB;AACzD,SAAO,OAAO,QAAQ,YAAY,QAAQ,QAAQ,UAAU,OAAQ,IAA2B,SAAS;AAC1G;AAEA,SAAS,iBAAiB,QAA4B;AACpD,QAAM,KAAK,OAAO;AAClB,QAAM,UAAU,OAAO,GAAG,YAAY,WAAY,GAAG,UAAqB;AAC1E,QAAM,UAAU,OAAO,GAAG,YAAY,WAAY,GAAG,UAAqB;AAC1E,SAAO,QAAQ,SAAS,IAAI,UAAU;AACxC;AAEA,eAAe,cACb,SACA,YACA,KACA,QAA8E,qBAC/D;AACf,MAAI,WAAW,WAAW,EAAG;AAC7B,QAAM,YAAY,KAAK,KAAK,QAAQ,KAAK,SAAS,oBAAoB;AACtE,QAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,QAAM,aAAa,KAAK,KAAK,WAAW,mBAAmB;AAC3D,QAAM,UAAU,WAAW;AAAA,IAAI,CAAC,MAC9B,KAAK,UAAU;AAAA,MACb;AAAA,MACA,WAAW,IAAI,YAAY;AAAA,MAC3B,UAAU,EAAE;AAAA,MACZ,MAAM,EAAE;AAAA,MACR,MAAM,EAAE;AAAA,MACR,QAAQ,EAAE;AAAA,MACV,kBAAkB,EAAE;AAAA,IACtB,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,SAAS,GAAG;AACtB,UAAM,WAAW,YAAY,GAAG,QAAQ,KAAK,IAAI,CAAC;AAAA,GAAM,OAAO;AAAA,EACjE;AACF;AAEA,eAAsB,cACpB,SAC8B;AAC9B,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP,gBAAgB;AAAA,IAChB,SAAS;AAAA,IACT;AAAA,IACA,gBAAgB;AAAA,IAChB,iBAAiB;AAAA,EACnB,IAAI;AAEJ,MAAI,CAAC,OAAO,SAAS,WAAW,KAAK,eAAe,GAAG;AACrD,UAAM,IAAI,MAAM,8CAA8C;AAAA,EAChE;AAEA,QAAM,OAAO,QAAQ,QAAQ,MAAM,oBAAI,KAAK,IAAI;AAChD,QAAM,QAAQ,IAAI,QAAQ;AAG1B,QAAM,CAAC,aAAa,cAAc,gBAAgB,IAAI,MAAM,QAAQ,IAAI;AAAA,IACtE,SAAS,QAAQ,QAAQ,gBAAgB,IAAI,QAAQ,QAAQ,CAAC,CAAC;AAAA,IAC/D,QAAQ,oBAAoB;AAAA,IAC5B,SAAS,QAAQ,QAAQ,qBAAqB,IAAI,QAAQ,QAAQ,CAAC,CAAC;AAAA,EACtE,CAAC;AAED,QAAM,cAAuF,CAAC;AAE9F,MAAI,SAAS,OAAO;AAClB,eAAW,KAAK,aAAa;AAC3B,kBAAY,KAAK,EAAE,QAAQ,GAAG,cAAc,MAAM,CAAC;AAAA,IACrD;AACA,eAAW,KAAK,kBAAkB;AAChC,kBAAY,KAAK,EAAE,QAAQ,GAAG,cAAc,UAAU,CAAC;AAAA,IACzD;AAAA,EACF;AACA,aAAW,KAAK,cAAc;AAC5B,gBAAY,KAAK,EAAE,QAAQ,GAAG,cAAc,OAAO,CAAC;AAAA,EACtD;AAGA,QAAM,aAA+B,CAAC;AACtC,QAAM,wBAAwB,oBAAI,IAAwB;AAC1D,aAAW,EAAE,QAAQ,aAAa,KAAK,aAAa;AAClD,UAAM,KAAK,iBAAiB,MAAM;AAClC,QAAI,GAAG,WAAW,EAAG;AAErB,UAAM,OAAO,KAAK,MAAM,EAAE;AAC1B,QAAI,CAAC,OAAO,SAAS,IAAI,EAAG;AAE5B,UAAM,QAAQ,QAAQ;AACtB,QAAI,QAAQ,YAAa;AAEzB,UAAM,KAAK,OAAO;AAClB,UAAM,SACJ,OAAO,GAAG,WAAW,WAAY,GAAG,SAAoB;AAE1D,QAAI,iBAAiB,WAAW,YAAa;AAE7C,eAAW,KAAK;AAAA,MACd,IAAI,OAAO,YAAY;AAAA,MACvB,MAAM,OAAO;AAAA,MACb,MAAM;AAAA,MACN;AAAA,MACA,kBAAkB;AAAA,MAClB;AAAA,IACF,CAAC;AACD,0BAAsB,IAAI,OAAO,YAAY,IAAI,MAAM;AAAA,EACzD;AAEA,MAAI,QAAQ;AACV,WAAO;AAAA,MACL,QAAQ;AAAA,MACR;AAAA,MACA;AAAA,MACA;AAAA,MACA,aAAa;AAAA,MACb,oBAAoB;AAAA,MACpB,YAAY;AAAA,MACZ,QAAQ,CAAC;AAAA,IACX;AAAA,EACF;AAGA,QAAM,SAA6D,CAAC;AACpE,MAAI;AACF,UAAM,cAAc,SAAS,YAAY,KAAK,qBAAqB;AAAA,EACrE,SAAS,UAAU;AACjB,WAAO,KAAK;AAAA,MACV,IAAI;AAAA,MACJ,MAAM,KAAK,KAAK,QAAQ,KAAK,SAAS,sBAAsB,mBAAmB;AAAA,MAC/E,OAAO,oBAAoB,QAAQ,SAAS,UAAU,OAAO,QAAQ;AAAA,IACvE,CAAC;AAAA,EACH;AAEA,QAAM,iBAAmC,CAAC;AAC1C,QAAM,gBAAkC,CAAC;AACzC,QAAM,sBAAsB,oBAAI,IAAY;AAC5C,QAAM,4BAA4B,CAAC,cAA8B;AAC/D,QAAI,UAAU,SAAS,QAAQ;AAC7B,0BAAoB,IAAI,cAAc;AAAA,IACxC,WAAW,UAAU,SAAS,OAAO;AACnC,0BAAoB,IAAI,aAAa;AAAA,IACvC;AAAA,EACF;AAEA,aAAW,aAAa,YAAY;AAClC,QAAI;AACF,YAAM,OAAO,UAAU,IAAI;AAC3B,qBAAe,KAAK,SAAS;AAC7B,gCAA0B,SAAS;AAAA,IACrC,SAAS,KAAK;AACZ,YAAM,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAE/D,UAAI,aAAa,KAAK,QAAQ,GAAG;AAC/B,sBAAc,KAAK,SAAS;AAC5B,kCAA0B,SAAS;AAAA,MACrC,OAAO;AACL,eAAO,KAAK,EAAE,IAAI,UAAU,IAAI,MAAM,UAAU,MAAM,OAAO,QAAQ,CAAC;AAAA,MACxE;AAAA,IACF;AAAA,EACF;AAEA,QAAM,wBAAwB,OAC5B,kBACA,UACG;AACH,QAAI;AACF,YAAM,cAAc,SAAS,kBAAkB,KAAK,KAAK;AAAA,IAC3D,SAAS,UAAU;AACjB,aAAO,KAAK;AAAA,QACV,IAAI;AAAA,QACJ,MAAM,KAAK,KAAK,QAAQ,KAAK,SAAS,sBAAsB,mBAAmB;AAAA,QAC/E,OAAO,oBAAoB,QAAQ,SAAS,UAAU,OAAO,QAAQ;AAAA,MACvE,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,sBAAsB,gBAAgB,mBAAmB;AAC/D,QAAM,sBAAsB,eAAe,sBAAsB;AAEjE,QAAM,iBAAiB,CAAC,GAAG,gBAAgB,GAAG,aAAa;AAE3D,QAAM,uBACJ,QAGA;AACF,MAAI,OAAO,yBAAyB,YAAY;AAC9C,yBAAqB,KAAK,SAAS,IAAI,IAAI,eAAe,IAAI,CAAC,cAAc,UAAU,IAAI,CAAC,CAAC;AAAA,EAC/F,WAAW,OAAQ,QAAyE,qCAAqC,YAAY;AAC3I,IAAC,QAAwE,iCAAiC;AAAA,EAC5G;AAGA,MAAI,KAAK;AACP,eAAW,cAAc,qBAAqB;AAC5C,UAAI;AACF,YAAI,OAAO,IAAI,2BAA2B,YAAY;AACpD,gBAAM,IAAI,uBAAuB,YAAY,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,QACzE,OAAO;AACL,gBAAM,IAAI,iBAAiB,YAAY,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,QACnE;AAAA,MACF,SAAS,UAAU;AAEjB,eAAO,KAAK;AAAA,UACV,IAAI;AAAA,UACJ,MAAM;AAAA,UACN,OAAO,oBAAoB,QAAQ,SAAS,UAAU,OAAO,QAAQ;AAAA,QACvE,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAEA,QAAM,qBAAqB,eACxB,IAAI,CAAC,cAAc,sBAAsB,IAAI,UAAU,EAAE,CAAC,EAC1D,OAAO,CAAC,WAAiC,QAAQ,YAAY,aAAa,MAAM;AACnF,MAAI,mBAAmB,SAAS,GAAG;AACjC,UAAM,mBACJ,QAGA;AACF,QAAI,OAAO,qBAAqB,YAAY;AAC1C,UAAI;AACF,cAAM,iBAAiB,KAAK,SAAS,kBAAkB;AACvD,cAAM,QAAQ,uBAAuB;AAAA,MACvC,SAAS,SAAS;AAChB,eAAO,KAAK;AAAA,UACV,IAAI;AAAA,UACJ,MAAM,KAAK,KAAK,QAAQ,KAAK,SAAS,iBAAiB;AAAA,UACvD,OAAO,mBAAmB,QAAQ,QAAQ,UAAU,OAAO,OAAO;AAAA,QACpE,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AAAA,IACL,QAAQ;AAAA,IACR;AAAA,IACA;AAAA,IACA;AAAA,IACA,aAAa,eAAe;AAAA,IAC5B,oBAAoB,cAAc;AAAA,IAClC,YAAY,OAAO;AAAA,IACnB;AAAA,EACF;AACF;","names":[]}