memory-braid 0.3.7 → 0.4.0

package/src/mem0-client.ts

@@ -112,6 +112,64 @@ function asNonEmptyString(value: unknown): string | undefined {
   return trimmed ? trimmed : undefined;
 }
 
+function cloneConfigValue(value: unknown): unknown {
+  if (Array.isArray(value)) {
+    return value.map((entry) => cloneConfigValue(entry));
+  }
+  if (isObjectLike(value)) {
+    return cloneConfigRecord(value);
+  }
+  return value;
+}
+
+function cloneConfigRecord(value: Record<string, unknown>): Record<string, unknown> {
+  const out: Record<string, unknown> = {};
+  for (const [key, entry] of Object.entries(value)) {
+    out[key] = cloneConfigValue(entry);
+  }
+  return out;
+}
+
+function shouldReplaceProviderSection(
+  baseValue: Record<string, unknown>,
+  overrideValue: Record<string, unknown>,
+): boolean {
+  const baseProvider = asNonEmptyString(baseValue.provider)?.toLowerCase();
+  const overrideProvider = asNonEmptyString(overrideValue.provider)?.toLowerCase();
+  return Boolean(baseProvider && overrideProvider && baseProvider !== overrideProvider);
+}
+
+function deepMergeConfigRecords(
+  baseValue: Record<string, unknown>,
+  overrideValue: Record<string, unknown>,
+): Record<string, unknown> {
+  const merged = cloneConfigRecord(baseValue);
+  for (const [key, overrideEntry] of Object.entries(overrideValue)) {
+    if (typeof overrideEntry === "undefined") {
+      continue;
+    }
+    const currentBase = merged[key];
+    if (isObjectLike(currentBase) && isObjectLike(overrideEntry)) {
+      merged[key] = shouldReplaceProviderSection(currentBase, overrideEntry)
+        ? cloneConfigRecord(overrideEntry)
+        : deepMergeConfigRecords(currentBase, overrideEntry);
+      continue;
+    }
+    merged[key] = cloneConfigValue(overrideEntry);
+  }
+  return merged;
+}
+
+export function mergeOssConfigWithDefaults(
+  defaults: Record<string, unknown>,
+  override: Record<string, unknown>,
+): Record<string, unknown> {
+  if (!isLikelyOssConfig(override)) {
+    return cloneConfigRecord(defaults);
+  }
+  return deepMergeConfigRecords(defaults, override);
+}
+
 function asOssMemoryCtor(value: unknown): OssMemoryCtor | undefined {
   if (typeof value !== "function") {
     return undefined;
@@ -424,12 +482,11 @@ export class Mem0Adapter {
         );
       }
 
-      const providedConfig = this.cfg.mem0.ossConfig;
+      const providedConfig = asRecord(this.cfg.mem0.ossConfig);
       const hasCustomConfig = isLikelyOssConfig(providedConfig);
-      const baseConfig = hasCustomConfig
-        ? { ...providedConfig }
-        : buildDefaultOssConfig(this.cfg, this.stateDir);
-      const configToUse = applyOssStorageDefaults(baseConfig, this.stateDir);
+      const defaultConfig = buildDefaultOssConfig(this.cfg, this.stateDir);
+      const mergedConfig = mergeOssConfigWithDefaults(defaultConfig, providedConfig);
+      const configToUse = applyOssStorageDefaults(mergedConfig, this.stateDir);
       await ensureSqliteParentDirs(configToUse);
 
       this.ossClient = new Memory(configToUse);

package/src/state.ts

@@ -1,27 +1,43 @@
 import fs from "node:fs/promises";
 import path from "node:path";
-import type { BootstrapState, CaptureDedupeState, ReconcileState } from "./types.js";
+import type {
+  CaptureDedupeState,
+  LifecycleState,
+  PluginStatsState,
+} from "./types.js";
 
-const DEFAULT_BOOTSTRAP: BootstrapState = {
+const DEFAULT_CAPTURE_DEDUPE: CaptureDedupeState = {
   version: 1,
-  completed: false,
+  seen: {},
 };
 
-const DEFAULT_RECONCILE: ReconcileState = {
+const DEFAULT_LIFECYCLE: LifecycleState = {
   version: 1,
   entries: {},
 };
 
-const DEFAULT_CAPTURE_DEDUPE: CaptureDedupeState = {
+const DEFAULT_STATS: PluginStatsState = {
   version: 1,
-  seen: {},
+  capture: {
+    runs: 0,
+    runsWithCandidates: 0,
+    runsNoCandidates: 0,
+    candidates: 0,
+    dedupeSkipped: 0,
+    persisted: 0,
+    mem0AddAttempts: 0,
+    mem0AddWithId: 0,
+    mem0AddWithoutId: 0,
+    entityAnnotatedCandidates: 0,
+    totalEntitiesAttached: 0,
+  },
 };
 
 export type StatePaths = {
   rootDir: string;
-  bootstrapFile: string;
-  reconcileFile: string;
   captureDedupeFile: string;
+  lifecycleFile: string;
+  statsFile: string;
   stateLockFile: string;
 };
 
@@ -29,9 +45,9 @@ export function createStatePaths(stateDir: string): StatePaths {
   const rootDir = path.join(stateDir, "memory-braid");
   return {
     rootDir,
-    bootstrapFile: path.join(rootDir, "bootstrap-checkpoint.v1.json"),
-    reconcileFile: path.join(rootDir, "reconcile-state.v1.json"),
     captureDedupeFile: path.join(rootDir, "capture-dedupe.v1.json"),
+    lifecycleFile: path.join(rootDir, "lifecycle.v1.json"),
+    statsFile: path.join(rootDir, "stats.v1.json"),
     stateLockFile: path.join(rootDir, "state.v1.lock"),
   };
 }
@@ -57,41 +73,52 @@ async function writeJsonFile(filePath: string, value: unknown): Promise<void> {
   await fs.rename(tmpPath, filePath);
 }
 
-export async function readBootstrapState(paths: StatePaths): Promise<BootstrapState> {
-  const value = await readJsonFile(paths.bootstrapFile, DEFAULT_BOOTSTRAP);
-  return { ...DEFAULT_BOOTSTRAP, ...value };
+export async function readCaptureDedupeState(paths: StatePaths): Promise<CaptureDedupeState> {
+  const value = await readJsonFile(paths.captureDedupeFile, DEFAULT_CAPTURE_DEDUPE);
+  return {
+    version: 1,
+    seen: value.seen ?? {},
+  };
 }
 
-export async function writeBootstrapState(paths: StatePaths, state: BootstrapState): Promise<void> {
-  await writeJsonFile(paths.bootstrapFile, state);
+export async function writeCaptureDedupeState(
+  paths: StatePaths,
+  state: CaptureDedupeState,
+): Promise<void> {
+  await writeJsonFile(paths.captureDedupeFile, state);
 }
 
-export async function readReconcileState(paths: StatePaths): Promise<ReconcileState> {
-  const value = await readJsonFile(paths.reconcileFile, DEFAULT_RECONCILE);
+export async function readLifecycleState(paths: StatePaths): Promise<LifecycleState> {
+  const value = await readJsonFile(paths.lifecycleFile, DEFAULT_LIFECYCLE);
   return {
     version: 1,
     entries: value.entries ?? {},
-    lastRunAt: value.lastRunAt,
+    lastCleanupAt: value.lastCleanupAt,
+    lastCleanupReason: value.lastCleanupReason,
+    lastCleanupScanned: value.lastCleanupScanned,
+    lastCleanupExpired: value.lastCleanupExpired,
+    lastCleanupDeleted: value.lastCleanupDeleted,
+    lastCleanupFailed: value.lastCleanupFailed,
   };
 }
 
-export async function writeReconcileState(paths: StatePaths, state: ReconcileState): Promise<void> {
-  await writeJsonFile(paths.reconcileFile, state);
+export async function writeLifecycleState(paths: StatePaths, state: LifecycleState): Promise<void> {
+  await writeJsonFile(paths.lifecycleFile, state);
 }
 
-export async function readCaptureDedupeState(paths: StatePaths): Promise<CaptureDedupeState> {
-  const value = await readJsonFile(paths.captureDedupeFile, DEFAULT_CAPTURE_DEDUPE);
+export async function readStatsState(paths: StatePaths): Promise<PluginStatsState> {
+  const value = await readJsonFile(paths.statsFile, DEFAULT_STATS);
   return {
     version: 1,
-    seen: value.seen ?? {},
+    capture: {
+      ...DEFAULT_STATS.capture,
+      ...(value.capture ?? {}),
+    },
   };
 }
 
-export async function writeCaptureDedupeState(
-  paths: StatePaths,
-  state: CaptureDedupeState,
-): Promise<void> {
-  await writeJsonFile(paths.captureDedupeFile, state);
+export async function writeStatsState(paths: StatePaths, state: PluginStatsState): Promise<void> {
+  await writeJsonFile(paths.statsFile, state);
 }
 
 export async function withStateLock<T>(

package/src/types.ts

@@ -1,22 +1,11 @@
 export type MemoryBraidSource = "local" | "mem0";
 
-export type ManagedSourceType = "markdown" | "session";
-
-export type PersistedSourceType = ManagedSourceType | "capture";
-
 export type ScopeKey = {
   workspaceHash: string;
   agentId: string;
   sessionKey?: string;
 };
 
-export type TargetWorkspace = {
-  workspaceDir: string;
-  stateDir: string;
-  agentId: string;
-  workspaceHash: string;
-};
-
 export type MemoryBraidResult = {
   id?: string;
   source: MemoryBraidSource;
@@ -31,61 +20,54 @@ export type MemoryBraidResult = {
   contentHash?: string;
 };
 
-export type ManifestChunk = {
-  chunkKey: string;
-  contentHash: string;
-  sourceType: ManagedSourceType;
-  text: string;
-  path: string;
-  workspaceHash: string;
-  agentId: string;
-  updatedAt: number;
+export type CaptureDedupeState = {
+  version: 1;
+  seen: Record<string, number>;
 };
 
-export type IndexedEntry = {
-  chunkKey: string;
-  id?: string;
+export type LifecycleEntry = {
+  memoryId: string;
   contentHash: string;
-  sourceType: PersistedSourceType;
-  path?: string;
   workspaceHash: string;
   agentId: string;
+  sessionKey?: string;
+  category?: string;
+  createdAt: number;
+  lastCapturedAt: number;
+  lastRecalledAt?: number;
+  recallCount: number;
   updatedAt: number;
-  missingCount?: number;
 };
 
-export type ReconcileState = {
+export type LifecycleState = {
   version: 1;
-  entries: Record<string, IndexedEntry>;
-  lastRunAt?: string;
+  entries: Record<string, LifecycleEntry>;
+  lastCleanupAt?: string;
+  lastCleanupReason?: "startup" | "interval" | "command";
+  lastCleanupScanned?: number;
+  lastCleanupExpired?: number;
+  lastCleanupDeleted?: number;
+  lastCleanupFailed?: number;
 };
 
-export type BootstrapState = {
-  version: 1;
-  completed: boolean;
-  startedAt?: string;
-  completedAt?: string;
-  lastError?: string;
-  summary?: {
-    reason: string;
-    total: number;
-    upserted: number;
-    deleted: number;
-    unchanged: number;
-  };
+export type CaptureStats = {
+  runs: number;
+  runsWithCandidates: number;
+  runsNoCandidates: number;
+  candidates: number;
+  dedupeSkipped: number;
+  persisted: number;
+  mem0AddAttempts: number;
+  mem0AddWithId: number;
+  mem0AddWithoutId: number;
+  entityAnnotatedCandidates: number;
+  totalEntitiesAttached: number;
+  lastRunAt?: string;
 };
 
-export type CaptureDedupeState = {
+export type PluginStatsState = {
   version: 1;
-  seen: Record<string, number>;
-};
-
-export type ReconcileSummary = {
-  reason: string;
-  total: number;
-  upserted: number;
-  deleted: number;
-  unchanged: number;
+  capture: CaptureStats;
 };
 
 export type ExtractedCandidate = {

package/src/bootstrap.ts

@@ -1,82 +0,0 @@
-import type { MemoryBraidConfig } from "./config.js";
-import { MemoryBraidLogger } from "./logger.js";
-import { Mem0Adapter } from "./mem0-client.js";
-import {
-  readBootstrapState,
-  type StatePaths,
-  writeBootstrapState,
-} from "./state.js";
-import type { TargetWorkspace } from "./types.js";
-import { runReconcileOnce } from "./reconcile.js";
-
-export async function runBootstrapIfNeeded(params: {
-  cfg: MemoryBraidConfig;
-  mem0: Mem0Adapter;
-  statePaths: StatePaths;
-  log: MemoryBraidLogger;
-  targets: TargetWorkspace[];
-  runId?: string;
-}): Promise<{ started: boolean; completed: boolean }> {
-  const runId = params.runId ?? params.log.newRunId();
-
-  if (!params.cfg.bootstrap.enabled) {
-    return { started: false, completed: false };
-  }
-
-  const existing = await readBootstrapState(params.statePaths);
-  if (existing.completed) {
-    return { started: false, completed: true };
-  }
-
-  params.log.debug("memory_braid.bootstrap.begin", {
-    runId,
-    targets: params.targets.length,
-  }, true);
-
-  await writeBootstrapState(params.statePaths, {
-    version: 1,
-    completed: false,
-    startedAt: new Date().toISOString(),
-    lastError: undefined,
-  });
-
-  try {
-    const summary = await runReconcileOnce({
-      cfg: params.cfg,
-      mem0: params.mem0,
-      statePaths: params.statePaths,
-      log: params.log,
-      targets: params.targets,
-      reason: "bootstrap",
-      runId,
-      deleteStale: false,
-    });
-
-    await writeBootstrapState(params.statePaths, {
-      version: 1,
-      completed: true,
-      startedAt: existing.startedAt ?? new Date().toISOString(),
-      completedAt: new Date().toISOString(),
-      summary,
-    });
-
-    params.log.debug("memory_braid.bootstrap.complete", {
-      runId,
-      ...summary,
-    }, true);
-    return { started: true, completed: true };
-  } catch (err) {
-    const error = err instanceof Error ? err.message : String(err);
-    await writeBootstrapState(params.statePaths, {
-      version: 1,
-      completed: false,
-      startedAt: existing.startedAt ?? new Date().toISOString(),
-      lastError: error,
-    });
-    params.log.warn("memory_braid.bootstrap.error", {
-      runId,
-      error,
-    });
-    return { started: true, completed: false };
-  }
-}

package/src/reconcile.ts

@@ -1,278 +0,0 @@
-import fs from "node:fs/promises";
-import path from "node:path";
-import { buildMarkdownChunks, buildSessionChunks, sha256 } from "./chunking.js";
-import type { MemoryBraidConfig } from "./config.js";
-import { MemoryBraidLogger } from "./logger.js";
-import { Mem0Adapter } from "./mem0-client.js";
-import {
-  readReconcileState,
-  type StatePaths,
-  withStateLock,
-  writeReconcileState,
-} from "./state.js";
-import type { IndexedEntry, ManifestChunk, ReconcileSummary, TargetWorkspace } from "./types.js";
-
-type OpenClawConfigLike = {
-  agents?: {
-    defaults?: {
-      workspace?: string;
-    };
-    list?: Array<{
-      id?: string;
-      workspace?: string;
-      default?: boolean;
-    }>;
-  };
-};
-
-function normalizeAgentId(value: string | undefined): string {
-  const trimmed = (value ?? "main").trim().toLowerCase();
-  return trimmed || "main";
-}
-
-function resolveWorkspacePath(input: string | undefined, fallback?: string): string | undefined {
-  const value = (input ?? "").trim() || (fallback ?? "").trim();
-  if (!value) {
-    return undefined;
-  }
-  return path.resolve(value);
-}
-
-async function workspaceHashForDir(workspaceDir: string): Promise<string> {
-  try {
-    const real = await fs.realpath(workspaceDir);
-    return sha256(real.toLowerCase());
-  } catch {
-    return sha256(path.resolve(workspaceDir).toLowerCase());
-  }
-}
-
-export async function resolveTargets(params: {
-  config?: OpenClawConfigLike;
-  stateDir: string;
-  fallbackWorkspaceDir?: string;
-}): Promise<TargetWorkspace[]> {
-  const targets: TargetWorkspace[] = [];
-  const seen = new Set<string>();
-
-  const fallbackWorkspace = resolveWorkspacePath(
-    params.config?.agents?.defaults?.workspace,
-    params.fallbackWorkspaceDir,
-  );
-  if (fallbackWorkspace) {
-    const agentId = "main";
-    const key = `${agentId}|${fallbackWorkspace}`;
-    seen.add(key);
-    targets.push({
-      workspaceDir: fallbackWorkspace,
-      stateDir: params.stateDir,
-      agentId,
-      workspaceHash: await workspaceHashForDir(fallbackWorkspace),
-    });
-  }
-
-  for (const entry of params.config?.agents?.list ?? []) {
-    const workspace = resolveWorkspacePath(entry.workspace, fallbackWorkspace);
-    if (!workspace) {
-      continue;
-    }
-    const agentId = normalizeAgentId(entry.id);
-    const key = `${agentId}|${workspace}`;
-    if (seen.has(key)) {
-      continue;
-    }
-    seen.add(key);
-    targets.push({
-      workspaceDir: workspace,
-      stateDir: params.stateDir,
-      agentId,
-      workspaceHash: await workspaceHashForDir(workspace),
-    });
-  }
-
-  return targets;
-}
-
-export async function buildManagedManifest(params: {
-  targets: TargetWorkspace[];
-  cfg: MemoryBraidConfig;
-}): Promise<ManifestChunk[]> {
-  const out: ManifestChunk[] = [];
-  for (const target of params.targets) {
-    if (params.cfg.bootstrap.includeMarkdown) {
-      out.push(...(await buildMarkdownChunks(target)));
-    }
-    if (params.cfg.bootstrap.includeSessions) {
-      out.push(...(await buildSessionChunks(target, params.cfg.bootstrap.sessionLookbackDays)));
-    }
-  }
-  return out;
-}
-
-export async function runReconcileOnce(params: {
-  cfg: MemoryBraidConfig;
-  mem0: Mem0Adapter;
-  statePaths: StatePaths;
-  log: MemoryBraidLogger;
-  targets: TargetWorkspace[];
-  reason: string;
-  runId?: string;
-  deleteStale?: boolean;
-}): Promise<ReconcileSummary> {
-  const runId = params.runId ?? params.log.newRunId();
-  const startedAt = Date.now();
-
-  return withStateLock(params.statePaths.stateLockFile, async () => {
-    params.log.debug("memory_braid.reconcile.begin", {
-      runId,
-      reason: params.reason,
-      targets: params.targets.length,
-    }, true);
-
-    const state = await readReconcileState(params.statePaths);
-    const manifest = await buildManagedManifest({
-      targets: params.targets,
-      cfg: params.cfg,
-    });
-
-    const byKey = new Map<string, ManifestChunk>();
-    for (const chunk of manifest) {
-      byKey.set(chunk.chunkKey, chunk);
-    }
-
-    let upserted = 0;
-    let deleted = 0;
-    let unchanged = 0;
-    const now = Date.now();
-
-    // Mark managed entries that are no longer present.
-    for (const [chunkKey, entry] of Object.entries(state.entries)) {
-      if (entry.sourceType === "capture") {
-        continue;
-      }
-      if (byKey.has(chunkKey)) {
-        continue;
-      }
-      const missingCount = (entry.missingCount ?? 0) + 1;
-      const shouldDelete =
-        (params.deleteStale ?? params.cfg.reconcile.deleteStale) && missingCount >= 2;
-      if (!shouldDelete) {
-        state.entries[chunkKey] = {
-          ...entry,
-          missingCount,
-          updatedAt: now,
-        };
-        continue;
-      }
-
-      const deletedRemote = await params.mem0.deleteMemory({
-        memoryId: entry.id,
-        scope: {
-          workspaceHash: entry.workspaceHash,
-          agentId: entry.agentId,
-        },
-        runId,
-      });
-      if (deletedRemote || !entry.id) {
-        delete state.entries[chunkKey];
-        deleted += 1;
-      } else {
-        state.entries[chunkKey] = {
-          ...entry,
-          missingCount,
-          updatedAt: now,
-        };
-      }
-    }
-
-    const allChunks = Array.from(byKey.values());
-    const batchSize = Math.max(1, params.cfg.reconcile.batchSize);
-    for (let offset = 0; offset < allChunks.length; offset += batchSize) {
-      const batch = allChunks.slice(offset, offset + batchSize);
-      for (const chunk of batch) {
-        const existing = state.entries[chunk.chunkKey];
-        if (existing && existing.contentHash === chunk.contentHash && existing.sourceType !== "capture") {
-          unchanged += 1;
-          state.entries[chunk.chunkKey] = {
-            ...existing,
-            missingCount: 0,
-            updatedAt: now,
-          };
-          continue;
-        }
-
-        if (existing?.id && existing.sourceType !== "capture") {
-          await params.mem0.deleteMemory({
-            memoryId: existing.id,
-            scope: {
-              workspaceHash: existing.workspaceHash,
-              agentId: existing.agentId,
-            },
-            runId,
-          });
-        }
-
-        const metadata = {
-          sourceType: chunk.sourceType,
-          path: chunk.path,
-          workspaceHash: chunk.workspaceHash,
-          agentId: chunk.agentId,
-          chunkKey: chunk.chunkKey,
-          contentHash: chunk.contentHash,
-          indexedAt: new Date(now).toISOString(),
-        };
-
-        const addResult = await params.mem0.addMemory({
-          text: chunk.text,
-          scope: {
-            workspaceHash: chunk.workspaceHash,
-            agentId: chunk.agentId,
-          },
-          metadata,
-          runId,
-        });
-
-        const next: IndexedEntry = {
-          chunkKey: chunk.chunkKey,
-          id: addResult.id ?? existing?.id,
-          contentHash: chunk.contentHash,
-          sourceType: chunk.sourceType,
-          path: chunk.path,
-          workspaceHash: chunk.workspaceHash,
-          agentId: chunk.agentId,
-          updatedAt: now,
-          missingCount: 0,
-        };
-        state.entries[chunk.chunkKey] = next;
-        upserted += 1;
-      }
-
-      params.log.debug("memory_braid.reconcile.progress", {
-        runId,
-        reason: params.reason,
-        processed: Math.min(offset + batch.length, allChunks.length),
-        total: allChunks.length,
-      });
-    }
-
-    const summary: ReconcileSummary = {
-      reason: params.reason,
-      total: allChunks.length,
-      upserted,
-      deleted,
-      unchanged,
-    };
-
-    state.lastRunAt = new Date(now).toISOString();
-    await writeReconcileState(params.statePaths, state);
-
-    params.log.debug("memory_braid.reconcile.complete", {
-      runId,
-      reason: params.reason,
-      ...summary,
-      durMs: Date.now() - startedAt,
-    }, true);
-
-    return summary;
-  });
-}