memory-braid 0.2.0

package/src/reconcile.ts

@@ -0,0 +1,278 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { buildMarkdownChunks, buildSessionChunks, sha256 } from "./chunking.js";
+import type { MemoryBraidConfig } from "./config.js";
+import { MemoryBraidLogger } from "./logger.js";
+import { Mem0Adapter } from "./mem0-client.js";
+import {
+  readReconcileState,
+  type StatePaths,
+  withStateLock,
+  writeReconcileState,
+} from "./state.js";
+import type { IndexedEntry, ManifestChunk, ReconcileSummary, TargetWorkspace } from "./types.js";
+
+type OpenClawConfigLike = {
+  agents?: {
+    defaults?: {
+      workspace?: string;
+    };
+    list?: Array<{
+      id?: string;
+      workspace?: string;
+      default?: boolean;
+    }>;
+  };
+};
+
+function normalizeAgentId(value: string | undefined): string {
+  const trimmed = (value ?? "main").trim().toLowerCase();
+  return trimmed || "main";
+}
+
+function resolveWorkspacePath(input: string | undefined, fallback?: string): string | undefined {
+  const value = (input ?? "").trim() || (fallback ?? "").trim();
+  if (!value) {
+    return undefined;
+  }
+  return path.resolve(value);
+}
+
+async function workspaceHashForDir(workspaceDir: string): Promise<string> {
+  try {
+    const real = await fs.realpath(workspaceDir);
+    return sha256(real.toLowerCase());
+  } catch {
+    return sha256(path.resolve(workspaceDir).toLowerCase());
+  }
+}
+
+export async function resolveTargets(params: {
+  config?: OpenClawConfigLike;
+  stateDir: string;
+  fallbackWorkspaceDir?: string;
+}): Promise<TargetWorkspace[]> {
+  const targets: TargetWorkspace[] = [];
+  const seen = new Set<string>();
+
+  const fallbackWorkspace = resolveWorkspacePath(
+    params.config?.agents?.defaults?.workspace,
+    params.fallbackWorkspaceDir,
+  );
+  if (fallbackWorkspace) {
+    const agentId = "main";
+    const key = `${agentId}|${fallbackWorkspace}`;
+    seen.add(key);
+    targets.push({
+      workspaceDir: fallbackWorkspace,
+      stateDir: params.stateDir,
+      agentId,
+      workspaceHash: await workspaceHashForDir(fallbackWorkspace),
+    });
+  }
+
+  for (const entry of params.config?.agents?.list ?? []) {
+    const workspace = resolveWorkspacePath(entry.workspace, fallbackWorkspace);
+    if (!workspace) {
+      continue;
+    }
+    const agentId = normalizeAgentId(entry.id);
+    const key = `${agentId}|${workspace}`;
+    if (seen.has(key)) {
+      continue;
+    }
+    seen.add(key);
+    targets.push({
+      workspaceDir: workspace,
+      stateDir: params.stateDir,
+      agentId,
+      workspaceHash: await workspaceHashForDir(workspace),
+    });
+  }
+
+  return targets;
+}
+
+export async function buildManagedManifest(params: {
+  targets: TargetWorkspace[];
+  cfg: MemoryBraidConfig;
+}): Promise<ManifestChunk[]> {
+  const out: ManifestChunk[] = [];
+  for (const target of params.targets) {
+    if (params.cfg.bootstrap.includeMarkdown) {
+      out.push(...(await buildMarkdownChunks(target)));
+    }
+    if (params.cfg.bootstrap.includeSessions) {
+      out.push(...(await buildSessionChunks(target, params.cfg.bootstrap.sessionLookbackDays)));
+    }
+  }
+  return out;
+}
+
+export async function runReconcileOnce(params: {
+  cfg: MemoryBraidConfig;
+  mem0: Mem0Adapter;
+  statePaths: StatePaths;
+  log: MemoryBraidLogger;
+  targets: TargetWorkspace[];
+  reason: string;
+  runId?: string;
+  deleteStale?: boolean;
+}): Promise<ReconcileSummary> {
+  const runId = params.runId ?? params.log.newRunId();
+  const startedAt = Date.now();
+
+  return withStateLock(params.statePaths.stateLockFile, async () => {
+    params.log.debug("memory_braid.reconcile.begin", {
+      runId,
+      reason: params.reason,
+      targets: params.targets.length,
+    }, true);
+
+    const state = await readReconcileState(params.statePaths);
+    const manifest = await buildManagedManifest({
+      targets: params.targets,
+      cfg: params.cfg,
+    });
+
+    const byKey = new Map<string, ManifestChunk>();
+    for (const chunk of manifest) {
+      byKey.set(chunk.chunkKey, chunk);
+    }
+
+    let upserted = 0;
+    let deleted = 0;
+    let unchanged = 0;
+    const now = Date.now();
+
+    // Mark managed entries that are no longer present.
+    for (const [chunkKey, entry] of Object.entries(state.entries)) {
+      if (entry.sourceType === "capture") {
+        continue;
+      }
+      if (byKey.has(chunkKey)) {
+        continue;
+      }
+      const missingCount = (entry.missingCount ?? 0) + 1;
+      const shouldDelete =
+        (params.deleteStale ?? params.cfg.reconcile.deleteStale) && missingCount >= 2;
+      if (!shouldDelete) {
+        state.entries[chunkKey] = {
+          ...entry,
+          missingCount,
+          updatedAt: now,
+        };
+        continue;
+      }
+
+      const deletedRemote = await params.mem0.deleteMemory({
+        memoryId: entry.id,
+        scope: {
+          workspaceHash: entry.workspaceHash,
+          agentId: entry.agentId,
+        },
+        runId,
+      });
+      if (deletedRemote || !entry.id) {
+        delete state.entries[chunkKey];
+        deleted += 1;
+      } else {
+        state.entries[chunkKey] = {
+          ...entry,
+          missingCount,
+          updatedAt: now,
+        };
+      }
+    }
+
+    const allChunks = Array.from(byKey.values());
+    const batchSize = Math.max(1, params.cfg.reconcile.batchSize);
+    for (let offset = 0; offset < allChunks.length; offset += batchSize) {
+      const batch = allChunks.slice(offset, offset + batchSize);
+      for (const chunk of batch) {
+        const existing = state.entries[chunk.chunkKey];
+        if (existing && existing.contentHash === chunk.contentHash && existing.sourceType !== "capture") {
+          unchanged += 1;
+          state.entries[chunk.chunkKey] = {
+            ...existing,
+            missingCount: 0,
+            updatedAt: now,
+          };
+          continue;
+        }
+
+        if (existing?.id && existing.sourceType !== "capture") {
+          await params.mem0.deleteMemory({
+            memoryId: existing.id,
+            scope: {
+              workspaceHash: existing.workspaceHash,
+              agentId: existing.agentId,
+            },
+            runId,
+          });
+        }
+
+        const metadata = {
+          sourceType: chunk.sourceType,
+          path: chunk.path,
+          workspaceHash: chunk.workspaceHash,
+          agentId: chunk.agentId,
+          chunkKey: chunk.chunkKey,
+          contentHash: chunk.contentHash,
+          indexedAt: new Date(now).toISOString(),
+        };
+
+        const addResult = await params.mem0.addMemory({
+          text: chunk.text,
+          scope: {
+            workspaceHash: chunk.workspaceHash,
+            agentId: chunk.agentId,
+          },
+          metadata,
+          runId,
+        });
+
+        const next: IndexedEntry = {
+          chunkKey: chunk.chunkKey,
+          id: addResult.id ?? existing?.id,
+          contentHash: chunk.contentHash,
+          sourceType: chunk.sourceType,
+          path: chunk.path,
+          workspaceHash: chunk.workspaceHash,
+          agentId: chunk.agentId,
+          updatedAt: now,
+          missingCount: 0,
+        };
+        state.entries[chunk.chunkKey] = next;
+        upserted += 1;
+      }
+
+      params.log.debug("memory_braid.reconcile.progress", {
+        runId,
+        reason: params.reason,
+        processed: Math.min(offset + batch.length, allChunks.length),
+        total: allChunks.length,
+      });
+    }
+
+    const summary: ReconcileSummary = {
+      reason: params.reason,
+      total: allChunks.length,
+      upserted,
+      deleted,
+      unchanged,
+    };
+
+    state.lastRunAt = new Date(now).toISOString();
+    await writeReconcileState(params.statePaths, state);
+
+    params.log.debug("memory_braid.reconcile.complete", {
+      runId,
+      reason: params.reason,
+      ...summary,
+      durMs: Date.now() - startedAt,
+    }, true);
+
+    return summary;
+  });
+}

package/src/state.ts

@@ -0,0 +1,130 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import type { BootstrapState, CaptureDedupeState, ReconcileState } from "./types.js";
+
+const DEFAULT_BOOTSTRAP: BootstrapState = {
+  version: 1,
+  completed: false,
+};
+
+const DEFAULT_RECONCILE: ReconcileState = {
+  version: 1,
+  entries: {},
+};
+
+const DEFAULT_CAPTURE_DEDUPE: CaptureDedupeState = {
+  version: 1,
+  seen: {},
+};
+
+export type StatePaths = {
+  rootDir: string;
+  bootstrapFile: string;
+  reconcileFile: string;
+  captureDedupeFile: string;
+  stateLockFile: string;
+};
+
+export function createStatePaths(stateDir: string): StatePaths {
+  const rootDir = path.join(stateDir, "memory-braid");
+  return {
+    rootDir,
+    bootstrapFile: path.join(rootDir, "bootstrap-checkpoint.v1.json"),
+    reconcileFile: path.join(rootDir, "reconcile-state.v1.json"),
+    captureDedupeFile: path.join(rootDir, "capture-dedupe.v1.json"),
+    stateLockFile: path.join(rootDir, "state.v1.lock"),
+  };
+}
+
+export async function ensureStateDir(paths: StatePaths): Promise<void> {
+  await fs.mkdir(paths.rootDir, { recursive: true });
+}
+
+async function readJsonFile<T>(filePath: string, fallback: T): Promise<T> {
+  try {
+    const raw = await fs.readFile(filePath, "utf8");
+    const parsed = JSON.parse(raw) as T;
+    return parsed;
+  } catch {
+    return fallback;
+  }
+}
+
+async function writeJsonFile(filePath: string, value: unknown): Promise<void> {
+  await fs.mkdir(path.dirname(filePath), { recursive: true });
+  const tmpPath = `${filePath}.${Date.now()}.${Math.random().toString(36).slice(2)}.tmp`;
+  await fs.writeFile(tmpPath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
+  await fs.rename(tmpPath, filePath);
+}
+
+export async function readBootstrapState(paths: StatePaths): Promise<BootstrapState> {
+  const value = await readJsonFile(paths.bootstrapFile, DEFAULT_BOOTSTRAP);
+  return { ...DEFAULT_BOOTSTRAP, ...value };
+}
+
+export async function writeBootstrapState(paths: StatePaths, state: BootstrapState): Promise<void> {
+  await writeJsonFile(paths.bootstrapFile, state);
+}
+
+export async function readReconcileState(paths: StatePaths): Promise<ReconcileState> {
+  const value = await readJsonFile(paths.reconcileFile, DEFAULT_RECONCILE);
+  return {
+    version: 1,
+    entries: value.entries ?? {},
+    lastRunAt: value.lastRunAt,
+  };
+}
+
+export async function writeReconcileState(paths: StatePaths, state: ReconcileState): Promise<void> {
+  await writeJsonFile(paths.reconcileFile, state);
+}
+
+export async function readCaptureDedupeState(paths: StatePaths): Promise<CaptureDedupeState> {
+  const value = await readJsonFile(paths.captureDedupeFile, DEFAULT_CAPTURE_DEDUPE);
+  return {
+    version: 1,
+    seen: value.seen ?? {},
+  };
+}
+
+export async function writeCaptureDedupeState(
+  paths: StatePaths,
+  state: CaptureDedupeState,
+): Promise<void> {
+  await writeJsonFile(paths.captureDedupeFile, state);
+}
+
+export async function withStateLock<T>(
+  lockFilePath: string,
+  fn: () => Promise<T>,
+  options?: { retries?: number; retryDelayMs?: number },
+): Promise<T> {
+  const retries = options?.retries ?? 12;
+  const retryDelayMs = options?.retryDelayMs ?? 150;
+  await fs.mkdir(path.dirname(lockFilePath), { recursive: true });
+
+  let handle: fs.FileHandle | null = null;
+  for (let attempt = 0; attempt <= retries; attempt += 1) {
+    try {
+      handle = await fs.open(lockFilePath, "wx");
+      break;
+    } catch (err) {
+      const code = (err as { code?: string }).code;
+      if (code !== "EEXIST" || attempt >= retries) {
+        throw err;
+      }
+      await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
+    }
+  }
+
+  if (!handle) {
+    throw new Error(`Failed to acquire lock for ${lockFilePath}`);
+  }
+
+  try {
+    return await fn();
+  } finally {
+    await handle.close().catch(() => undefined);
+    await fs.unlink(lockFilePath).catch(() => undefined);
+  }
+}

package/src/types.ts

@@ -0,0 +1,96 @@
+export type MemoryBraidSource = "local" | "mem0";
+
+export type ManagedSourceType = "markdown" | "session";
+
+export type PersistedSourceType = ManagedSourceType | "capture";
+
+export type ScopeKey = {
+  workspaceHash: string;
+  agentId: string;
+  sessionKey?: string;
+};
+
+export type TargetWorkspace = {
+  workspaceDir: string;
+  stateDir: string;
+  agentId: string;
+  workspaceHash: string;
+};
+
+export type MemoryBraidResult = {
+  id?: string;
+  source: MemoryBraidSource;
+  path?: string;
+  startLine?: number;
+  endLine?: number;
+  snippet: string;
+  score: number;
+  mergedScore?: number;
+  metadata?: Record<string, unknown>;
+  chunkKey?: string;
+  contentHash?: string;
+};
+
+export type ManifestChunk = {
+  chunkKey: string;
+  contentHash: string;
+  sourceType: ManagedSourceType;
+  text: string;
+  path: string;
+  workspaceHash: string;
+  agentId: string;
+  updatedAt: number;
+};
+
+export type IndexedEntry = {
+  chunkKey: string;
+  id?: string;
+  contentHash: string;
+  sourceType: PersistedSourceType;
+  path?: string;
+  workspaceHash: string;
+  agentId: string;
+  updatedAt: number;
+  missingCount?: number;
+};
+
+export type ReconcileState = {
+  version: 1;
+  entries: Record<string, IndexedEntry>;
+  lastRunAt?: string;
+};
+
+export type BootstrapState = {
+  version: 1;
+  completed: boolean;
+  startedAt?: string;
+  completedAt?: string;
+  lastError?: string;
+  summary?: {
+    reason: string;
+    total: number;
+    upserted: number;
+    deleted: number;
+    unchanged: number;
+  };
+};
+
+export type CaptureDedupeState = {
+  version: 1;
+  seen: Record<string, number>;
+};
+
+export type ReconcileSummary = {
+  reason: string;
+  total: number;
+  upserted: number;
+  deleted: number;
+  unchanged: number;
+};
+
+export type ExtractedCandidate = {
+  text: string;
+  category: "preference" | "decision" | "fact" | "task" | "other";
+  score: number;
+  source: "heuristic" | "ml";
+};