memory-braid 0.2.0

package/src/chunking.ts

@@ -0,0 +1,280 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import type { ManagedSourceType, ManifestChunk, TargetWorkspace } from "./types.js";
+
+export function sha256(value: string): string {
+  return crypto.createHash("sha256").update(value).digest("hex");
+}
+
+export function normalizeWhitespace(value: string): string {
+  return value.replace(/\s+/g, " ").trim();
+}
+
+export function normalizeForHash(value: string): string {
+  return normalizeWhitespace(value).toLowerCase();
+}
+
+function buildChunkKey(params: {
+  workspaceHash: string;
+  agentId: string;
+  sourceType: ManagedSourceType;
+  path: string;
+  index: number;
+  text: string;
+}): string {
+  return sha256(
+    [
+      params.workspaceHash,
+      params.agentId,
+      params.sourceType,
+      params.path,
+      String(params.index),
+      normalizeForHash(params.text),
+    ].join("|"),
+  );
+}
+
+export function chunkText(value: string, chunkSize = 1200, overlap = 180): string[] {
+  const text = value.trim();
+  if (!text) {
+    return [];
+  }
+  const out: string[] = [];
+  let cursor = 0;
+  while (cursor < text.length) {
+    const end = Math.min(text.length, cursor + chunkSize);
+    out.push(text.slice(cursor, end).trim());
+    if (end >= text.length) {
+      break;
+    }
+    cursor = Math.max(cursor + 1, end - overlap);
+  }
+  return out.filter(Boolean);
+}
+
+async function walkMarkdownFiles(dir: string, out: string[]): Promise<void> {
+  const entries = await fs.readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    if (entry.isSymbolicLink()) {
+      continue;
+    }
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      await walkMarkdownFiles(full, out);
+      continue;
+    }
+    if (entry.isFile() && entry.name.toLowerCase().endsWith(".md")) {
+      out.push(full);
+    }
+  }
+}
+
+export async function listCanonicalMarkdownMemoryFiles(workspaceDir: string): Promise<string[]> {
+  const result: string[] = [];
+  const candidates = [path.join(workspaceDir, "MEMORY.md"), path.join(workspaceDir, "memory.md")];
+  for (const candidate of candidates) {
+    try {
+      const stat = await fs.lstat(candidate);
+      if (stat.isFile() && !stat.isSymbolicLink()) {
+        result.push(candidate);
+      }
+    } catch {
+      // ignore
+    }
+  }
+
+  const memoryDir = path.join(workspaceDir, "memory");
+  try {
+    const stat = await fs.lstat(memoryDir);
+    if (stat.isDirectory() && !stat.isSymbolicLink()) {
+      await walkMarkdownFiles(memoryDir, result);
+    }
+  } catch {
+    // ignore
+  }
+
+  return Array.from(new Set(result.map((filePath) => path.resolve(filePath))));
+}
+
+function normalizeSessionMessageText(content: unknown): string | null {
+  if (typeof content === "string") {
+    const normalized = normalizeWhitespace(content);
+    return normalized || null;
+  }
+  if (!Array.isArray(content)) {
+    return null;
+  }
+
+  const parts: string[] = [];
+  for (const block of content) {
+    if (!block || typeof block !== "object") {
+      continue;
+    }
+    const item = block as { type?: unknown; text?: unknown };
+    if (item.type === "text" && typeof item.text === "string") {
+      const normalized = normalizeWhitespace(item.text);
+      if (normalized) {
+        parts.push(normalized);
+      }
+    }
+  }
+  if (parts.length === 0) {
+    return null;
+  }
+  return parts.join(" ");
+}
+
+export async function listRecentSessionFiles(
+  stateDir: string,
+  agentId: string,
+  lookbackDays: number,
+): Promise<string[]> {
+  const dir = path.join(stateDir, "agents", agentId, "sessions");
+  const threshold = Date.now() - lookbackDays * 24 * 60 * 60 * 1000;
+  try {
+    const entries = await fs.readdir(dir, { withFileTypes: true });
+    const files: string[] = [];
+    for (const entry of entries) {
+      if (!entry.isFile() || !entry.name.endsWith(".jsonl")) {
+        continue;
+      }
+      const abs = path.join(dir, entry.name);
+      try {
+        const stat = await fs.stat(abs);
+        if (stat.mtimeMs >= threshold) {
+          files.push(abs);
+        }
+      } catch {
+        // ignore single file failure
+      }
+    }
+    return files;
+  } catch {
+    return [];
+  }
+}
+
+export async function buildMarkdownChunks(target: TargetWorkspace): Promise<ManifestChunk[]> {
+  const files = await listCanonicalMarkdownMemoryFiles(target.workspaceDir);
+  const chunks: ManifestChunk[] = [];
+
+  for (const filePath of files) {
+    let raw = "";
+    let statMtime = Date.now();
+    try {
+      raw = await fs.readFile(filePath, "utf8");
+      const stat = await fs.stat(filePath);
+      statMtime = stat.mtimeMs;
+    } catch {
+      continue;
+    }
+
+    const relPath = path.relative(target.workspaceDir, filePath).replace(/\\/g, "/");
+    const pieces = chunkText(raw);
+    pieces.forEach((piece, index) => {
+      chunks.push({
+        chunkKey: buildChunkKey({
+          workspaceHash: target.workspaceHash,
+          agentId: target.agentId,
+          sourceType: "markdown",
+          path: relPath,
+          index,
+          text: piece,
+        }),
+        contentHash: sha256(normalizeForHash(piece)),
+        sourceType: "markdown",
+        text: piece,
+        path: relPath,
+        workspaceHash: target.workspaceHash,
+        agentId: target.agentId,
+        updatedAt: statMtime,
+      });
+    });
+  }
+
+  return chunks;
+}
+
+export async function buildSessionChunks(
+  target: TargetWorkspace,
+  lookbackDays: number,
+): Promise<ManifestChunk[]> {
+  const files = await listRecentSessionFiles(target.stateDir, target.agentId, lookbackDays);
+  const chunks: ManifestChunk[] = [];
+
+  for (const filePath of files) {
+    let raw = "";
+    let statMtime = Date.now();
+    try {
+      raw = await fs.readFile(filePath, "utf8");
+      const stat = await fs.stat(filePath);
+      statMtime = stat.mtimeMs;
+    } catch {
+      continue;
+    }
+
+    const lines = raw.split("\n");
+    const conversationParts: string[] = [];
+    for (const line of lines) {
+      if (!line.trim()) {
+        continue;
+      }
+      let parsed: unknown;
+      try {
+        parsed = JSON.parse(line);
+      } catch {
+        continue;
+      }
+      if (!parsed || typeof parsed !== "object") {
+        continue;
+      }
+      const record = parsed as { type?: unknown; message?: unknown };
+      if (record.type !== "message") {
+        continue;
+      }
+      const message = record.message as { role?: unknown; content?: unknown } | undefined;
+      if (!message || typeof message.role !== "string") {
+        continue;
+      }
+      if (message.role !== "user" && message.role !== "assistant") {
+        continue;
+      }
+      const text = normalizeSessionMessageText(message.content);
+      if (!text) {
+        continue;
+      }
+      const roleLabel = message.role === "user" ? "User" : "Assistant";
+      conversationParts.push(`${roleLabel}: ${text}`);
+    }
+
+    if (conversationParts.length === 0) {
+      continue;
+    }
+
+    const sessionText = conversationParts.join("\n");
+    const sessionRelPath = path.join("sessions", target.agentId, path.basename(filePath)).replace(/\\/g, "/");
+    const pieces = chunkText(sessionText);
+    pieces.forEach((piece, index) => {
+      chunks.push({
+        chunkKey: buildChunkKey({
+          workspaceHash: target.workspaceHash,
+          agentId: target.agentId,
+          sourceType: "session",
+          path: sessionRelPath,
+          index,
+          text: piece,
+        }),
+        contentHash: sha256(normalizeForHash(piece)),
+        sourceType: "session",
+        text: piece,
+        path: sessionRelPath,
+        workspaceHash: target.workspaceHash,
+        agentId: target.agentId,
+        updatedAt: statMtime,
+      });
+    });
+  }
+
+  return chunks;
+}

package/src/config.ts

@@ -0,0 +1,271 @@
+export type MemoryBraidConfig = {
+  enabled: boolean;
+  mem0: {
+    mode: "cloud" | "oss";
+    apiKey?: string;
+    host?: string;
+    organizationId?: string;
+    projectId?: string;
+    ossConfig: Record<string, unknown>;
+  };
+  recall: {
+    maxResults: number;
+    injectTopK: number;
+    merge: {
+      strategy: "rrf";
+      rrfK: number;
+      localWeight: number;
+      mem0Weight: number;
+    };
+  };
+  capture: {
+    enabled: boolean;
+    extraction: {
+      mode: "heuristic" | "heuristic_plus_ml";
+    };
+    ml: {
+      provider?: "openai" | "anthropic" | "gemini";
+      model?: string;
+      timeoutMs: number;
+      maxItemsPerRun: number;
+    };
+  };
+  bootstrap: {
+    enabled: boolean;
+    startupMode: "async";
+    includeMarkdown: boolean;
+    includeSessions: boolean;
+    sessionLookbackDays: number;
+    batchSize: number;
+    concurrency: number;
+  };
+  reconcile: {
+    enabled: boolean;
+    intervalMinutes: number;
+    batchSize: number;
+    deleteStale: boolean;
+  };
+  dedupe: {
+    lexical: {
+      minJaccard: number;
+    };
+    semantic: {
+      enabled: boolean;
+      minScore: number;
+    };
+  };
+  debug: {
+    enabled: boolean;
+    includePayloads: boolean;
+    maxSnippetChars: number;
+    logSamplingRate: number;
+  };
+};
+
+const DEFAULTS: MemoryBraidConfig = {
+  enabled: true,
+  mem0: {
+    mode: "cloud",
+    apiKey: undefined,
+    host: undefined,
+    organizationId: undefined,
+    projectId: undefined,
+    ossConfig: {},
+  },
+  recall: {
+    maxResults: 8,
+    injectTopK: 5,
+    merge: {
+      strategy: "rrf",
+      rrfK: 60,
+      localWeight: 1,
+      mem0Weight: 1,
+    },
+  },
+  capture: {
+    enabled: true,
+    extraction: {
+      mode: "heuristic",
+    },
+    ml: {
+      provider: undefined,
+      model: undefined,
+      timeoutMs: 2500,
+      maxItemsPerRun: 6,
+    },
+  },
+  bootstrap: {
+    enabled: true,
+    startupMode: "async",
+    includeMarkdown: true,
+    includeSessions: true,
+    sessionLookbackDays: 90,
+    batchSize: 50,
+    concurrency: 3,
+  },
+  reconcile: {
+    enabled: true,
+    intervalMinutes: 30,
+    batchSize: 100,
+    deleteStale: true,
+  },
+  dedupe: {
+    lexical: {
+      minJaccard: 0.3,
+    },
+    semantic: {
+      enabled: true,
+      minScore: 0.92,
+    },
+  },
+  debug: {
+    enabled: false,
+    includePayloads: false,
+    maxSnippetChars: 500,
+    logSamplingRate: 1,
+  },
+};
+
+function asRecord(value: unknown): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return {};
+  }
+  return value as Record<string, unknown>;
+}
+
+function asBoolean(value: unknown, fallback: boolean): boolean {
+  return typeof value === "boolean" ? value : fallback;
+}
+
+function asString(value: unknown): string | undefined {
+  if (typeof value !== "string") {
+    return undefined;
+  }
+  const trimmed = value.trim();
+  return trimmed ? trimmed : undefined;
+}
+
+function asNumber(value: unknown, fallback: number, min: number, max: number): number {
+  const n = typeof value === "number" && Number.isFinite(value) ? value : fallback;
+  return Math.min(max, Math.max(min, n));
+}
+
+function asInt(value: unknown, fallback: number, min: number, max: number): number {
+  return Math.round(asNumber(value, fallback, min, max));
+}
+
+export function parseConfig(raw: unknown): MemoryBraidConfig {
+  const root = asRecord(raw);
+  const mem0 = asRecord(root.mem0);
+  const recall = asRecord(root.recall);
+  const merge = asRecord(recall.merge);
+  const capture = asRecord(root.capture);
+  const extraction = asRecord(capture.extraction);
+  const ml = asRecord(capture.ml);
+  const bootstrap = asRecord(root.bootstrap);
+  const reconcile = asRecord(root.reconcile);
+  const dedupe = asRecord(root.dedupe);
+  const lexical = asRecord(dedupe.lexical);
+  const semantic = asRecord(dedupe.semantic);
+  const debug = asRecord(root.debug);
+
+  const mode = mem0.mode === "oss" ? "oss" : "cloud";
+  const extractionMode =
+    extraction.mode === "heuristic_plus_ml" ? "heuristic_plus_ml" : "heuristic";
+
+  return {
+    enabled: asBoolean(root.enabled, DEFAULTS.enabled),
+    mem0: {
+      mode,
+      apiKey: asString(mem0.apiKey),
+      host: asString(mem0.host),
+      organizationId: asString(mem0.organizationId),
+      projectId: asString(mem0.projectId),
+      ossConfig: asRecord(mem0.ossConfig),
+    },
+    recall: {
+      maxResults: asInt(recall.maxResults, DEFAULTS.recall.maxResults, 1, 50),
+      injectTopK: asInt(recall.injectTopK, DEFAULTS.recall.injectTopK, 1, 20),
+      merge: {
+        strategy: "rrf",
+        rrfK: asInt(merge.rrfK, DEFAULTS.recall.merge.rrfK, 1, 500),
+        localWeight: asNumber(merge.localWeight, DEFAULTS.recall.merge.localWeight, 0, 5),
+        mem0Weight: asNumber(merge.mem0Weight, DEFAULTS.recall.merge.mem0Weight, 0, 5),
+      },
+    },
+    capture: {
+      enabled: asBoolean(capture.enabled, DEFAULTS.capture.enabled),
+      extraction: {
+        mode: extractionMode,
+      },
+      ml: {
+        provider:
+          ml.provider === "openai" || ml.provider === "anthropic" || ml.provider === "gemini"
+            ? ml.provider
+            : DEFAULTS.capture.ml.provider,
+        model: asString(ml.model),
+        timeoutMs: asInt(ml.timeoutMs, DEFAULTS.capture.ml.timeoutMs, 250, 30_000),
+        maxItemsPerRun: asInt(ml.maxItemsPerRun, DEFAULTS.capture.ml.maxItemsPerRun, 1, 50),
+      },
+    },
+    bootstrap: {
+      enabled: asBoolean(bootstrap.enabled, DEFAULTS.bootstrap.enabled),
+      startupMode: "async",
+      includeMarkdown: asBoolean(bootstrap.includeMarkdown, DEFAULTS.bootstrap.includeMarkdown),
+      includeSessions: asBoolean(bootstrap.includeSessions, DEFAULTS.bootstrap.includeSessions),
+      sessionLookbackDays: asInt(
+        bootstrap.sessionLookbackDays,
+        DEFAULTS.bootstrap.sessionLookbackDays,
+        1,
+        3650,
+      ),
+      batchSize: asInt(bootstrap.batchSize, DEFAULTS.bootstrap.batchSize, 1, 1000),
+      concurrency: asInt(bootstrap.concurrency, DEFAULTS.bootstrap.concurrency, 1, 16),
+    },
+    reconcile: {
+      enabled: asBoolean(reconcile.enabled, DEFAULTS.reconcile.enabled),
+      intervalMinutes: asInt(
+        reconcile.intervalMinutes,
+        DEFAULTS.reconcile.intervalMinutes,
+        1,
+        1440,
+      ),
+      batchSize: asInt(reconcile.batchSize, DEFAULTS.reconcile.batchSize, 1, 5000),
+      deleteStale: asBoolean(reconcile.deleteStale, DEFAULTS.reconcile.deleteStale),
+    },
+    dedupe: {
+      lexical: {
+        minJaccard: asNumber(lexical.minJaccard, DEFAULTS.dedupe.lexical.minJaccard, 0, 1),
+      },
+      semantic: {
+        enabled: asBoolean(semantic.enabled, DEFAULTS.dedupe.semantic.enabled),
+        minScore: asNumber(semantic.minScore, DEFAULTS.dedupe.semantic.minScore, 0, 1),
+      },
+    },
+    debug: {
+      enabled: asBoolean(debug.enabled, DEFAULTS.debug.enabled),
+      includePayloads: asBoolean(debug.includePayloads, DEFAULTS.debug.includePayloads),
+      maxSnippetChars: asInt(debug.maxSnippetChars, DEFAULTS.debug.maxSnippetChars, 40, 8000),
+      logSamplingRate: asNumber(debug.logSamplingRate, DEFAULTS.debug.logSamplingRate, 0, 1),
+    },
+  };
+}
+
+export const pluginConfigSchema = {
+  parse(value: unknown) {
+    return parseConfig(value);
+  },
+  safeParse(value: unknown) {
+    try {
+      const data = parseConfig(value);
+      return { success: true as const, data };
+    } catch (err) {
+      return {
+        success: false as const,
+        error: {
+          issues: [{ path: [], message: err instanceof Error ? err.message : String(err) }],
+        },
+      };
+    }
+  },
+};

package/src/dedupe.ts

@@ -0,0 +1,96 @@
+import { normalizeForHash, sha256 } from "./chunking.js";
+import type { MemoryBraidResult } from "./types.js";
+
+export type SemanticCompareFn = (
+  left: MemoryBraidResult,
+  right: MemoryBraidResult,
+) => Promise<number | undefined>;
+
+export type DedupeOptions = {
+  lexicalMinJaccard: number;
+  semanticEnabled: boolean;
+  semanticMinScore: number;
+  semanticCompare?: SemanticCompareFn;
+};
+
+function tokenize(value: string): Set<string> {
+  const tokens = value
+    .toLowerCase()
+    .replace(/[^a-z0-9\s]/gi, " ")
+    .split(/\s+/)
+    .map((token) => token.trim())
+    .filter((token) => token.length >= 2);
+  return new Set(tokens);
+}
+
+function jaccard(a: Set<string>, b: Set<string>): number {
+  if (a.size === 0 || b.size === 0) {
+    return 0;
+  }
+  let intersection = 0;
+  for (const token of a) {
+    if (b.has(token)) {
+      intersection += 1;
+    }
+  }
+  const union = a.size + b.size - intersection;
+  return union <= 0 ? 0 : intersection / union;
+}
+
+export async function stagedDedupe(
+  input: MemoryBraidResult[],
+  options: DedupeOptions,
+): Promise<MemoryBraidResult[]> {
+  const out: MemoryBraidResult[] = [];
+  const exact = new Set<string>();
+
+  for (const candidate of input) {
+    const normalized = normalizeForHash(candidate.snippet);
+    if (!normalized) {
+      continue;
+    }
+
+    const exactKey = sha256(normalized);
+    if (exact.has(exactKey)) {
+      continue;
+    }
+
+    const candidateTokens = tokenize(normalized);
+    let duplicate = false;
+
+    for (const chosen of out) {
+      const chosenTokens = tokenize(normalizeForHash(chosen.snippet));
+      const lexicalScore = jaccard(candidateTokens, chosenTokens);
+      if (lexicalScore < options.lexicalMinJaccard) {
+        continue;
+      }
+
+      if (!options.semanticEnabled) {
+        duplicate = true;
+        break;
+      }
+
+      if (!options.semanticCompare) {
+        duplicate = true;
+        break;
+      }
+
+      const semantic = await options.semanticCompare(candidate, chosen);
+      if (typeof semantic !== "number") {
+        duplicate = true;
+        break;
+      }
+      if (semantic >= options.semanticMinScore) {
+        duplicate = true;
+        break;
+      }
+    }
+
+    if (!duplicate) {
+      exact.add(exactKey);
+      out.push(candidate);
+    }
+  }
+
+  return out;
+}