@agentmemory/agentmemory 0.7.0

package/src/state/index-persistence.ts

@@ -0,0 +1,63 @@
+import { SearchIndex } from "./search-index.js";
+import { VectorIndex } from "./vector-index.js";
+import type { StateKV } from "./kv.js";
+import { KV } from "./schema.js";
+
+const DEBOUNCE_MS = 5000;
+
+export class IndexPersistence {
+  private timer: ReturnType<typeof setTimeout> | null = null;
+
+  constructor(
+    private kv: StateKV,
+    private bm25: SearchIndex,
+    private vector: VectorIndex | null,
+  ) {}
+
+  scheduleSave(): void {
+    if (this.timer) clearTimeout(this.timer);
+    this.timer = setTimeout(() => this.save(), DEBOUNCE_MS);
+  }
+
+  async save(): Promise<void> {
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+    await this.kv.set(KV.bm25Index, "data", this.bm25.serialize());
+    if (this.vector && this.vector.size > 0) {
+      await this.kv.set(KV.bm25Index, "vectors", this.vector.serialize());
+    }
+  }
+
+  async load(): Promise<{
+    bm25: SearchIndex | null;
+    vector: VectorIndex | null;
+  }> {
+    let bm25: SearchIndex | null = null;
+    let vector: VectorIndex | null = null;
+
+    const bm25Data = await this.kv
+      .get<string>(KV.bm25Index, "data")
+      .catch(() => null);
+    if (bm25Data && typeof bm25Data === "string") {
+      bm25 = SearchIndex.deserialize(bm25Data);
+    }
+
+    const vecData = await this.kv
+      .get<string>(KV.bm25Index, "vectors")
+      .catch(() => null);
+    if (vecData && typeof vecData === "string") {
+      vector = VectorIndex.deserialize(vecData);
+    }
+
+    return { bm25, vector };
+  }
+
+  stop(): void {
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+  }
+}

package/src/state/keyed-mutex.ts

@@ -0,0 +1,18 @@
+const locks = new Map<string, Promise<void>>();
+
+export function withKeyedLock<T>(
+  key: string,
+  fn: () => Promise<T>,
+): Promise<T> {
+  const prev = locks.get(key) ?? Promise.resolve();
+  const next = prev.then(fn, fn);
+  const cleanup = next.then(
+    () => {},
+    () => {},
+  );
+  locks.set(key, cleanup);
+  cleanup.then(() => {
+    if (locks.get(key) === cleanup) locks.delete(key);
+  });
+  return next;
+}

package/src/state/kv.ts

@@ -0,0 +1,33 @@
+import type { ISdk } from 'iii-sdk'
+
+export class StateKV {
+  constructor(private sdk: ISdk) {}
+
+  async get<T = unknown>(scope: string, key: string): Promise<T | null> {
+    return this.sdk.trigger<{ scope: string; key: string }, T | null>(
+      'state::get',
+      { scope, key },
+    )
+  }
+
+  async set<T = unknown>(scope: string, key: string, data: T): Promise<T> {
+    return this.sdk.trigger<{ scope: string; key: string; data: T }, T>(
+      'state::set',
+      { scope, key, data },
+    )
+  }
+
+  async delete(scope: string, key: string): Promise<void> {
+    return this.sdk.trigger<{ scope: string; key: string }, void>(
+      'state::delete',
+      { scope, key },
+    )
+  }
+
+  async list<T = unknown>(scope: string): Promise<T[]> {
+    return this.sdk.trigger<{ scope: string }, T[]>(
+      'state::list',
+      { scope },
+    )
+  }
+}

package/src/state/schema.ts

@@ -0,0 +1,71 @@
+import { createHash } from "node:crypto";
+
+export const KV = {
+  sessions: "mem:sessions",
+  observations: (sessionId: string) => `mem:obs:${sessionId}`,
+  memories: "mem:memories",
+  summaries: "mem:summaries",
+  config: "mem:config",
+  metrics: "mem:metrics",
+  health: "mem:health",
+  embeddings: (obsId: string) => `mem:emb:${obsId}`,
+  bm25Index: "mem:index:bm25",
+  relations: "mem:relations",
+  profiles: "mem:profiles",
+  claudeBridge: "mem:claude-bridge",
+  graphNodes: "mem:graph:nodes",
+  graphEdges: "mem:graph:edges",
+  semantic: "mem:semantic",
+  procedural: "mem:procedural",
+  teamShared: (teamId: string) => `mem:team:${teamId}:shared`,
+  teamUsers: (teamId: string, userId: string) =>
+    `mem:team:${teamId}:users:${userId}`,
+  teamProfile: (teamId: string) => `mem:team:${teamId}:profile`,
+  audit: "mem:audit",
+  actions: "mem:actions",
+  actionEdges: "mem:action-edges",
+  leases: "mem:leases",
+  routines: "mem:routines",
+  routineRuns: "mem:routine-runs",
+  signals: "mem:signals",
+  checkpoints: "mem:checkpoints",
+  mesh: "mem:mesh",
+  sketches: "mem:sketches",
+  facets: "mem:facets",
+  sentinels: "mem:sentinels",
+  crystals: "mem:crystals",
+  lessons: "mem:lessons",
+  graphEdgeHistory: "mem:graph:edge-history",
+  enrichedChunks: (sessionId: string) => `mem:enriched:${sessionId}`,
+  latentEmbeddings: (obsId: string) => `mem:latent:${obsId}`,
+  retentionScores: "mem:retention",
+} as const;
+
+export const STREAM = {
+  name: "mem-live",
+  group: (sessionId: string) => sessionId,
+  viewerGroup: "viewer",
+} as const;
+
+export function generateId(prefix: string): string {
+  const ts = Date.now().toString(36);
+  const rand = crypto.randomUUID().replace(/-/g, "").slice(0, 12);
+  return `${prefix}_${ts}_${rand}`;
+}
+
+export function fingerprintId(prefix: string, content: string): string {
+  const hash = createHash("sha256").update(content).digest("hex");
+  return `${prefix}_${hash.slice(0, 16)}`;
+}
+
+export function jaccardSimilarity(a: string, b: string): number {
+  const setA = new Set(a.split(/\s+/).filter((t) => t.length > 2));
+  const setB = new Set(b.split(/\s+/).filter((t) => t.length > 2));
+  if (setA.size === 0 && setB.size === 0) return 1;
+  if (setA.size === 0 || setB.size === 0) return 0;
+  let intersection = 0;
+  for (const word of setA) {
+    if (setB.has(word)) intersection++;
+  }
+  return intersection / (setA.size + setB.size - intersection);
+}

package/src/state/search-index.ts

@@ -0,0 +1,245 @@
+import type { CompressedObservation } from "../types.js";
+import { stem } from "./stemmer.js";
+import { getSynonyms } from "./synonyms.js";
+
+interface IndexEntry {
+  obsId: string;
+  sessionId: string;
+  termCount: number;
+}
+
+export class SearchIndex {
+  private entries: Map<string, IndexEntry> = new Map();
+  private invertedIndex: Map<string, Set<string>> = new Map();
+  private docTermCounts: Map<string, Map<string, number>> = new Map();
+  private totalDocLength = 0;
+  private sortedTerms: string[] | null = null;
+
+  private readonly k1 = 1.2;
+  private readonly b = 0.75;
+
+  add(obs: CompressedObservation): void {
+    const terms = this.extractTerms(obs);
+    const termFreq = new Map<string, number>();
+    let termCount = 0;
+
+    for (const term of terms) {
+      termFreq.set(term, (termFreq.get(term) || 0) + 1);
+      termCount++;
+    }
+
+    this.entries.set(obs.id, {
+      obsId: obs.id,
+      sessionId: obs.sessionId,
+      termCount,
+    });
+    this.docTermCounts.set(obs.id, termFreq);
+    this.totalDocLength += termCount;
+
+    for (const term of termFreq.keys()) {
+      if (!this.invertedIndex.has(term)) {
+        this.invertedIndex.set(term, new Set());
+      }
+      this.invertedIndex.get(term)!.add(obs.id);
+    }
+
+    this.sortedTerms = null;
+  }
+
+  search(
+    query: string,
+    limit = 20,
+  ): Array<{ obsId: string; sessionId: string; score: number }> {
+    const rawTerms = this.tokenize(query.toLowerCase());
+    if (rawTerms.length === 0) return [];
+
+    const N = this.entries.size;
+    if (N === 0) return [];
+    const avgDocLen = this.totalDocLength / N;
+
+    const queryTerms: Array<{ term: string; weight: number }> = [];
+    const seen = new Set<string>();
+    for (const term of rawTerms) {
+      if (!seen.has(term)) {
+        seen.add(term);
+        queryTerms.push({ term, weight: 1.0 });
+      }
+      for (const syn of getSynonyms(term)) {
+        if (!seen.has(syn)) {
+          seen.add(syn);
+          queryTerms.push({ term: syn, weight: 0.7 });
+        }
+      }
+    }
+
+    const scores = new Map<string, number>();
+    const sorted = this.getSortedTerms();
+
+    for (const { term, weight } of queryTerms) {
+      const matchingDocs = this.invertedIndex.get(term);
+      if (matchingDocs) {
+        const df = matchingDocs.size;
+        const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
+
+        for (const obsId of matchingDocs) {
+          const entry = this.entries.get(obsId)!;
+          const docTerms = this.docTermCounts.get(obsId);
+          const tf = docTerms?.get(term) || 0;
+          const docLen = entry.termCount;
+
+          const numerator = tf * (this.k1 + 1);
+          const denominator =
+            tf + this.k1 * (1 - this.b + this.b * (docLen / avgDocLen));
+          const bm25Score = idf * (numerator / denominator) * weight;
+
+          scores.set(obsId, (scores.get(obsId) || 0) + bm25Score);
+        }
+      }
+
+      const startIdx = this.lowerBound(sorted, term);
+      for (let si = startIdx; si < sorted.length; si++) {
+        const indexTerm = sorted[si];
+        if (!indexTerm.startsWith(term)) break;
+        if (indexTerm === term) continue;
+
+        const obsIds = this.invertedIndex.get(indexTerm)!;
+        const prefixDf = obsIds.size;
+        const prefixIdf =
+          Math.log((N - prefixDf + 0.5) / (prefixDf + 0.5) + 1) * 0.5;
+        for (const obsId of obsIds) {
+          const entry = this.entries.get(obsId)!;
+          const docTerms = this.docTermCounts.get(obsId);
+          const tf = docTerms?.get(indexTerm) || 0;
+          const docLen = entry.termCount;
+          const numerator = tf * (this.k1 + 1);
+          const denominator =
+            tf + this.k1 * (1 - this.b + this.b * (docLen / avgDocLen));
+          scores.set(
+            obsId,
+            (scores.get(obsId) || 0) + prefixIdf * (numerator / denominator) * weight,
+          );
+        }
+      }
+    }
+
+    return Array.from(scores.entries())
+      .map(([obsId, score]) => {
+        const entry = this.entries.get(obsId)!;
+        return { obsId, sessionId: entry.sessionId, score };
+      })
+      .sort((a, b) => b.score - a.score)
+      .slice(0, limit);
+  }
+
+  get size(): number {
+    return this.entries.size;
+  }
+
+  clear(): void {
+    this.entries.clear();
+    this.invertedIndex.clear();
+    this.docTermCounts.clear();
+    this.totalDocLength = 0;
+    this.sortedTerms = null;
+  }
+
+  restoreFrom(other: SearchIndex): void {
+    this.entries = new Map(
+      Array.from(other.entries.entries()).map(([k, v]) => [k, { ...v }]),
+    );
+    this.invertedIndex = new Map(
+      Array.from(other.invertedIndex.entries()).map(([k, v]) => [
+        k,
+        new Set(v),
+      ]),
+    );
+    this.docTermCounts = new Map(
+      Array.from(other.docTermCounts.entries()).map(([k, v]) => [
+        k,
+        new Map(v),
+      ]),
+    );
+    this.totalDocLength = other.totalDocLength;
+    this.sortedTerms = null;
+  }
+
+  serialize(): string {
+    const entries = Array.from(this.entries.entries());
+    const inverted = Array.from(this.invertedIndex.entries()).map(
+      ([term, ids]) => [term, Array.from(ids)] as [string, string[]],
+    );
+    const docTerms = Array.from(this.docTermCounts.entries()).map(
+      ([id, counts]) =>
+        [id, Array.from(counts.entries())] as [string, [string, number][]],
+    );
+    return JSON.stringify({
+      v: 2,
+      entries,
+      inverted,
+      docTerms,
+      totalDocLength: this.totalDocLength,
+    });
+  }
+
+  static deserialize(json: string): SearchIndex {
+    try {
+      const idx = new SearchIndex();
+      const data = JSON.parse(json);
+      if (!data?.entries || !data?.inverted || !data?.docTerms) return idx;
+      for (const [key, val] of data.entries) {
+        idx.entries.set(key, val);
+      }
+      for (const [term, ids] of data.inverted) {
+        idx.invertedIndex.set(term, new Set(ids));
+      }
+      for (const [id, counts] of data.docTerms) {
+        idx.docTermCounts.set(id, new Map(counts));
+      }
+      const rawLen = Number(data.totalDocLength);
+      idx.totalDocLength =
+        Number.isFinite(rawLen) && rawLen >= 0 ? Math.floor(rawLen) : 0;
+      return idx;
+    } catch {
+      return new SearchIndex();
+    }
+  }
+
+  private extractTerms(obs: CompressedObservation): string[] {
+    const parts = [
+      obs.title,
+      obs.subtitle || "",
+      obs.narrative,
+      ...obs.facts,
+      ...obs.concepts,
+      ...obs.files,
+      obs.type,
+    ];
+    return this.tokenize(parts.join(" ").toLowerCase());
+  }
+
+  private tokenize(text: string): string[] {
+    return text
+      .replace(/[^\w\s/.\-_]/g, " ")
+      .split(/\s+/)
+      .filter((t) => t.length > 1)
+      .map((t) => stem(t));
+  }
+
+  private getSortedTerms(): string[] {
+    if (!this.sortedTerms) {
+      this.sortedTerms = Array.from(this.invertedIndex.keys()).sort();
+    }
+    return this.sortedTerms;
+  }
+
+  private lowerBound(arr: string[], target: string): number {
+    let lo = 0;
+    let hi = arr.length;
+    while (lo < hi) {
+      const mid = (lo + hi) >>> 1;
+      if (arr[mid] < target) lo = mid + 1;
+      else hi = mid;
+    }
+    return lo;
+  }
+}

package/src/state/stemmer.ts

@@ -0,0 +1,104 @@
+const step2map: Record<string, string> = {
+  ational: "ate", tional: "tion", enci: "ence", anci: "ance",
+  izer: "ize", iser: "ise", abli: "able", alli: "al",
+  entli: "ent", eli: "e", ousli: "ous", ization: "ize",
+  isation: "ise", ation: "ate", ator: "ate", alism: "al",
+  iveness: "ive", fulness: "ful", ousness: "ous", aliti: "al",
+  iviti: "ive", biliti: "ble",
+};
+
+const step3map: Record<string, string> = {
+  icate: "ic", ative: "", alize: "al", alise: "al",
+  iciti: "ic", ical: "ic", ful: "", ness: "",
+};
+
+function hasVowel(s: string): boolean {
+  return /[aeiou]/.test(s);
+}
+
+function measure(s: string): number {
+  const reduced = s.replace(/[^aeiouy]+/g, "C").replace(/[aeiouy]+/g, "V");
+  const m = reduced.match(/VC/g);
+  return m ? m.length : 0;
+}
+
+function endsDoubleConsonant(s: string): boolean {
+  return s.length >= 2 && s[s.length - 1] === s[s.length - 2] && !/[aeiou]/.test(s[s.length - 1]);
+}
+
+function endsCVC(s: string): boolean {
+  if (s.length < 3) return false;
+  const c1 = s[s.length - 3], v = s[s.length - 2], c2 = s[s.length - 1];
+  return !/[aeiou]/.test(c1) && /[aeiou]/.test(v) && !/[aeiouwxy]/.test(c2);
+}
+
+export function stem(word: string): string {
+  if (word.length <= 2) return word;
+
+  let w = word;
+
+  if (w.endsWith("sses")) w = w.slice(0, -2);
+  else if (w.endsWith("ies")) w = w.slice(0, -2);
+  else if (!w.endsWith("ss") && w.endsWith("s")) w = w.slice(0, -1);
+
+  if (w.endsWith("eed")) {
+    if (measure(w.slice(0, -3)) > 0) w = w.slice(0, -1);
+  } else if (w.endsWith("ed") && hasVowel(w.slice(0, -2))) {
+    w = w.slice(0, -2);
+    if (w.endsWith("at") || w.endsWith("bl") || w.endsWith("iz")) w += "e";
+    else if (endsDoubleConsonant(w) && !/[lsz]$/.test(w)) w = w.slice(0, -1);
+    else if (measure(w) === 1 && endsCVC(w)) w += "e";
+  } else if (w.endsWith("ing") && hasVowel(w.slice(0, -3))) {
+    w = w.slice(0, -3);
+    if (w.endsWith("at") || w.endsWith("bl") || w.endsWith("iz")) w += "e";
+    else if (endsDoubleConsonant(w) && !/[lsz]$/.test(w)) w = w.slice(0, -1);
+    else if (measure(w) === 1 && endsCVC(w)) w += "e";
+  }
+
+  if (w.endsWith("y") && hasVowel(w.slice(0, -1))) {
+    w = w.slice(0, -1) + "i";
+  }
+
+  for (const [suffix, replacement] of Object.entries(step2map)) {
+    if (w.endsWith(suffix)) {
+      const base = w.slice(0, -suffix.length);
+      if (measure(base) > 0) w = base + replacement;
+      break;
+    }
+  }
+
+  for (const [suffix, replacement] of Object.entries(step3map)) {
+    if (w.endsWith(suffix)) {
+      const base = w.slice(0, -suffix.length);
+      if (measure(base) > 0) w = base + replacement;
+      break;
+    }
+  }
+
+  if (w.endsWith("al") || w.endsWith("ance") || w.endsWith("ence") ||
+      w.endsWith("er") || w.endsWith("ic") || w.endsWith("able") ||
+      w.endsWith("ible") || w.endsWith("ant") || w.endsWith("ement") ||
+      w.endsWith("ment") || w.endsWith("ent") || w.endsWith("tion") ||
+      w.endsWith("sion") || w.endsWith("ou") || w.endsWith("ism") ||
+      w.endsWith("ate") || w.endsWith("iti") || w.endsWith("ous") ||
+      w.endsWith("ive") || w.endsWith("ize") || w.endsWith("ise")) {
+    const suffixLen = w.match(/(ement|ment|tion|sion|ance|ence|able|ible|ism|ate|iti|ous|ive|ize|ise|ant|ent|al|er|ic|ou)$/)?.[0]?.length ?? 0;
+    if (suffixLen > 0) {
+      const base = w.slice(0, -suffixLen);
+      if (measure(base) > 1) w = base;
+    }
+  }
+
+  if (w.endsWith("e")) {
+    const base = w.slice(0, -1);
+    if (measure(base) > 1 || (measure(base) === 1 && !endsCVC(base))) {
+      w = base;
+    }
+  }
+
+  if (endsDoubleConsonant(w) && w.endsWith("l") && measure(w.slice(0, -1)) > 1) {
+    w = w.slice(0, -1);
+  }
+
+  return w;
+}

package/src/state/synonyms.ts

@@ -0,0 +1,63 @@
+import { stem } from "./stemmer.js";
+
+const SYNONYM_GROUPS: string[][] = [
+  ["auth", "authentication", "authn", "authenticating"],
+  ["authz", "authorization", "authorizing"],
+  ["db", "database", "datastore"],
+  ["perf", "performance", "latency", "throughput", "slow", "bottleneck"],
+  ["optim", "optimization", "optimizing", "optimise", "query-optimization"],
+  ["k8s", "kubernetes", "kube"],
+  ["config", "configuration", "configuring", "setup"],
+  ["deps", "dependencies", "dependency"],
+  ["env", "environment"],
+  ["fn", "function"],
+  ["impl", "implementation", "implementing"],
+  ["msg", "message", "messaging"],
+  ["repo", "repository"],
+  ["req", "request"],
+  ["res", "response"],
+  ["ts", "typescript"],
+  ["js", "javascript"],
+  ["pg", "postgres", "postgresql"],
+  ["err", "error", "errors"],
+  ["api", "endpoint", "endpoints"],
+  ["ci", "continuous-integration"],
+  ["cd", "continuous-deployment"],
+  ["test", "testing", "tests"],
+  ["doc", "documentation", "docs"],
+  ["infra", "infrastructure"],
+  ["deploy", "deployment", "deploying"],
+  ["cache", "caching", "cached"],
+  ["log", "logging", "logs"],
+  ["monitor", "monitoring"],
+  ["observe", "observability"],
+  ["sec", "security", "secure"],
+  ["validate", "validation", "validating"],
+  ["migrate", "migration", "migrations"],
+  ["debug", "debugging"],
+  ["container", "containerization", "docker"],
+  ["crash", "crashloop", "crashloopbackoff"],
+  ["webhook", "webhooks", "callback"],
+  ["middleware", "mw"],
+  ["paginate", "pagination"],
+  ["serialize", "serialization"],
+  ["encrypt", "encryption"],
+  ["hash", "hashing"],
+];
+
+const synonymMap = new Map<string, Set<string>>();
+
+for (const group of SYNONYM_GROUPS) {
+  const stemmed = group.map(t => stem(t.toLowerCase()));
+  for (const s of stemmed) {
+    if (!synonymMap.has(s)) synonymMap.set(s, new Set());
+    for (const other of stemmed) {
+      if (other !== s) synonymMap.get(s)!.add(other);
+    }
+  }
+}
+
+export function getSynonyms(stemmedTerm: string): string[] {
+  const syns = synonymMap.get(stemmedTerm);
+  return syns ? [...syns] : [];
+}