@agentprojectcontext/apx 1.24.0 → 1.27.1

package/src/core/memory/embeddings.js

@@ -0,0 +1,132 @@
+// Text embeddings for the cross-channel memory RAG (Pieza 2).
+//
+// The provider is configurable (config.memory.embeddings) and resolved through
+// the engine registry at ./embed-engines — exactly like TTS/STT. Ships with
+// ollama (local, default), openai, gemini, and the offline tf fallback. Pick
+// "auto" (chain router) or a single provider; an explicit `provider` opt wins.
+//
+// Fallback: a deterministic, dependency-free feature-hashing term-frequency
+// vector ("tf"). It is NOT as good as a real embedding, but it keeps the
+// retriever working when no provider is reachable — the whole memory system
+// must degrade gracefully, never throw into the daemon's request path.
+//
+// Every vector is tagged with the `embedder` that produced it (e.g.
+// "ollama:nomic-embed-text", "openai:text-embedding-3-small", or "tf") and its
+// `dim`. Cosine similarity is only meaningful within one embedder space, so
+// callers must compare like with like (the store records the tag per row and
+// the broker filters on it).
+
+// NOTE: this module is the leaf the engine adapters import (l2normalize /
+// tfEmbed). It also imports the registry — ESM live bindings make this safe
+// because selectEmbedEngine is only referenced at call time, never at init.
+import { selectEmbedEngine } from "./embed-engines/index.js";
+
+const TF_DIM = 256;
+
+// Deterministic 32-bit string hash (FNV-1a). Used to bucket tokens for the TF
+// fallback embedder.
+function fnv1a(str) {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < str.length; i++) {
+    h ^= str.charCodeAt(i);
+    h = Math.imul(h, 0x01000193);
+  }
+  return h >>> 0;
+}
+
+function tokenize(text) {
+  return String(text || "")
+    .toLowerCase()
+    .normalize("NFKD")
+    .replace(/[̀-ͯ]/g, "")
+    .split(/[^a-z0-9_]+/)
+    .filter((t) => t.length >= 2 && t.length <= 40);
+}
+
+// Feature-hashing TF vector, L2-normalised. Deterministic and offline.
+export function tfEmbed(text, dim = TF_DIM) {
+  const counts = new Map();
+  for (const tok of tokenize(text)) {
+    const bucket = fnv1a(tok) % dim;
+    counts.set(bucket, (counts.get(bucket) || 0) + 1);
+  }
+  const vec = new Array(dim).fill(0);
+  for (const [bucket, c] of counts) {
+    // Sublinear TF scaling so a repeated word doesn't dominate.
+    vec[bucket] = 1 + Math.log(c);
+  }
+  return l2normalize(vec);
+}
+
+export function l2normalize(vec) {
+  let norm = 0;
+  for (const v of vec) norm += v * v;
+  norm = Math.sqrt(norm);
+  if (norm === 0) return vec;
+  return vec.map((v) => v / norm);
+}
+
+export function cosineSim(a, b) {
+  if (!a || !b || a.length !== b.length) return 0;
+  let dot = 0;
+  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
+  return dot; // both vectors are L2-normalised, so dot == cosine
+}
+
+// Embed a single string. Never throws — falls back to the offline TF embedder
+// on any provider error/timeout. Returns { vector, embedder, dim }.
+//
+// opts: { globalConfig, provider, forceTf, timeoutMs, signal }
+//   - globalConfig: read config.memory.embeddings to pick the provider
+//   - provider: explicit override ("ollama"|"openai"|"gemini"|"tf"); wins over config
+//   - forceTf: skip the provider, use the offline embedder (batch fast-path)
+function tfResult(text) {
+  const v = tfEmbed(text);
+  return { vector: v, embedder: "tf", dim: v.length };
+}
+
+export async function embedOne(text, opts = {}) {
+  const clean = String(text || "").slice(0, 8000);
+  if (!clean.trim()) return tfResult("");
+  if (opts.forceTf) return tfResult(clean);
+
+  let adapter, engineConfig;
+  try {
+    ({ adapter, engineConfig } = await selectEmbedEngine({
+      globalConfig: opts.globalConfig,
+      provider: opts.provider,
+    }));
+  } catch {
+    return tfResult(clean);
+  }
+
+  const timeoutMs = opts.timeoutMs || engineConfig?.timeout_ms || 4000;
+  try {
+    const out = await adapter.embed({
+      text: clean,
+      config: engineConfig || {},
+      parentEnginesCfg: opts.globalConfig?.engines,
+      timeoutMs,
+      signal: opts.signal,
+    });
+    if (!out || !Array.isArray(out.vector) || out.vector.length === 0) return tfResult(clean);
+    return out;
+  } catch {
+    return tfResult(clean);
+  }
+}
+
+// Embed many strings. Probes the provider once with the first item; if that
+// falls back to TF, the rest go straight to TF (so a down host doesn't cost
+// one timeout per chunk). Returns an array of { vector, embedder, dim }.
+export async function embedBatch(texts, opts = {}) {
+  const list = Array.isArray(texts) ? texts : [texts];
+  if (list.length === 0) return [];
+  const first = await embedOne(list[0], opts);
+  const out = [first];
+  const forceTf = first.embedder === "tf" ? { ...opts, forceTf: true } : opts;
+  for (let i = 1; i < list.length; i++) {
+    out.push(await embedOne(list[i], forceTf));
+  }
+  return out;
+}

package/src/core/memory/index.js

@@ -0,0 +1,161 @@
+// Cross-channel memory subsystem — public surface + lifecycle.
+//
+// Pieces:
+//   1. Auto-write   → ~/.apx/memory.md (self-memory.js + the `remember` tool)
+//   2. RAG          → embeddings.js + store.js + indexer.js (this module owns
+//                     the store singleton and the incremental index timer)
+//   3. Compaction   → compactor.js (+ messages-store reader changes)
+//   4. Broker       → broker.js (memoryBlockFor below)
+//
+// Everything is best-effort and degrades gracefully: if the store can't open,
+// if Ollama is down, or if anything throws, the daemon keeps serving — the
+// affected piece simply contributes nothing.
+
+import path from "node:path";
+import { APX_HOME } from "../config.js";
+import { ensureSelfMemoryFile } from "../agent/self-memory.js";
+import fs from "node:fs";
+import { openMemoryStore } from "./store.js";
+import { indexNewMessages, CURSOR_PATH } from "./indexer.js";
+import { buildMemoryBlock } from "./broker.js";
+import { compactChannelIfNeeded } from "./compactor.js";
+import { buildActiveThreadsBlock } from "./active-threads.js";
+
+export { compactChannelIfNeeded, buildActiveThreadsBlock };
+
+const DB_PATH = path.join(APX_HOME, "memory.db");
+const JSON_PATH = path.join(APX_HOME, "memory-index.jsonl");
+
+let _store = null;
+let _ready = null;
+let _timer = null;
+let _cfg = {};
+let _indexing = false;
+
+// Run one index pass unless one is already in flight (a full re-embed can take
+// longer than the timer interval — overlapping passes would race on clear()).
+function indexOnce(note) {
+  if (_indexing || !_store) return Promise.resolve();
+  _indexing = true;
+  return indexNewMessages(_store, { embed: embedOptsFromConfig(_cfg), log: note })
+    .catch(() => {})
+    .finally(() => {
+      _indexing = false;
+    });
+}
+
+export function memoryEnabled(config) {
+  return config?.memory?.enabled !== false;
+}
+
+// The embeddings provider is resolved from config.memory.embeddings inside
+// embedOne/embedBatch via the engine registry — so all we forward here is the
+// live config. (Legacy memory.embed_* keys are still honored by the registry's
+// back-compat shim in embed-engines/index.js.)
+function embedOptsFromConfig(config) {
+  return { globalConfig: config };
+}
+
+// Boot the subsystem (Pieza 1 file creation + Pieza 2 store/index). Safe to
+// call once from the daemon. Never throws.
+export async function initMemory({ config, log } = {}) {
+  const note = typeof log === "function" ? log : () => {};
+  try {
+    const created = ensureSelfMemoryFile();
+    if (created) note("memory: created ~/.apx/memory.md");
+  } catch {
+    /* best-effort */
+  }
+  if (!memoryEnabled(config)) {
+    note("memory: RAG disabled by config (memory.enabled=false)");
+    return null;
+  }
+  _cfg = config || {};
+  _ready = (async () => {
+    try {
+      _store = await openMemoryStore({ dbPath: DB_PATH, jsonPath: JSON_PATH, log: note });
+      // Initial index in the background — never blocks boot.
+      indexOnce(note);
+      return _store;
+    } catch (e) {
+      note(`memory: store init failed (${e?.message || e})`);
+      return null;
+    }
+  })();
+
+  const everyMs = (config?.memory?.index_interval_s || 60) * 1000;
+  _timer = setInterval(() => indexOnce(note), everyMs);
+  if (_timer.unref) _timer.unref();
+
+  return _ready;
+}
+
+// Rebuild the vector store from scratch under the CURRENTLY configured embedder.
+// Switching provider/model changes the embedder space, which makes old rows
+// (tagged with the previous embedder) invisible to new queries — they're not
+// re-embedded incrementally because the cursor marks them as already indexed.
+// This clears the store + cursor and re-embeds every message. Never throws.
+export async function reindexMemory({ config } = {}) {
+  if (config) _cfg = config;
+  const store = await getMemoryStore();
+  if (!store) return { cleared: false, indexed: 0, error: "store unavailable" };
+  const before = store.count();
+  try {
+    store.clear();
+  } catch {
+    /* best-effort */
+  }
+  try {
+    fs.rmSync(CURSOR_PATH, { force: true });
+  } catch {
+    /* best-effort */
+  }
+  await indexNewMessages(store, { embed: embedOptsFromConfig(_cfg), log: () => {} }).catch(() => {});
+  return { cleared: before, indexed: store.count() };
+}
+
+export async function getMemoryStore() {
+  if (_ready) {
+    try {
+      return await _ready;
+    } catch {
+      return _store;
+    }
+  }
+  return _store;
+}
+
+export function stopMemory() {
+  if (_timer) {
+    clearInterval(_timer);
+    _timer = null;
+  }
+  if (_store) {
+    try {
+      _store.close();
+    } catch {
+      /* ignore */
+    }
+    _store = null;
+  }
+  _ready = null;
+}
+
+// Build the [MEMORIA RELEVANTE] block for a turn (Pieza 4). Never throws —
+// returns "" on any failure so the prompt builder can drop the block.
+export async function memoryBlockFor(message, { config, channel, budgetMs } = {}) {
+  try {
+    if (!memoryEnabled(config)) return "";
+    const store = await getMemoryStore();
+    return await buildMemoryBlock(message, {
+      store,
+      config,
+      channel,
+      budgetMs: budgetMs || config?.memory?.broker_budget_ms || 800,
+      topK: config?.memory?.rag_top_k || 5,
+      embed: embedOptsFromConfig(config),
+    });
+  } catch {
+    return "";
+  }
+}

package/src/core/memory/indexer.js

@@ -0,0 +1,257 @@
+// Incremental indexer for the cross-channel memory RAG (Pieza 2).
+//
+// Walks ~/.apx/messages/<channel>/YYYY-MM-DD.jsonl for every channel (telegram,
+// web, deck, voice, …) plus ~/.apx/memory.md, turns each relevant record into a
+// chunk, embeds the NEW ones, and upserts them into the vector store.
+//
+// Incremental: a cursor file (~/.apx/memory-cursor.json) records the last
+// indexed timestamp per channel, so each run only embeds messages newer than
+// the last one. memory.md entries are re-derived every run but keyed by a
+// content hash, so re-indexing them is idempotent (cheap, few entries).
+//
+// What gets indexed:
+//   - type:"user" and type:"agent" turns (full body, capped)
+//   - type:"tool" results, truncated to 400 chars, tagged [tool:<name>]
+//   - memory.md entries as separate chunks tagged [memory]
+//
+// Everything is best-effort — a failure logs and returns, never throws into the
+// daemon.
+
+import fs from "node:fs";
+import path from "node:path";
+import { GLOBAL_MESSAGES_DIR, APX_HOME } from "../config.js";
+import { SELF_MEMORY_PATH, parseSelfMemoryEntries } from "../agent/self-memory.js";
+import { embedBatch, embedOne } from "./embeddings.js";
+
+export const CURSOR_PATH = path.join(APX_HOME, "memory-cursor.json");
+
+const BODY_CAP = 1200; // chars kept per user/agent chunk
+const TOOL_CAP = 400; // chars kept per tool-result chunk
+
+function fnv1aHex(str) {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < str.length; i++) {
+    h ^= str.charCodeAt(i);
+    h = Math.imul(h, 0x01000193);
+  }
+  return (h >>> 0).toString(16);
+}
+
+function readCursor(cursorPath) {
+  try {
+    return JSON.parse(fs.readFileSync(cursorPath, "utf8"));
+  } catch {
+    return { channels: {} };
+  }
+}
+
+function writeCursor(cursorPath, cursor) {
+  try {
+    fs.mkdirSync(path.dirname(cursorPath), { recursive: true });
+    const tmp = `${cursorPath}.tmp`;
+    fs.writeFileSync(tmp, JSON.stringify(cursor, null, 2));
+    fs.renameSync(tmp, cursorPath);
+  } catch {
+    /* best-effort */
+  }
+}
+
+// Parse one .jsonl line into a minimal record we care about.
+function parseLine(line) {
+  const t = line.trim();
+  if (!t) return null;
+  try {
+    return JSON.parse(t);
+  } catch {
+    return null;
+  }
+}
+
+// Worth indexing? Skip trivial/noise turns ("ok", "?", "}", a lone emoji,
+// "/reset") — they pollute retrieval (especially under the TF fallback) without
+// carrying recallable content. Require some real text: ≥12 chars and ≥2
+// word-tokens.
+function meaningfulBody(body) {
+  const t = String(body || "").trim();
+  if (t.length < 12) return false;
+  if (/^\/(reset|new|start|help)\b/i.test(t)) return false;
+  const words = t.match(/[\p{L}\p{N}]{2,}/gu) || [];
+  return words.length >= 2;
+}
+
+// Build a chunk {id, source, channel, ts, tag, text} from a raw message record,
+// or null if it shouldn't be indexed.
+function chunkFromMessage(obj, channel) {
+  const meta = obj.meta || {};
+  const type = obj.type || meta.type;
+  const body = String(obj.body || "").trim();
+  if (!body || !meaningfulBody(body)) return null;
+  const ts = obj.ts || "";
+  const idBase = `${channel}:${ts}:${meta.message_id ?? meta.external_id ?? fnv1aHex(body)}`;
+  if (type === "user" || type === "agent") {
+    return {
+      id: `msg:${idBase}`,
+      source: "message",
+      channel,
+      ts,
+      tag: type,
+      text: body.slice(0, BODY_CAP),
+    };
+  }
+  if (type === "tool") {
+    const toolName = meta.tool_name || meta.tool || "tool";
+    return {
+      id: `tool:${idBase}`,
+      source: "message",
+      channel,
+      ts,
+      tag: `tool:${toolName}`,
+      text: `[tool result: ${toolName}] ${body}`.slice(0, TOOL_CAP),
+    };
+  }
+  return null;
+}
+
+// Collect new (not-yet-indexed) chunks from all channel message files.
+function collectMessageChunks(store, cursor, messagesDir) {
+  const fresh = [];
+  const maxTsByChannel = {};
+  if (!fs.existsSync(messagesDir)) return { fresh, maxTsByChannel };
+  for (const channel of fs.readdirSync(messagesDir)) {
+    const dir = path.join(messagesDir, channel);
+    let stat;
+    try {
+      stat = fs.statSync(dir);
+    } catch {
+      continue;
+    }
+    if (!stat.isDirectory()) continue;
+    const since = cursor.channels?.[channel] || "";
+    let maxTs = since;
+    for (const f of fs.readdirSync(dir).sort()) {
+      if (!/^\d{4}-\d{2}-\d{2}\.jsonl$/.test(f)) continue;
+      // Skip whole day files older than the cursor day.
+      if (since && f.slice(0, 10) < since.slice(0, 10)) continue;
+      let text;
+      try {
+        text = fs.readFileSync(path.join(dir, f), "utf8");
+      } catch {
+        continue;
+      }
+      for (const line of text.split("\n")) {
+        const obj = parseLine(line);
+        if (!obj || !obj.ts) continue;
+        if (since && obj.ts <= since) continue;
+        if (obj.ts > maxTs) maxTs = obj.ts;
+        const chunk = chunkFromMessage(obj, channel);
+        if (chunk && !store.hasId(chunk.id)) fresh.push(chunk);
+      }
+    }
+    if (maxTs) maxTsByChannel[channel] = maxTs;
+  }
+  return { fresh, maxTsByChannel };
+}
+
+// Collect memory.md entries as chunks (idempotent — keyed by content hash).
+function collectMemoryChunks(store, memoryPath) {
+  const fresh = [];
+  let entries = [];
+  try {
+    const text = fs.readFileSync(memoryPath, "utf8");
+    entries = parseSelfMemoryEntries(text);
+  } catch {
+    return fresh;
+  }
+  for (const e of entries) {
+    const id = `memory:${e.date || "0"}:${fnv1aHex(e.text)}`;
+    if (store.hasId(id)) continue;
+    fresh.push({
+      id,
+      source: "memory",
+      channel: e.channel || "memory",
+      ts: e.ts || (e.date ? `${e.date}T00:00:00Z` : ""),
+      tag: "memory",
+      text: e.text.slice(0, BODY_CAP),
+    });
+  }
+  return fresh;
+}
+
+// Run one incremental indexing pass. Returns { indexed, backend }.
+// `opts.embed` overrides embedding options (baseUrl/model/timeoutMs) for tests.
+export async function indexNewMessages(store, opts = {}) {
+  const log = typeof opts.log === "function" ? opts.log : () => {};
+  const cursorPath = opts.cursorPath || CURSOR_PATH;
+  const messagesDir = opts.messagesDir || GLOBAL_MESSAGES_DIR;
+  const memoryPath = opts.memoryPath || SELF_MEMORY_PATH;
+  const embedOpts = opts.embed || {};
+  const limit = opts.limit || 1000;
+
+  const cursor = readCursor(cursorPath);
+  cursor.channels = cursor.channels || {};
+
+  // Embedder-consistency guard. Cosine only works within one embedder space, so
+  // the whole store must share an embedder family ("ollama" vs "tf"). Probe the
+  // currently-active embedder and reconcile:
+  //   - upgrade (tf → ollama, i.e. Ollama came back): wipe + full re-index so
+  //     the history is re-embedded in the better space.
+  //   - downgrade (ollama → tf, i.e. Ollama went down): skip this pass entirely
+  //     so we never pollute a good nomic store with TF rows.
+  const probe = await embedOne("apx memory embedder probe", embedOpts);
+  const family = probe.embedder.startsWith("ollama") ? "ollama" : "tf";
+  const stored = cursor.embedder || null;
+  if (stored && stored !== family && store.count() > 0) {
+    if (family === "ollama") {
+      try {
+        store.clear?.();
+      } catch {
+        /* ignore */
+      }
+      cursor.channels = {};
+      log(`memory: embedder ${stored}→ollama — full re-index`);
+    } else {
+      log("memory: Ollama unavailable — skipping index to keep embedder consistent");
+      cursor.embedder = stored; // keep the good signature
+      writeCursor(cursorPath, cursor);
+      return { indexed: 0, backend: store.backend, skipped: "embedder-downgrade" };
+    }
+  }
+  cursor.embedder = family;
+
+  const { fresh: msgChunks, maxTsByChannel } = collectMessageChunks(store, cursor, messagesDir);
+  const memChunks = collectMemoryChunks(store, memoryPath);
+  let chunks = [...msgChunks, ...memChunks];
+  if (chunks.length === 0) return { indexed: 0, backend: store.backend };
+
+  // Cap per-run work so a huge first index doesn't block; the rest is picked
+  // up next pass (cursor only advances for what we actually embedded).
+  let capped = false;
+  if (chunks.length > limit) {
+    chunks = chunks.slice(0, limit);
+    capped = true;
+  }
+
+  const vecs = await embedBatch(
+    chunks.map((c) => c.text),
+    embedOpts
+  );
+  const rows = chunks.map((c, i) => ({
+    ...c,
+    embedder: vecs[i].embedder,
+    dim: vecs[i].dim,
+    vector: vecs[i].vector,
+  }));
+  const indexed = store.upsert(rows);
+
+  // Only advance the per-channel cursor when we weren't capped (otherwise we'd
+  // skip the chunks that didn't fit this pass).
+  if (!capped) {
+    for (const [channel, ts] of Object.entries(maxTsByChannel)) {
+      cursor.channels[channel] = ts;
+    }
+    writeCursor(cursorPath, cursor);
+  }
+
+  log(`memory: indexed ${indexed} chunk(s)${capped ? " (capped, more next pass)" : ""}`);
+  return { indexed, backend: store.backend, capped };
+}