lynkr 9.9.1 → 9.11.0

package/src/memory/tencentdb-launcher.js

@@ -0,0 +1,459 @@
+/**
+ * TencentDB-Agent-Memory Sidecar Launcher
+ *
+ * Launches the TencentDB-Agent-Memory stack (memory-core gateway +
+ * memory-hub panel/knowledge services) via dockerode when
+ * TENCENTDB_MEMORY_ENABLED=true — the same in-process pattern as the
+ * Headroom sidecar launcher, extended to a two-container stack.
+ *
+ * Mirrors the project's own deploy/global-images/start-*.sh scripts:
+ *  - shared docker network `tdai-memory-stack` with service aliases
+ *  - memory-core config generated locally and mounted read-only
+ *  - first-boot admin user init, key persisted to data/tencentdb-memory/
+ *
+ * The memory services need an LLM for extraction/summarization; by default
+ * that points back at Lynkr's own OpenAI-compatible endpoint, so the whole
+ * stack runs on whatever providers Lynkr already has.
+ *
+ * Containers use restart=unless-stopped and are intentionally NOT stopped
+ * when Lynkr exits — a team memory hub outlives any one proxy process.
+ * Remove with: docker rm -f tdai-memory-core tdai-memory-hub
+ */
+
+let Docker;
+try {
+  Docker = require("dockerode");
+} catch {
+  Docker = null;
+}
+const path = require("path");
+const fs = require("fs");
+const crypto = require("crypto");
+const logger = require("../logger");
+const config = require("../config");
+
+const docker = Docker ? new Docker() : null;
+
+const DATA_DIR = path.join(process.cwd(), "data", "tencentdb-memory");
+const ADMIN_KEY_FILE = path.join(DATA_DIR, ".admin-key");
+const CORE_CONFIG_FILE = path.join(DATA_DIR, "tdai-gateway.yaml");
+
+let isStarting = false;
+
+function sidecarConfig() {
+  return config.tencentdbMemory ?? {};
+}
+
+/**
+ * Gateway config for memory-core, matching the template the project's
+ * start-memory-core.sh generates. Mounted read-only into the container.
+ */
+function buildCoreConfigYaml() {
+  const { llm, promptMode } = sidecarConfig();
+  return `# Generated by Lynkr's TencentDB memory launcher — overwritten on every start.
+deployMode: standalone
+stateBackend: local
+
+server:
+  port: 8420
+  host: 0.0.0.0
+
+data:
+  baseDir: /data/tdai-memory
+
+llm:
+  baseUrl: "${llm.baseUrl}"
+  apiKey: "${llm.apiKey}"
+  model: "${llm.model}"
+  maxTokens: 32000
+  timeoutMs: 300000
+
+memory:
+  promptMode: ${promptMode}
+  capture: { enabled: true }
+  extraction:
+    enabled: true
+    enableDedup: true
+    maxMemoriesPerSession: 20
+  persona:
+    triggerEveryN: 50
+    maxScenes: 15
+  pipeline:
+    everyNConversations: 5
+    enableWarmup: true
+    l1IdleTimeoutSeconds: 600
+    l2DelayAfterL1Seconds: 90
+    l2MinIntervalSeconds: 900
+    l2MaxIntervalSeconds: 3600
+  recall:
+    enabled: true
+    maxResults: 5
+    scoreThreshold: 0.3
+    strategy: hybrid
+    timeoutMs: 5000
+  storeBackend: sqlite
+  embedding:
+    provider: none
+
+skill:
+  enabled: true
+  routing:
+    mode: bm25
+    searchTopK: 20
+  extraction:
+    enabled: true
+    maxIterations: 16
+    queue:
+      backend: local
+      keyPrefix: tdai
+      resultTtlSeconds: 86400
+      lockTtlMs: 600000
+      maxRetries: 2
+      retryBackoffsMs: [5000, 15000]
+  resources:
+    maxResourceSizeBytes: 5000000
+`;
+}
+
+/**
+ * sk-mem-<32 alphanumeric chars> — the user-key format memory-core expects.
+ */
+function generateUserKey() {
+  let raw = "";
+  while (raw.length < 32) {
+    raw += crypto.randomBytes(48).toString("base64").replace(/[^A-Za-z0-9]/g, "");
+  }
+  return `sk-mem-${raw.slice(0, 32)}`;
+}
+
+function maskKey(key) {
+  if (!key || key.length < 16) return "****";
+  return `${key.slice(0, 11)}****${key.slice(-4)}`;
+}
+
+async function ensureNetwork(name) {
+  try {
+    await docker.getNetwork(name).inspect();
+  } catch (err) {
+    if (err.statusCode !== 404) throw err;
+    logger.info({ network: name }, "[tdai] Creating docker network");
+    await docker.createNetwork({ Name: name, CheckDuplicate: true });
+  }
+}
+
+async function getExistingContainer(containerName) {
+  const containers = await docker.listContainers({
+    all: true,
+    filters: { name: [containerName] },
+  });
+  const match = containers.find(
+    (c) => c.Names.includes(`/${containerName}`) || c.Names.includes(containerName)
+  );
+  return match ? docker.getContainer(match.Id) : null;
+}
+
+async function imageExists(imageName) {
+  try {
+    await docker.getImage(imageName).inspect();
+    return true;
+  } catch (err) {
+    if (err.statusCode === 404) return false;
+    throw err;
+  }
+}
+
+function pullImage(imageName) {
+  logger.info({ image: imageName }, "[tdai] Pulling image (first start may take a few minutes)");
+  return new Promise((resolve, reject) => {
+    docker.pull(imageName, (err, stream) => {
+      if (err) return reject(err);
+      docker.modem.followProgress(stream, (progressErr, output) => {
+        if (progressErr) return reject(progressErr);
+        logger.info({ image: imageName }, "[tdai] Image pull complete");
+        resolve(output);
+      });
+    });
+  });
+}
+
+/**
+ * Poll an HTTP endpoint until it responds (any status < 500).
+ */
+async function waitForHttp(url, { maxRetries = 90, intervalMs = 1000, label } = {}) {
+  for (let i = 0; i < maxRetries; i++) {
+    try {
+      const response = await fetch(url, { signal: AbortSignal.timeout(2000) });
+      if (response.status < 500) return true;
+    } catch {
+      // not up yet
+    }
+    if (i % 15 === 14) {
+      logger.debug({ url, attempt: i + 1, maxRetries }, `[tdai] Waiting for ${label ?? url}`);
+    }
+    await new Promise((resolve) => setTimeout(resolve, intervalMs));
+  }
+  throw new Error(`${label ?? url} did not become healthy after ${maxRetries}s`);
+}
+
+/**
+ * Create+start one container, or start it if it exists but is stopped.
+ * Returns the action taken.
+ */
+async function ensureContainer(spec) {
+  const existing = await getExistingContainer(spec.name);
+
+  if (existing) {
+    const info = await existing.inspect();
+    if (info.State.Running) return "existing_running";
+    logger.info({ container: spec.name }, "[tdai] Starting existing container");
+    await existing.start();
+    return "started_existing";
+  }
+
+  if (!(await imageExists(spec.image))) {
+    await pullImage(spec.image);
+  }
+
+  logger.info({ container: spec.name, image: spec.image }, "[tdai] Creating container");
+  const container = await docker.createContainer({
+    Image: spec.image,
+    name: spec.name,
+    Env: spec.env,
+    ExposedPorts: Object.fromEntries(
+      Object.values(spec.ports).map((p) => [`${p.container}/tcp`, {}])
+    ),
+    HostConfig: {
+      PortBindings: Object.fromEntries(
+        Object.values(spec.ports).map((p) => [
+          `${p.container}/tcp`,
+          [{ HostPort: String(p.host) }],
+        ])
+      ),
+      Binds: spec.binds,
+      NetworkMode: spec.network,
+      ExtraHosts: ["host.docker.internal:host-gateway"],
+      RestartPolicy: { Name: "unless-stopped" },
+    },
+    NetworkingConfig: {
+      EndpointsConfig: {
+        [spec.network]: { Aliases: [spec.alias] },
+      },
+    },
+  });
+  await container.start();
+  return "created_new";
+}
+
+/**
+ * First-boot admin bootstrap, mirroring start-memory-core.sh:
+ * generate an sk-mem- key (or reuse the persisted one), init-admin with it,
+ * verify, and persist to .admin-key with tight permissions.
+ */
+async function ensureAdminUser(gatewayUrl) {
+  let adminKey;
+  if (fs.existsSync(ADMIN_KEY_FILE)) {
+    adminKey = fs.readFileSync(ADMIN_KEY_FILE, "utf8").trim();
+  } else {
+    adminKey = generateUserKey();
+  }
+
+  const response = await fetch(`${gatewayUrl}/v3/internal/meta/user/init-admin`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "x-tdai-service-id": "default",
+    },
+    body: JSON.stringify({ username: "admin", user_key: adminKey }),
+    signal: AbortSignal.timeout(10000),
+  });
+
+  if (response.status === 200) {
+    fs.writeFileSync(ADMIN_KEY_FILE, adminKey, { mode: 0o600 });
+    logger.info({ key: maskKey(adminKey), keyFile: ADMIN_KEY_FILE }, "[tdai] Admin user created");
+  } else if (response.status === 409) {
+    if (!fs.existsSync(ADMIN_KEY_FILE)) {
+      logger.warn(
+        "[tdai] Admin user already exists but .admin-key is missing — the key cannot be recovered. " +
+        "Reset with: docker rm -f tdai-memory-core && docker volume rm tdai-memory-core-data"
+      );
+      return null;
+    }
+  } else {
+    logger.warn({ status: response.status }, "[tdai] init-admin returned unexpected status");
+    return null;
+  }
+
+  // Verify the key actually authenticates
+  try {
+    const verify = await fetch(`${gatewayUrl}/v3/meta/auth/verify`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "x-tdai-service-id": "default",
+      },
+      body: JSON.stringify({ user_key: adminKey }),
+      signal: AbortSignal.timeout(5000),
+    });
+    if (verify.status !== 200) {
+      logger.warn({ status: verify.status }, "[tdai] Admin key failed verification — volume and .admin-key may be out of sync");
+      return null;
+    }
+  } catch (err) {
+    logger.debug({ err: err.message }, "[tdai] Admin key verification skipped");
+  }
+
+  return adminKey;
+}
+
+/**
+ * Ensure the full memory stack (core + hub) is running.
+ * Idempotent: containers started by the project's own scripts are detected
+ * by name and reused.
+ */
+async function ensureRunning() {
+  const cfg = sidecarConfig();
+
+  if (!cfg.enabled) return { started: false, reason: "disabled" };
+  if (!cfg.docker?.enabled) return { started: false, reason: "docker_disabled" };
+  if (!docker) return { started: false, reason: "dockerode_unavailable" };
+  if (isStarting) return { started: false, reason: "already_starting" };
+
+  isStarting = true;
+  try {
+    const { network, core, hub } = cfg.docker;
+    const gatewayUrl = `http://localhost:${core.port}`;
+
+    await ensureNetwork(network);
+
+    // ── memory-core ──
+    fs.mkdirSync(DATA_DIR, { recursive: true });
+    fs.writeFileSync(CORE_CONFIG_FILE, buildCoreConfigYaml());
+
+    const coreAction = await ensureContainer({
+      name: core.containerName,
+      image: core.image,
+      alias: "memory-core",
+      network,
+      ports: { gateway: { host: core.port, container: 8420 } },
+      binds: [
+        `${core.volume}:/data/tdai-memory`,
+        `${CORE_CONFIG_FILE}:/data/config/tdai-gateway.yaml:ro`,
+      ],
+      env: [
+        "TDAI_GATEWAY_PORT=8420",
+        "TDAI_GATEWAY_HOST=0.0.0.0",
+        "TDAI_GATEWAY_API_KEY=", // empty = no bearer gate (local deployment)
+        "TDAI_DATA_DIR=/data/tdai-memory",
+      ],
+    });
+    await waitForHttp(`${gatewayUrl}/`, { label: "memory-core" });
+    logger.info({ action: coreAction, url: gatewayUrl }, "[tdai] memory-core ready");
+
+    const adminKey = await ensureAdminUser(gatewayUrl);
+
+    // ── memory-hub (panel + knowledge) ──
+    const { llm } = cfg;
+    const hubAction = await ensureContainer({
+      name: hub.containerName,
+      image: hub.image,
+      alias: "memory-hub",
+      network,
+      ports: {
+        panel: { host: hub.panelPort, container: 8125 },
+        knowledge: { host: hub.knowledgePort, container: 8424 },
+      },
+      binds: [`${hub.volume}:/data/knowledge`],
+      env: [
+        "PANEL_PORT=8125",
+        "KNOWLEDGE_PORT=8424",
+        `KNOWLEDGE_PUBLIC_BASE_URL=http://host.docker.internal:${hub.knowledgePort}/v3`,
+        "REMOTE_INSTANCE_ID=default",
+        "REMOTE_INSTANCE_NAME=default",
+        "REMOTE_INSTANCE_URL=http://memory-core:8420",
+        "REMOTE_INSTANCE_KEY=local",
+        // Panel's "client base URL" card points at Lynkr — Lynkr is the proxy here
+        `REMOTE_INSTANCE_PROXY_URL=http://localhost:${config.port}`,
+        "LLM_MODE=custom",
+        `LLM_PROTOCOL=${llm.protocol}`,
+        `LLM_API_KEY=${llm.apiKey}`,
+        `LLM_BASE_URL=${llm.baseUrl}`,
+        `LLM_MODEL=${llm.model}`,
+        "KNOWLEDGE_LLM_BINDING_SYNC=0",
+      ],
+    });
+    await waitForHttp(`http://localhost:${hub.knowledgePort}/health`, { label: "memory-hub", maxRetries: 120 });
+    logger.info({ action: hubAction }, "[tdai] memory-hub ready");
+
+    return {
+      started: true,
+      actions: { core: coreAction, hub: hubAction },
+      adminKey: adminKey ? maskKey(adminKey) : null,
+      endpoints: {
+        gateway: gatewayUrl,
+        panel: `http://localhost:${hub.panelPort}/`,
+        knowledge: `http://localhost:${hub.knowledgePort}/v3`,
+      },
+    };
+  } finally {
+    isStarting = false;
+  }
+}
+
+/**
+ * Stop both containers (kept on disk; volumes untouched).
+ * Not called automatically on Lynkr shutdown — see module header.
+ */
+async function stop() {
+  if (!docker) return;
+  const { core, hub } = sidecarConfig().docker ?? {};
+  for (const name of [hub?.containerName, core?.containerName].filter(Boolean)) {
+    try {
+      const container = await getExistingContainer(name);
+      if (container) {
+        const info = await container.inspect();
+        if (info.State.Running) {
+          logger.info({ container: name }, "[tdai] Stopping container");
+          await container.stop({ t: 10 });
+        }
+      }
+    } catch (err) {
+      logger.warn({ err, container: name }, "[tdai] Failed to stop container");
+    }
+  }
+}
+
+async function getStatus() {
+  if (!docker) return { available: false };
+  const { core, hub } = sidecarConfig().docker ?? {};
+  const status = { available: true, containers: {} };
+  for (const [key, name] of [["core", core?.containerName], ["hub", hub?.containerName]]) {
+    if (!name) continue;
+    try {
+      const container = await getExistingContainer(name);
+      if (!container) {
+        status.containers[key] = { exists: false, running: false };
+        continue;
+      }
+      const info = await container.inspect();
+      status.containers[key] = {
+        exists: true,
+        running: info.State.Running,
+        status: info.State.Status,
+        image: info.Config.Image,
+      };
+    } catch (err) {
+      status.containers[key] = { exists: false, running: false, error: err.message };
+    }
+  }
+  return status;
+}
+
+module.exports = {
+  ensureRunning,
+  stop,
+  getStatus,
+  // exported for tests
+  buildCoreConfigYaml,
+  generateUserKey,
+  maskKey,
+};

package/src/memory/wiki.js

@@ -0,0 +1,171 @@
+/**
+ * Wiki — Large-Block Content Registry (TencentDB-Agent-Memory inspired)
+ *
+ * Registers large content blocks (docs, configs, big tool outputs) the
+ * first time they appear in old conversation history, then replaces later
+ * near-duplicates with a short reference + summary instead of resending
+ * the full block. Entries persist in the memories table (type='wiki') so
+ * dedup works across sessions.
+ */
+
+const store = require("./store");
+const distill = require("../context/distill");
+const config = require("../config");
+const logger = require("../logger");
+
+const CHARS_PER_TOKEN = 4;
+const MAX_CACHED_ENTRIES = 500;
+const MAX_SIGNATURE_LINES = 200;
+
+// In-memory entry cache: [{ id, summary, signature: Set<string> }]
+let entryCache = null;
+
+function wikiConfig() {
+  return config.memory?.wiki ?? {};
+}
+
+function minChars() {
+  return (wikiConfig().minTokens ?? 500) * CHARS_PER_TOKEN;
+}
+
+/**
+ * Load persisted wiki entries into the in-memory cache (lazy, once).
+ */
+function loadCache() {
+  if (entryCache) return entryCache;
+
+  entryCache = [];
+  try {
+    const rows = store.getMemoriesByType("wiki", MAX_CACHED_ENTRIES);
+    for (const row of rows) {
+      const meta = typeof row.metadata === "string" ? JSON.parse(row.metadata) : row.metadata;
+      if (!meta?.signatureLines?.length) continue;
+      entryCache.push({
+        id: row.id,
+        summary: row.content,
+        signature: new Set(meta.signatureLines),
+      });
+    }
+  } catch (err) {
+    logger.warn({ err }, "[wiki] Failed to load wiki entries, starting empty");
+  }
+  return entryCache;
+}
+
+/**
+ * Build a one-line summary for a content block: first heading or first
+ * meaningful line, plus a size note.
+ */
+function makeSummary(text) {
+  const lines = distill.normalizeText(text).split("\n").filter(Boolean);
+  const heading = lines.find(l => /^#{1,4}\s|^[A-Z][^a-z]*$/.test(l.trim()));
+  const first = (heading || lines[0] || "").trim().slice(0, 120);
+  return `${first} (${lines.length} lines, ~${Math.ceil(text.length / CHARS_PER_TOKEN)} tokens)`;
+}
+
+/**
+ * Register a large block, or return a compact reference if a similar
+ * block is already known.
+ *
+ * @param {string} text - Content block from old history
+ * @returns {{ref: string, id: number, saved: number}|null} Reference when a
+ *   similar entry exists; null when the block was registered or is too small
+ */
+function registerOrDereference(text) {
+  if (wikiConfig().enabled === false) return null;
+  if (!text || text.length < minChars()) return null;
+
+  const threshold = wikiConfig().similarityThreshold ?? 0.85;
+  const signature = distill.extractSignature(text);
+  const cache = loadCache();
+
+  for (const entry of cache) {
+    const sim = distill.jaccardSimilarity(signature, entry.signature);
+    if (sim >= threshold) {
+      const ref = `[wiki:${entry.id} — ${(sim * 100).toFixed(0)}% match] ${entry.summary}`;
+      return { ref, id: entry.id, saved: text.length - ref.length };
+    }
+  }
+
+  // No match — register for future dedup
+  try {
+    const summary = makeSummary(text);
+    const memory = store.createMemory({
+      sessionId: null, // wiki entries are global
+      content: summary,
+      type: "wiki",
+      category: "reference",
+      importance: 0.3,
+      metadata: {
+        signatureLines: Array.from(signature).slice(0, MAX_SIGNATURE_LINES),
+        originalChars: text.length,
+      },
+    });
+    cache.push({ id: memory.id, summary, signature });
+    if (cache.length > MAX_CACHED_ENTRIES) cache.shift();
+  } catch (err) {
+    logger.warn({ err }, "[wiki] Failed to register wiki entry");
+  }
+
+  return null;
+}
+
+/**
+ * Replace large repeated blocks inside old-history messages with wiki
+ * references. Only text and tool_result content is touched.
+ *
+ * @param {Array} messages - Old (about-to-be-summarized) messages
+ * @returns {{messages: Array, stats: {registered: number, dereferenced: number, charsSaved: number}}}
+ */
+function dereferenceMessages(messages) {
+  const stats = { registered: 0, dereferenced: 0, charsSaved: 0 };
+  if (wikiConfig().enabled === false || !messages?.length) {
+    return { messages: messages || [], stats };
+  }
+
+  const processText = (text) => {
+    const before = loadCache().length;
+    const result = registerOrDereference(text);
+    if (result) {
+      stats.dereferenced++;
+      stats.charsSaved += result.saved;
+      return result.ref;
+    }
+    if (loadCache().length > before) stats.registered++;
+    return text;
+  };
+
+  const processed = messages.map(msg => {
+    if (typeof msg.content === "string") {
+      if (msg.content.length < minChars()) return msg;
+      return { ...msg, content: processText(msg.content) };
+    }
+    if (!Array.isArray(msg.content)) return msg;
+
+    const newContent = msg.content.map(block => {
+      if (block.type === "text" && block.text?.length >= minChars()) {
+        return { ...block, text: processText(block.text) };
+      }
+      if (block.type === "tool_result" && typeof block.content === "string" &&
+          block.content.length >= minChars()) {
+        return { ...block, content: processText(block.content) };
+      }
+      return block;
+    });
+    return { ...msg, content: newContent };
+  });
+
+  return { messages: processed, stats };
+}
+
+/** Reset the in-memory cache (test support). */
+function resetCache() {
+  entryCache = null;
+}
+
+module.exports = {
+  registerOrDereference,
+  dereferenceMessages,
+  makeSummary,
+  resetCache,
+};