lynkr 9.10.0 → 9.12.0

package/src/memory/skills-cache.js

@@ -0,0 +1,180 @@
+/**
+ * Skills Cache — Compression Strategy Memory (TencentDB-Agent-Memory inspired)
+ *
+ * Remembers which compression method worked (and how well) for each
+ * structural shape of tool output. Validated skills (savings >= minSavings)
+ * persist in the memories table (type='skill') and let future compressions:
+ *  - skip section-dedup for shapes where it never helps (latency win)
+ *  - compress proven-compressible shapes more aggressively (token win)
+ */
+
+const store = require("./store");
+const distill = require("../context/distill");
+const config = require("../config");
+const logger = require("../logger");
+
+const MAX_CACHED_SKILLS = 1000;
+const SIGNATURE_LINES = 30;
+
+// signature -> { method, avgSavings, hits, persistedId }
+let skillMap = null;
+
+function skillsConfig() {
+  return config.memory?.skills ?? {};
+}
+
+/**
+ * Structural shape signature for a tool output: leading token of each of
+ * the first N normalized lines. Two grep outputs, two test runs, or two
+ * JSON blobs with the same shape produce the same signature even when the
+ * values differ.
+ */
+function shapeSignature(text) {
+  if (!text) return "empty";
+  const lines = distill.normalizeText(text).split("\n").slice(0, SIGNATURE_LINES);
+  const shape = lines
+    .map(l => {
+      const trimmed = l.trim();
+      if (!trimmed) return "";
+      // Leading structural token: punctuation kept, words folded
+      const lead = trimmed.match(/^[{}[\]"'\-+*#>|]|^[A-Za-z_$]+|^\d+/);
+      return lead ? (/^\d+$/.test(lead[0]) ? "N" : lead[0]) : trimmed[0];
+    })
+    .join(",");
+  // djb2 hash keeps keys short
+  let hash = 5381;
+  for (let i = 0; i < shape.length; i++) {
+    hash = ((hash << 5) + hash + shape.charCodeAt(i)) | 0;
+  }
+  return `s${lines.length}_${(hash >>> 0).toString(36)}`;
+}
+
+function loadCache() {
+  if (skillMap) return skillMap;
+
+  skillMap = new Map();
+  try {
+    const rows = store.getMemoriesByType("skill", MAX_CACHED_SKILLS);
+    for (const row of rows) {
+      const meta = typeof row.metadata === "string" ? JSON.parse(row.metadata) : row.metadata;
+      if (!meta?.signature) continue;
+      skillMap.set(meta.signature, {
+        method: meta.method,
+        avgSavings: meta.avgSavings ?? 0,
+        hits: meta.hits ?? 1,
+        persistedId: row.id,
+      });
+    }
+  } catch (err) {
+    logger.warn({ err }, "[skills] Failed to load skills, starting empty");
+  }
+  return skillMap;
+}
+
+/**
+ * Look up a known compression skill for a shape signature.
+ * @returns {{method: string, avgSavings: number, hits: number}|null}
+ */
+function lookup(signature) {
+  if (skillsConfig().enabled === false) return null;
+  return loadCache().get(signature) ?? null;
+}
+
+/**
+ * Record a compression outcome. Kept in memory always (so known-futile
+ * shapes can skip dedup work); persisted only when the running average
+ * savings clears minSavings — the "validated skill" bar.
+ *
+ * @param {string} signature - shapeSignature() of the original text
+ * @param {string} method - compression method that ran ('delta'|'distill'|'passthrough')
+ * @param {number} savings - fraction saved, 0..1
+ */
+function record(signature, method, savings) {
+  if (skillsConfig().enabled === false || !signature) return;
+
+  const cache = loadCache();
+  const existing = cache.get(signature);
+  const entry = existing
+    ? {
+        ...existing,
+        method,
+        hits: existing.hits + 1,
+        avgSavings: (existing.avgSavings * existing.hits + savings) / (existing.hits + 1),
+      }
+    : { method, avgSavings: savings, hits: 1, persistedId: null };
+
+  cache.set(signature, entry);
+  if (cache.size > MAX_CACHED_SKILLS) {
+    cache.delete(cache.keys().next().value);
+  }
+
+  const minSavings = skillsConfig().minSavings ?? 0.6;
+  if (entry.avgSavings < minSavings) return;
+
+  // Persist validated skill off the hot path
+  setImmediate(() => {
+    try {
+      const metadata = {
+        signature,
+        method: entry.method,
+        avgSavings: entry.avgSavings,
+        hits: entry.hits,
+      };
+      if (entry.persistedId) {
+        const current = store.getMemory(entry.persistedId);
+        if (current) {
+          store.updateMemory(entry.persistedId, { metadata });
+          return;
+        }
+      }
+      const memory = store.createMemory({
+        sessionId: null,
+        content: `Compression skill: ${entry.method} saves ${(entry.avgSavings * 100).toFixed(0)}% on shape ${signature}`,
+        type: "skill",
+        category: "optimization",
+        importance: 0.3,
+        metadata,
+      });
+      entry.persistedId = memory.id;
+    } catch (err) {
+      logger.warn({ err, signature }, "[skills] Failed to persist skill");
+    }
+  });
+}
+
+/**
+ * Compression hints for a given text, derived from validated skills.
+ *
+ * @param {string} text - Tool result text about to be compressed
+ * @returns {{signature: string, skipDedup: boolean, maxLengthFactor: number}}
+ */
+function getCompressionHints(text) {
+  const signature = shapeSignature(text);
+  const skill = lookup(signature);
+  const minSavings = skillsConfig().minSavings ?? 0.6;
+
+  if (!skill || skill.hits < 2) {
+    return { signature, skipDedup: false, maxLengthFactor: 1 };
+  }
+
+  return {
+    signature,
+    // Dedup/delta repeatedly achieved almost nothing — don't burn CPU on it
+    skipDedup: skill.method === "passthrough" || skill.avgSavings < 0.05,
+    // Proven highly-compressible shape — compress harder
+    maxLengthFactor: skill.avgSavings >= minSavings ? 0.8 : 1,
+  };
+}
+
+/** Reset the in-memory cache (test support). */
+function resetCache() {
+  skillMap = null;
+}
+
+module.exports = {
+  shapeSignature,
+  lookup,
+  record,
+  getCompressionHints,
+  resetCache,
+};

package/src/memory/tencentdb-launcher.js

@@ -0,0 +1,459 @@
+/**
+ * TencentDB-Agent-Memory Sidecar Launcher
+ *
+ * Launches the TencentDB-Agent-Memory stack (memory-core gateway +
+ * memory-hub panel/knowledge services) via dockerode when
+ * TENCENTDB_MEMORY_ENABLED=true — the same in-process pattern as the
+ * Headroom sidecar launcher, extended to a two-container stack.
+ *
+ * Mirrors the project's own deploy/global-images/start-*.sh scripts:
+ *  - shared docker network `tdai-memory-stack` with service aliases
+ *  - memory-core config generated locally and mounted read-only
+ *  - first-boot admin user init, key persisted to data/tencentdb-memory/
+ *
+ * The memory services need an LLM for extraction/summarization; by default
+ * that points back at Lynkr's own OpenAI-compatible endpoint, so the whole
+ * stack runs on whatever providers Lynkr already has.
+ *
+ * Containers use restart=unless-stopped and are intentionally NOT stopped
+ * when Lynkr exits — a team memory hub outlives any one proxy process.
+ * Remove with: docker rm -f tdai-memory-core tdai-memory-hub
+ */
+
+let Docker;
+try {
+  Docker = require("dockerode");
+} catch {
+  Docker = null;
+}
+const path = require("path");
+const fs = require("fs");
+const crypto = require("crypto");
+const logger = require("../logger");
+const config = require("../config");
+
+const docker = Docker ? new Docker() : null;
+
+const DATA_DIR = path.join(process.cwd(), "data", "tencentdb-memory");
+const ADMIN_KEY_FILE = path.join(DATA_DIR, ".admin-key");
+const CORE_CONFIG_FILE = path.join(DATA_DIR, "tdai-gateway.yaml");
+
+let isStarting = false;
+
+function sidecarConfig() {
+  return config.tencentdbMemory ?? {};
+}
+
+/**
+ * Gateway config for memory-core, matching the template the project's
+ * start-memory-core.sh generates. Mounted read-only into the container.
+ */
+function buildCoreConfigYaml() {
+  const { llm, promptMode } = sidecarConfig();
+  return `# Generated by Lynkr's TencentDB memory launcher — overwritten on every start.
+deployMode: standalone
+stateBackend: local
+
+server:
+  port: 8420
+  host: 0.0.0.0
+
+data:
+  baseDir: /data/tdai-memory
+
+llm:
+  baseUrl: "${llm.baseUrl}"
+  apiKey: "${llm.apiKey}"
+  model: "${llm.model}"
+  maxTokens: 32000
+  timeoutMs: 300000
+
+memory:
+  promptMode: ${promptMode}
+  capture: { enabled: true }
+  extraction:
+    enabled: true
+    enableDedup: true
+    maxMemoriesPerSession: 20
+  persona:
+    triggerEveryN: 50
+    maxScenes: 15
+  pipeline:
+    everyNConversations: 5
+    enableWarmup: true
+    l1IdleTimeoutSeconds: 600
+    l2DelayAfterL1Seconds: 90
+    l2MinIntervalSeconds: 900
+    l2MaxIntervalSeconds: 3600
+  recall:
+    enabled: true
+    maxResults: 5
+    scoreThreshold: 0.3
+    strategy: hybrid
+    timeoutMs: 5000
+  storeBackend: sqlite
+  embedding:
+    provider: none
+
+skill:
+  enabled: true
+  routing:
+    mode: bm25
+    searchTopK: 20
+  extraction:
+    enabled: true
+    maxIterations: 16
+    queue:
+      backend: local
+      keyPrefix: tdai
+      resultTtlSeconds: 86400
+      lockTtlMs: 600000
+      maxRetries: 2
+      retryBackoffsMs: [5000, 15000]
+  resources:
+    maxResourceSizeBytes: 5000000
+`;
+}
+
+/**
+ * sk-mem-<32 alphanumeric chars> — the user-key format memory-core expects.
+ */
+function generateUserKey() {
+  let raw = "";
+  while (raw.length < 32) {
+    raw += crypto.randomBytes(48).toString("base64").replace(/[^A-Za-z0-9]/g, "");
+  }
+  return `sk-mem-${raw.slice(0, 32)}`;
+}
+
+function maskKey(key) {
+  if (!key || key.length < 16) return "****";
+  return `${key.slice(0, 11)}****${key.slice(-4)}`;
+}
+
+async function ensureNetwork(name) {
+  try {
+    await docker.getNetwork(name).inspect();
+  } catch (err) {
+    if (err.statusCode !== 404) throw err;
+    logger.info({ network: name }, "[tdai] Creating docker network");
+    await docker.createNetwork({ Name: name, CheckDuplicate: true });
+  }
+}
+
+async function getExistingContainer(containerName) {
+  const containers = await docker.listContainers({
+    all: true,
+    filters: { name: [containerName] },
+  });
+  const match = containers.find(
+    (c) => c.Names.includes(`/${containerName}`) || c.Names.includes(containerName)
+  );
+  return match ? docker.getContainer(match.Id) : null;
+}
+
+async function imageExists(imageName) {
+  try {
+    await docker.getImage(imageName).inspect();
+    return true;
+  } catch (err) {
+    if (err.statusCode === 404) return false;
+    throw err;
+  }
+}
+
+function pullImage(imageName) {
+  logger.info({ image: imageName }, "[tdai] Pulling image (first start may take a few minutes)");
+  return new Promise((resolve, reject) => {
+    docker.pull(imageName, (err, stream) => {
+      if (err) return reject(err);
+      docker.modem.followProgress(stream, (progressErr, output) => {
+        if (progressErr) return reject(progressErr);
+        logger.info({ image: imageName }, "[tdai] Image pull complete");
+        resolve(output);
+      });
+    });
+  });
+}
+
+/**
+ * Poll an HTTP endpoint until it responds (any status < 500).
+ */
+async function waitForHttp(url, { maxRetries = 90, intervalMs = 1000, label } = {}) {
+  for (let i = 0; i < maxRetries; i++) {
+    try {
+      const response = await fetch(url, { signal: AbortSignal.timeout(2000) });
+      if (response.status < 500) return true;
+    } catch {
+      // not up yet
+    }
+    if (i % 15 === 14) {
+      logger.debug({ url, attempt: i + 1, maxRetries }, `[tdai] Waiting for ${label ?? url}`);
+    }
+    await new Promise((resolve) => setTimeout(resolve, intervalMs));
+  }
+  throw new Error(`${label ?? url} did not become healthy after ${maxRetries}s`);
+}
+
+/**
+ * Create+start one container, or start it if it exists but is stopped.
+ * Returns the action taken.
+ */
+async function ensureContainer(spec) {
+  const existing = await getExistingContainer(spec.name);
+
+  if (existing) {
+    const info = await existing.inspect();
+    if (info.State.Running) return "existing_running";
+    logger.info({ container: spec.name }, "[tdai] Starting existing container");
+    await existing.start();
+    return "started_existing";
+  }
+
+  if (!(await imageExists(spec.image))) {
+    await pullImage(spec.image);
+  }
+
+  logger.info({ container: spec.name, image: spec.image }, "[tdai] Creating container");
+  const container = await docker.createContainer({
+    Image: spec.image,
+    name: spec.name,
+    Env: spec.env,
+    ExposedPorts: Object.fromEntries(
+      Object.values(spec.ports).map((p) => [`${p.container}/tcp`, {}])
+    ),
+    HostConfig: {
+      PortBindings: Object.fromEntries(
+        Object.values(spec.ports).map((p) => [
+          `${p.container}/tcp`,
+          [{ HostPort: String(p.host) }],
+        ])
+      ),
+      Binds: spec.binds,
+      NetworkMode: spec.network,
+      ExtraHosts: ["host.docker.internal:host-gateway"],
+      RestartPolicy: { Name: "unless-stopped" },
+    },
+    NetworkingConfig: {
+      EndpointsConfig: {
+        [spec.network]: { Aliases: [spec.alias] },
+      },
+    },
+  });
+  await container.start();
+  return "created_new";
+}
+
+/**
+ * First-boot admin bootstrap, mirroring start-memory-core.sh:
+ * generate an sk-mem- key (or reuse the persisted one), init-admin with it,
+ * verify, and persist to .admin-key with tight permissions.
+ */
+async function ensureAdminUser(gatewayUrl) {
+  let adminKey;
+  if (fs.existsSync(ADMIN_KEY_FILE)) {
+    adminKey = fs.readFileSync(ADMIN_KEY_FILE, "utf8").trim();
+  } else {
+    adminKey = generateUserKey();
+  }
+
+  const response = await fetch(`${gatewayUrl}/v3/internal/meta/user/init-admin`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "x-tdai-service-id": "default",
+    },
+    body: JSON.stringify({ username: "admin", user_key: adminKey }),
+    signal: AbortSignal.timeout(10000),
+  });
+
+  if (response.status === 200) {
+    fs.writeFileSync(ADMIN_KEY_FILE, adminKey, { mode: 0o600 });
+    logger.info({ key: maskKey(adminKey), keyFile: ADMIN_KEY_FILE }, "[tdai] Admin user created");
+  } else if (response.status === 409) {
+    if (!fs.existsSync(ADMIN_KEY_FILE)) {
+      logger.warn(
+        "[tdai] Admin user already exists but .admin-key is missing — the key cannot be recovered. " +
+        "Reset with: docker rm -f tdai-memory-core && docker volume rm tdai-memory-core-data"
+      );
+      return null;
+    }
+  } else {
+    logger.warn({ status: response.status }, "[tdai] init-admin returned unexpected status");
+    return null;
+  }
+
+  // Verify the key actually authenticates
+  try {
+    const verify = await fetch(`${gatewayUrl}/v3/meta/auth/verify`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "x-tdai-service-id": "default",
+      },
+      body: JSON.stringify({ user_key: adminKey }),
+      signal: AbortSignal.timeout(5000),
+    });
+    if (verify.status !== 200) {
+      logger.warn({ status: verify.status }, "[tdai] Admin key failed verification — volume and .admin-key may be out of sync");
+      return null;
+    }
+  } catch (err) {
+    logger.debug({ err: err.message }, "[tdai] Admin key verification skipped");
+  }
+
+  return adminKey;
+}
+
+/**
+ * Ensure the full memory stack (core + hub) is running.
+ * Idempotent: containers started by the project's own scripts are detected
+ * by name and reused.
+ */
+async function ensureRunning() {
+  const cfg = sidecarConfig();
+
+  if (!cfg.enabled) return { started: false, reason: "disabled" };
+  if (!cfg.docker?.enabled) return { started: false, reason: "docker_disabled" };
+  if (!docker) return { started: false, reason: "dockerode_unavailable" };
+  if (isStarting) return { started: false, reason: "already_starting" };
+
+  isStarting = true;
+  try {
+    const { network, core, hub } = cfg.docker;
+    const gatewayUrl = `http://localhost:${core.port}`;
+
+    await ensureNetwork(network);
+
+    // ── memory-core ──
+    fs.mkdirSync(DATA_DIR, { recursive: true });
+    fs.writeFileSync(CORE_CONFIG_FILE, buildCoreConfigYaml());
+
+    const coreAction = await ensureContainer({
+      name: core.containerName,
+      image: core.image,
+      alias: "memory-core",
+      network,
+      ports: { gateway: { host: core.port, container: 8420 } },
+      binds: [
+        `${core.volume}:/data/tdai-memory`,
+        `${CORE_CONFIG_FILE}:/data/config/tdai-gateway.yaml:ro`,
+      ],
+      env: [
+        "TDAI_GATEWAY_PORT=8420",
+        "TDAI_GATEWAY_HOST=0.0.0.0",
+        "TDAI_GATEWAY_API_KEY=", // empty = no bearer gate (local deployment)
+        "TDAI_DATA_DIR=/data/tdai-memory",
+      ],
+    });
+    await waitForHttp(`${gatewayUrl}/`, { label: "memory-core" });
+    logger.info({ action: coreAction, url: gatewayUrl }, "[tdai] memory-core ready");
+
+    const adminKey = await ensureAdminUser(gatewayUrl);
+
+    // ── memory-hub (panel + knowledge) ──
+    const { llm } = cfg;
+    const hubAction = await ensureContainer({
+      name: hub.containerName,
+      image: hub.image,
+      alias: "memory-hub",
+      network,
+      ports: {
+        panel: { host: hub.panelPort, container: 8125 },
+        knowledge: { host: hub.knowledgePort, container: 8424 },
+      },
+      binds: [`${hub.volume}:/data/knowledge`],
+      env: [
+        "PANEL_PORT=8125",
+        "KNOWLEDGE_PORT=8424",
+        `KNOWLEDGE_PUBLIC_BASE_URL=http://host.docker.internal:${hub.knowledgePort}/v3`,
+        "REMOTE_INSTANCE_ID=default",
+        "REMOTE_INSTANCE_NAME=default",
+        "REMOTE_INSTANCE_URL=http://memory-core:8420",
+        "REMOTE_INSTANCE_KEY=local",
+        // Panel's "client base URL" card points at Lynkr — Lynkr is the proxy here
+        `REMOTE_INSTANCE_PROXY_URL=http://localhost:${config.port}`,
+        "LLM_MODE=custom",
+        `LLM_PROTOCOL=${llm.protocol}`,
+        `LLM_API_KEY=${llm.apiKey}`,
+        `LLM_BASE_URL=${llm.baseUrl}`,
+        `LLM_MODEL=${llm.model}`,
+        "KNOWLEDGE_LLM_BINDING_SYNC=0",
+      ],
+    });
+    await waitForHttp(`http://localhost:${hub.knowledgePort}/health`, { label: "memory-hub", maxRetries: 120 });
+    logger.info({ action: hubAction }, "[tdai] memory-hub ready");
+
+    return {
+      started: true,
+      actions: { core: coreAction, hub: hubAction },
+      adminKey: adminKey ? maskKey(adminKey) : null,
+      endpoints: {
+        gateway: gatewayUrl,
+        panel: `http://localhost:${hub.panelPort}/`,
+        knowledge: `http://localhost:${hub.knowledgePort}/v3`,
+      },
+    };
+  } finally {
+    isStarting = false;
+  }
+}
+
+/**
+ * Stop both containers (kept on disk; volumes untouched).
+ * Not called automatically on Lynkr shutdown — see module header.
+ */
+async function stop() {
+  if (!docker) return;
+  const { core, hub } = sidecarConfig().docker ?? {};
+  for (const name of [hub?.containerName, core?.containerName].filter(Boolean)) {
+    try {
+      const container = await getExistingContainer(name);
+      if (container) {
+        const info = await container.inspect();
+        if (info.State.Running) {
+          logger.info({ container: name }, "[tdai] Stopping container");
+          await container.stop({ t: 10 });
+        }
+      }
+    } catch (err) {
+      logger.warn({ err, container: name }, "[tdai] Failed to stop container");
+    }
+  }
+}
+
+async function getStatus() {
+  if (!docker) return { available: false };
+  const { core, hub } = sidecarConfig().docker ?? {};
+  const status = { available: true, containers: {} };
+  for (const [key, name] of [["core", core?.containerName], ["hub", hub?.containerName]]) {
+    if (!name) continue;
+    try {
+      const container = await getExistingContainer(name);
+      if (!container) {
+        status.containers[key] = { exists: false, running: false };
+        continue;
+      }
+      const info = await container.inspect();
+      status.containers[key] = {
+        exists: true,
+        running: info.State.Running,
+        status: info.State.Status,
+        image: info.Config.Image,
+      };
+    } catch (err) {
+      status.containers[key] = { exists: false, running: false, error: err.message };
+    }
+  }
+  return status;
+}
+
+module.exports = {
+  ensureRunning,
+  stop,
+  getStatus,
+  // exported for tests
+  buildCoreConfigYaml,
+  generateUserKey,
+  maskKey,
+};