@xdarkicex/openclaw-memory-libravdb 1.3.5

package/src/rpc.ts

@@ -0,0 +1,84 @@
+import type { RpcCallOptions, SidecarSocket } from "./types.js";
+
+interface PendingCall {
+  resolve(value: unknown): void;
+  reject(error: Error): void;
+  timer: ReturnType<typeof setTimeout>;
+}
+
+export class RpcClient {
+  private seq = 0;
+  private readonly pending = new Map<number, PendingCall>();
+  private buf = "";
+
+  constructor(
+    private readonly socket: SidecarSocket,
+    private readonly options: RpcCallOptions,
+  ) {
+    socket.setEncoding("utf8");
+    socket.on("data", (chunk) => this.handleData(chunk));
+    socket.on("close", () => this.rejectAll(new Error("Socket closed")));
+  }
+
+  async call<T>(method: string, _params: unknown): Promise<T> {
+    return await new Promise<T>((resolve, reject) => {
+      const id = ++this.seq;
+      const timer = setTimeout(() => {
+        this.pending.delete(id);
+        reject(new Error(`RPC timeout: ${method} (${this.options.timeoutMs}ms)`));
+      }, this.options.timeoutMs);
+
+      this.pending.set(id, { resolve, reject, timer });
+      this.socket.write(
+        `${JSON.stringify({ jsonrpc: "2.0", id, method, params: _params })}\n`,
+      );
+    });
+  }
+
+  private handleData(chunk: string): void {
+    this.buf += chunk;
+    const lines = this.buf.split("\n");
+    this.buf = lines.pop() ?? "";
+
+    for (const line of lines) {
+      if (!line.trim()) {
+        continue;
+      }
+
+      try {
+        const msg = JSON.parse(line) as {
+          id?: number;
+          result?: unknown;
+          error?: { message?: string };
+        };
+        if (typeof msg.id !== "number") {
+          continue;
+        }
+
+        const pending = this.pending.get(msg.id);
+        if (!pending) {
+          continue;
+        }
+
+        clearTimeout(pending.timer);
+        this.pending.delete(msg.id);
+
+        if (msg.error?.message) {
+          pending.reject(new Error(msg.error.message));
+        } else {
+          pending.resolve(msg.result);
+        }
+      } catch {
+        // Ignore malformed frames and keep parsing future lines.
+      }
+    }
+  }
+
+  private rejectAll(error: Error): void {
+    for (const [id, pending] of this.pending.entries()) {
+      clearTimeout(pending.timer);
+      this.pending.delete(id);
+      pending.reject(error);
+    }
+  }
+}

package/src/scoring.ts

@@ -0,0 +1,58 @@
+import type { SearchResult } from "./types.js";
+
+interface HybridOptions {
+  alpha?: number;
+  beta?: number;
+  gamma?: number;
+  delta?: number;
+  recencyLambdaSession?: number;
+  recencyLambdaUser?: number;
+  recencyLambdaGlobal?: number;
+  sessionId: string;
+  userId: string;
+}
+
+export function scoreCandidates(items: SearchResult[], opts: HybridOptions): SearchResult[] {
+  const now = Date.now();
+  const alpha = opts.alpha ?? 0.7;
+  const beta = opts.beta ?? 0.2;
+  const gamma = opts.gamma ?? 0.1;
+  const delta = opts.delta ?? 0.5;
+  // Lambda units are per-second decay constants.
+  const recencyLambdaSession = opts.recencyLambdaSession ?? 0.0001;
+  const recencyLambdaUser = opts.recencyLambdaUser ?? 0.00001;
+  const recencyLambdaGlobal = opts.recencyLambdaGlobal ?? 0.000002;
+
+  return items
+    .map((item) => {
+      const ts = typeof item.metadata.ts === "number" ? item.metadata.ts : now;
+      const lambda =
+        item.metadata.sessionId === opts.sessionId ? recencyLambdaSession
+          : item.metadata.userId === opts.userId ? recencyLambdaUser
+            : recencyLambdaGlobal;
+      const ageSeconds = Math.max(0, now - ts) / 1000;
+      const recency = Math.exp(-lambda * ageSeconds);
+      const scopeBoost =
+        item.metadata.sessionId === opts.sessionId ? 1.0
+          : item.metadata.userId === opts.userId ? 0.6
+            : 0.3;
+      const baseScore =
+        alpha * item.score +
+        beta * recency +
+        gamma * scopeBoost;
+      const rawDecayRate =
+        typeof item.metadata.decay_rate === "number" ? item.metadata.decay_rate : 0.0;
+      const decayRate = Math.min(1, Math.max(0, rawDecayRate));
+      const quality =
+        item.metadata.type === "summary"
+          ? 1.0 - delta * decayRate
+          : 1.0;
+      const finalScore = baseScore * quality;
+
+      return {
+        ...item,
+        finalScore,
+      };
+    })
+    .sort((a, b) => (b.finalScore ?? 0) - (a.finalScore ?? 0));
+}

package/src/sidecar.ts

@@ -0,0 +1,506 @@
+import { spawn } from "node:child_process";
+import { existsSync } from "node:fs";
+import net from "node:net";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import type { LoggerLike, PluginConfig, SidecarHandle, SidecarSocket } from "./types.js";
+
+type CloseHandler = () => void;
+type DataHandler = (chunk: string) => void;
+type ErrorHandler = (error: Error) => void;
+
+export interface SidecarRuntime {
+  prepareLaunch?(cfg: PluginConfig, env: Record<string, string>): void | Promise<void>;
+  resolveEndpoint(cfg: PluginConfig): string | Promise<string>;
+  createSocket(endpoint: string): SidecarSocket;
+  scheduleRestart(delayMs: number, restart: () => void): void;
+  stop?(): void | Promise<void>;
+}
+
+type HostSignal = "exit" | "SIGINT" | "SIGTERM" | "SIGHUP";
+type HostProcessLike = {
+  once(event: HostSignal, handler: () => void): void;
+  off?(event: HostSignal, handler: () => void): void;
+};
+
+class PlaceholderSocket implements SidecarSocket {
+  private readonly onData = new Set<DataHandler>();
+  private readonly onClose = new Set<CloseHandler>();
+  private readonly connectOnce = new Set<CloseHandler>();
+  private readonly errorOnce = new Set<ErrorHandler>();
+
+  constructor() {
+    queueMicrotask(() => {
+      for (const handler of this.connectOnce) {
+        handler();
+      }
+      this.connectOnce.clear();
+    });
+  }
+
+  setEncoding(_encoding: string): void {}
+
+  on(event: "data" | "close", handler: DataHandler | CloseHandler): void {
+    if (event === "data") {
+      this.onData.add(handler as DataHandler);
+      return;
+    }
+    this.onClose.add(handler as CloseHandler);
+  }
+
+  once(event: "connect" | "error", handler: CloseHandler | ErrorHandler): void {
+    if (event === "connect") {
+      this.connectOnce.add(handler as CloseHandler);
+      return;
+    }
+    this.errorOnce.add(handler as ErrorHandler);
+  }
+
+  write(chunk: string): void {
+    try {
+      const msg = JSON.parse(chunk);
+      const response = JSON.stringify({
+        jsonrpc: "2.0",
+        id: msg.id,
+        result: msg.method === "health" ? { ok: true } : {},
+      });
+      for (const handler of this.onData) {
+        handler(`${response}\n`);
+      }
+    } catch (error) {
+      const err = error instanceof Error ? error : new Error(String(error));
+      for (const handler of this.errorOnce) {
+        handler(err);
+      }
+      this.errorOnce.clear();
+    }
+  }
+
+  destroy(): void {
+    for (const handler of this.onClose) {
+      handler();
+    }
+  }
+}
+
+class SupervisorSocket implements SidecarSocket {
+  private readonly onData = new Set<DataHandler>();
+  private readonly onClose = new Set<CloseHandler>();
+  private readonly connectOnce = new Set<CloseHandler>();
+  private readonly errorOnce = new Set<ErrorHandler>();
+  private current?: SidecarSocket;
+  private encoding = "utf8";
+  private generation = 0;
+
+  bind(socket: SidecarSocket): void {
+    this.current = socket;
+    this.generation += 1;
+    const generation = this.generation;
+
+    socket.setEncoding(this.encoding);
+    socket.on("data", (chunk) => {
+      if (generation !== this.generation) {
+        return;
+      }
+      for (const handler of this.onData) {
+        handler(chunk);
+      }
+    });
+    socket.on("close", () => {
+      if (generation !== this.generation) {
+        return;
+      }
+      for (const handler of this.onClose) {
+        handler();
+      }
+    });
+
+    for (const handler of this.connectOnce) {
+      handler();
+    }
+    this.connectOnce.clear();
+  }
+
+  setEncoding(encoding: string): void {
+    this.encoding = encoding;
+    this.current?.setEncoding(encoding);
+  }
+
+  on(event: "data" | "close", handler: DataHandler | CloseHandler): void {
+    if (event === "data") {
+      this.onData.add(handler as DataHandler);
+      return;
+    }
+    this.onClose.add(handler as CloseHandler);
+  }
+
+  once(event: "connect" | "error", handler: CloseHandler | ErrorHandler): void {
+    if (event === "connect") {
+      if (this.current) {
+        (handler as CloseHandler)();
+        return;
+      }
+      this.connectOnce.add(handler as CloseHandler);
+      return;
+    }
+    this.errorOnce.add(handler as ErrorHandler);
+  }
+
+  write(chunk: string): void {
+    if (!this.current) {
+      throw new Error("Sidecar socket unavailable");
+    }
+    this.current.write(chunk);
+  }
+
+  destroy(): void {
+    this.current?.destroy();
+  }
+}
+
+class SidecarSupervisor implements SidecarHandle {
+  private retries = 0;
+  private degraded = false;
+  private shuttingDown = false;
+  public socket: SidecarSocket;
+
+  constructor(
+    private readonly cfg: PluginConfig,
+    private readonly logger: LoggerLike,
+    private readonly runtime: SidecarRuntime,
+  ) {
+    this.socket = new SupervisorSocket();
+  }
+
+  async start(): Promise<SidecarSocket> {
+    const endpoint = await this.spawnEndpoint();
+    const socket = await this.connectEndpoint(endpoint);
+    if (this.socket instanceof SupervisorSocket) {
+      this.socket.bind(socket);
+    } else {
+      this.socket = socket;
+    }
+    return socket;
+  }
+
+  isDegraded(): boolean {
+    return this.degraded;
+  }
+
+  async shutdown(): Promise<void> {
+    this.shuttingDown = true;
+    await this.runtime.stop?.();
+    this.socket.destroy();
+  }
+
+  private async spawnEndpoint(): Promise<string> {
+    const env = buildSidecarEnv(this.cfg);
+    await this.runtime.prepareLaunch?.(this.cfg, env);
+    return await this.runtime.resolveEndpoint(this.cfg);
+  }
+
+  private async connectEndpoint(endpoint: string): Promise<SidecarSocket> {
+    const socket = this.runtime.createSocket(endpoint);
+    socket.on("close", () => {
+      void this.handleExit(1);
+    });
+
+    if (isTcpEndpoint(endpoint)) {
+      this.logger.info?.(`[libravdb] using TCP endpoint ${endpoint}`);
+    }
+
+    return await new Promise<SidecarSocket>((resolve, reject) => {
+      socket.once("connect", () => resolve(socket));
+      socket.once("error", reject);
+    });
+  }
+
+  private async handleExit(code: number | null): Promise<void> {
+    if (this.shuttingDown) {
+      return;
+    }
+    if (code === 0) {
+      return;
+    }
+
+    const maxRetries = this.cfg.maxRetries ?? 3;
+    if (this.retries >= maxRetries) {
+      this.logger.error("[libravdb] sidecar retries exhausted; degraded mode");
+      this.degraded = true;
+      return;
+    }
+
+    const backoffMs = computeBackoffMs(this.retries);
+    this.retries += 1;
+    this.runtime.scheduleRestart(backoffMs, () => {
+      void this.start().catch((error) => {
+        const message = error instanceof Error ? error.message : String(error);
+        this.logger.error(`[libravdb] sidecar restart failed: ${message}`);
+      });
+    });
+  }
+}
+
+export async function startSidecar(
+  cfg: PluginConfig,
+  logger: LoggerLike = console,
+  runtime: SidecarRuntime = createDefaultRuntime(logger),
+): Promise<SidecarHandle> {
+  const supervisor = new SidecarSupervisor(cfg, logger, runtime);
+  await supervisor.start();
+  return supervisor;
+}
+
+export function computeBackoffMs(retries: number): number {
+  return Math.min(500 * Math.pow(2, retries), 16000);
+}
+
+export function isTcpEndpoint(endpoint: string): boolean {
+  return endpoint.startsWith("tcp:");
+}
+
+export function resolveEndpoint(cfg: PluginConfig): string {
+  const endpoint = isConfiguredEndpoint(cfg.sidecarPath)
+    ? (cfg.sidecarPath ?? "unix:/tmp/libravdb.sock")
+    : "unix:/tmp/libravdb.sock";
+
+  return endpoint.replace(/^unix:/, "");
+}
+
+export function buildSidecarEnv(cfg: PluginConfig): Record<string, string> {
+  const env: Record<string, string> = {};
+
+  if (cfg.dbPath) {
+    env.LIBRAVDB_DB_PATH = cfg.dbPath;
+  }
+  if (cfg.embeddingRuntimePath) {
+    env.LIBRAVDB_ONNX_RUNTIME = cfg.embeddingRuntimePath;
+  }
+  if (cfg.embeddingBackend) {
+    env.LIBRAVDB_EMBEDDING_BACKEND = cfg.embeddingBackend;
+  }
+  if (cfg.embeddingProfile) {
+    env.LIBRAVDB_EMBEDDING_PROFILE = cfg.embeddingProfile;
+  }
+  if (cfg.fallbackProfile) {
+    env.LIBRAVDB_FALLBACK_PROFILE = cfg.fallbackProfile;
+  }
+  if (cfg.embeddingModelPath) {
+    env.LIBRAVDB_EMBEDDING_MODEL = cfg.embeddingModelPath;
+  }
+  if (cfg.embeddingTokenizerPath) {
+    env.LIBRAVDB_EMBEDDING_TOKENIZER = cfg.embeddingTokenizerPath;
+  }
+  if (typeof cfg.embeddingDimensions === "number" && cfg.embeddingDimensions > 0) {
+    env.LIBRAVDB_EMBEDDING_DIMENSIONS = String(cfg.embeddingDimensions);
+  }
+  if (typeof cfg.embeddingNormalize === "boolean") {
+    env.LIBRAVDB_EMBEDDING_NORMALIZE = String(cfg.embeddingNormalize);
+  }
+  if (cfg.summarizerBackend) {
+    env.LIBRAVDB_SUMMARIZER_BACKEND = cfg.summarizerBackend;
+  }
+  if (cfg.summarizerProfile) {
+    env.LIBRAVDB_SUMMARIZER_PROFILE = cfg.summarizerProfile;
+  }
+  if (cfg.summarizerRuntimePath) {
+    env.LIBRAVDB_SUMMARIZER_RUNTIME = cfg.summarizerRuntimePath;
+  }
+  if (cfg.summarizerModelPath) {
+    env.LIBRAVDB_SUMMARIZER_MODEL_PATH = cfg.summarizerModelPath;
+  }
+  if (cfg.summarizerTokenizerPath) {
+    env.LIBRAVDB_SUMMARIZER_TOKENIZER = cfg.summarizerTokenizerPath;
+  }
+  if (cfg.summarizerModel) {
+    env.LIBRAVDB_SUMMARIZER_MODEL = cfg.summarizerModel;
+  }
+  if (cfg.summarizerEndpoint) {
+    env.LIBRAVDB_SUMMARIZER_ENDPOINT = cfg.summarizerEndpoint;
+  }
+  if (cfg.ollamaUrl && !env.LIBRAVDB_SUMMARIZER_ENDPOINT) {
+    env.LIBRAVDB_SUMMARIZER_ENDPOINT = cfg.ollamaUrl;
+  }
+  if (cfg.compactModel && !env.LIBRAVDB_SUMMARIZER_MODEL) {
+    env.LIBRAVDB_SUMMARIZER_MODEL = cfg.compactModel;
+  }
+  if (cfg.gatingWeights?.w1c != null) {
+    env.LIBRAVDB_GATING_W1C = String(cfg.gatingWeights.w1c);
+  }
+  if (cfg.gatingWeights?.w2c != null) {
+    env.LIBRAVDB_GATING_W2C = String(cfg.gatingWeights.w2c);
+  }
+  if (cfg.gatingWeights?.w3c != null) {
+    env.LIBRAVDB_GATING_W3C = String(cfg.gatingWeights.w3c);
+  }
+  if (cfg.gatingWeights?.w1t != null) {
+    env.LIBRAVDB_GATING_W1T = String(cfg.gatingWeights.w1t);
+  }
+  if (cfg.gatingWeights?.w2t != null) {
+    env.LIBRAVDB_GATING_W2T = String(cfg.gatingWeights.w2t);
+  }
+  if (cfg.gatingWeights?.w3t != null) {
+    env.LIBRAVDB_GATING_W3T = String(cfg.gatingWeights.w3t);
+  }
+  if (typeof cfg.gatingTechNorm === "number" && cfg.gatingTechNorm > 0) {
+    env.LIBRAVDB_GATING_TECH_NORM = String(cfg.gatingTechNorm);
+  }
+  if (typeof cfg.ingestionGateThreshold === "number" && cfg.ingestionGateThreshold >= 0) {
+    env.LIBRAVDB_GATING_THRESHOLD = String(cfg.ingestionGateThreshold);
+  }
+  if (typeof cfg.gatingCentroidK === "number" && cfg.gatingCentroidK > 0) {
+    env.LIBRAVDB_GATING_CENTROID_K = String(cfg.gatingCentroidK);
+  }
+
+  return env;
+}
+
+export function installSidecarProcessCleanup(host: HostProcessLike, stop: () => void): () => void {
+  const events: HostSignal[] = ["exit", "SIGINT", "SIGTERM", "SIGHUP"];
+  let stopped = false;
+  const stopOnce = () => {
+    if (stopped) {
+      return;
+    }
+    stopped = true;
+    stop();
+  };
+  for (const event of events) {
+    host.once(event, stopOnce);
+  }
+  return () => {
+    for (const event of events) {
+      host.off?.(event, stopOnce);
+    }
+  };
+}
+
+function createDefaultRuntime(logger: LoggerLike): SidecarRuntime {
+  let launchEnv: Record<string, string> = {};
+  let proc: ReturnType<typeof spawn> | null = null;
+  const stopProc = () => {
+    if (!proc || proc.killed) {
+      proc = null;
+      return;
+    }
+    proc.kill();
+    proc = null;
+  };
+  const removeCleanup = installSidecarProcessCleanup(process, stopProc);
+
+  return {
+    prepareLaunch(_cfg, env) {
+      launchEnv = { ...env };
+    },
+    async resolveEndpoint(cfg) {
+      if (isConfiguredEndpoint(cfg.sidecarPath)) {
+        return resolveEndpoint(cfg);
+      }
+
+      const binPath = resolveBinPath(cfg.sidecarPath);
+      return await new Promise<string>((resolve, reject) => {
+        stopProc();
+        const child = spawn(binPath, [], {
+          env: {
+            ...process.env,
+            ...launchEnv,
+          },
+          stdio: ["ignore", "pipe", "pipe"],
+        });
+        proc = child;
+
+        let stdoutBuffer = "";
+        let settled = false;
+
+        child.stdout.setEncoding("utf8");
+        child.stdout.on("data", (chunk: string) => {
+          if (settled) {
+            return;
+          }
+          stdoutBuffer += chunk;
+          const lines = stdoutBuffer.split("\n");
+          stdoutBuffer = lines.pop() ?? "";
+          const endpoint = lines.find((line) => line.trim().length > 0)?.trim();
+          if (!endpoint) {
+            return;
+          }
+          settled = true;
+          resolve(endpoint);
+        });
+
+        child.stderr.setEncoding("utf8");
+        child.stderr.on("data", (chunk: string) => {
+          if (cfg.logLevel === "debug") {
+            process.stderr.write(chunk);
+          } else {
+            logger.info?.(`[libravdb] sidecar: ${chunk.trim()}`);
+          }
+        });
+
+        child.once("error", (error) => {
+          if (settled) {
+            return;
+          }
+          settled = true;
+          reject(error);
+        });
+
+        child.once("exit", (code) => {
+          proc = null;
+          if (settled) {
+            return;
+          }
+          settled = true;
+          reject(new Error(`LibraVDB sidecar exited before advertising an endpoint (code ${code ?? "unknown"})`));
+        });
+      });
+    },
+    createSocket(endpoint) {
+      if (isTcpEndpoint(endpoint)) {
+        const address = endpoint.slice("tcp:".length);
+        const separator = address.lastIndexOf(":");
+        if (separator <= 0) {
+          throw new Error(`Invalid TCP sidecar endpoint: ${endpoint}`);
+        }
+        return net.connect({
+          host: address.slice(0, separator),
+          port: Number(address.slice(separator + 1)),
+        }) as unknown as SidecarSocket;
+      }
+      return net.connect(endpoint) as unknown as SidecarSocket;
+    },
+    scheduleRestart(delayMs, restart) {
+      setTimeout(restart, delayMs);
+    },
+    async stop() {
+      stopProc();
+      removeCleanup();
+    },
+  };
+}
+
+function isConfiguredEndpoint(value?: string): boolean {
+  return value?.startsWith("tcp:") === true || value?.startsWith("unix:") === true;
+}
+
+function resolveBinPath(sidecarPath?: string): string {
+  if (sidecarPath && sidecarPath !== "auto") {
+    return sidecarPath;
+  }
+
+  const binaryName = process.platform === "win32" ? "libravdb-sidecar.exe" : "libravdb-sidecar";
+  const baseDir = path.dirname(fileURLToPath(import.meta.url));
+  const candidates = [
+    path.resolve(baseDir, "../.sidecar-bin", binaryName),
+    path.resolve(baseDir, "../../.sidecar-bin", binaryName),
+  ];
+
+  for (const candidate of candidates) {
+    if (existsSync(candidate)) {
+      return candidate;
+    }
+  }
+
+  return candidates[0];
+}

package/src/tokens.ts

@@ -0,0 +1,36 @@
+import type { SearchResult } from "./types.js";
+
+export function estimateTokens(text: string): number {
+  const charsPerToken = detectCharsPerToken(text);
+  return Math.ceil(text.length / charsPerToken);
+}
+
+export function fitPromptBudget(items: SearchResult[], budget: number): SearchResult[] {
+  const selected: SearchResult[] = [];
+  let used = 0;
+
+  for (const item of items) {
+    const cost = estimateTokens(item.text);
+    if (used + cost > budget) {
+      continue;
+    }
+    selected.push(item);
+    used += cost;
+  }
+
+  return selected;
+}
+
+export function countTokens(messages: Array<{ content: string }>): number {
+  return messages.reduce((sum, msg) => sum + estimateTokens(msg.content), 0);
+}
+
+function detectCharsPerToken(text: string): number {
+  if (/[一-龯ぁ-ゖァ-ヺ가-힣]/u.test(text)) {
+    return 1.6;
+  }
+  if (/[Ѐ-ӿ؀-ۿ֐-׿]/u.test(text)) {
+    return 2.5;
+  }
+  return 4.0;
+}