@martian-engineering/lossless-claw 0.7.0 → 0.8.1

package/src/db/config.ts

@@ -1,345 +0,0 @@
-import { homedir } from "os";
-import { join } from "path";
-
-export type CacheAwareCompactionConfig = {
-  enabled: boolean;
-  maxColdCacheCatchupPasses: number;
-  hotCachePressureFactor: number;
-  hotCacheBudgetHeadroomRatio: number;
-};
-
-export type DynamicLeafChunkTokensConfig = {
-  enabled: boolean;
-  max: number;
-};
-
-export type LcmConfig = {
-  enabled: boolean;
-  databasePath: string;
-  /** Glob patterns for session keys to exclude from LCM storage entirely. */
-  ignoreSessionPatterns: string[];
-  /** Glob patterns for session keys that may read from LCM but never write to it. */
-  statelessSessionPatterns: string[];
-  /** When true, stateless session pattern matching is enforced. */
-  skipStatelessSessions: boolean;
-  contextThreshold: number;
-  freshTailCount: number;
-  newSessionRetainDepth: number;
-  leafMinFanout: number;
-  condensedMinFanout: number;
-  condensedMinFanoutHard: number;
-  incrementalMaxDepth: number;
-  leafChunkTokens: number;
-  /** Maximum raw parent-history tokens imported during first-time bootstrap. */
-  bootstrapMaxTokens?: number;
-  leafTargetTokens: number;
-  condensedTargetTokens: number;
-  maxExpandTokens: number;
-  largeFileTokenThreshold: number;
-  /** Provider override for compaction summarization. */
-  summaryProvider: string;
-  /** Model override for compaction summarization. */
-  summaryModel: string;
-  /** Provider override for large-file text summarization. */
-  largeFileSummaryProvider: string;
-  /** Model override for large-file text summarization. */
-  largeFileSummaryModel: string;
-  /** Provider override for lcm_expand_query sub-agent. */
-  expansionProvider: string;
-  /** Model override for lcm_expand_query sub-agent. */
-  expansionModel: string;
-  /** Max time to wait for delegated lcm_expand_query sub-agent completion. */
-  delegationTimeoutMs: number;
-  /** Max time to wait for a single model-backed LCM summarizer call. */
-  summaryTimeoutMs: number;
-  /** IANA timezone for timestamps in summaries (from TZ env or system default) */
-  timezone: string;
-  /** When true, retroactively delete HEARTBEAT_OK turn cycles from LCM storage. */
-  pruneHeartbeatOk: boolean;
-  /** Hard ceiling for assembly token budget — caps runtime-provided and fallback budgets. */
-  maxAssemblyTokenBudget?: number;
-  /** Maximum allowed overage factor for summaries relative to target tokens (default 3). */
-  summaryMaxOverageFactor: number;
-  /** Custom instructions injected into all summarization prompts. */
-  customInstructions: string;
-  /** Consecutive auth failures before the compaction circuit breaker trips (default 5). */
-  circuitBreakerThreshold: number;
-  /** Cooldown in milliseconds before the circuit breaker auto-resets (default 30 min). */
-  circuitBreakerCooldownMs: number;
-  /** Explicit fallback provider/model pairs for compaction summarization. */
-  fallbackProviders: Array<{ provider: string; model: string }>;
-  /** Cache-sensitive policy for incremental leaf compaction. */
-  cacheAwareCompaction: CacheAwareCompactionConfig;
-  /** Dynamic step-band policy for incremental leaf chunk sizing. */
-  dynamicLeafChunkTokens: DynamicLeafChunkTokensConfig;
-};
-
-/** Safely coerce an unknown value to a finite number, or return undefined. */
-function toNumber(value: unknown): number | undefined {
-  if (typeof value === "number" && Number.isFinite(value)) return value;
-  if (typeof value === "string") {
-    const n = Number(value);
-    if (Number.isFinite(n)) return n;
-  }
-  return undefined;
-}
-
-/** Safely parse a finite integer from an environment string, or return undefined.
- *  Unlike raw parseInt(), this returns undefined for NaN so ?? fallback works. */
-function parseFiniteInt(value: string | undefined): number | undefined {
-  if (value === undefined) return undefined;
-  const parsed = parseInt(value, 10);
-  return Number.isFinite(parsed) ? parsed : undefined;
-}
-
-/** Safely parse a finite float from an environment string, or return undefined. */
-function parseFiniteNumber(value: string | undefined): number | undefined {
-  if (value === undefined) return undefined;
-  const parsed = parseFloat(value);
-  return Number.isFinite(parsed) ? parsed : undefined;
-}
-
-/** Parse fallback providers from env string (format: "provider/model,provider/model"). */
-function parseFallbackProviders(value: string | undefined): Array<{ provider: string; model: string }> | undefined {
-  if (!value?.trim()) return undefined;
-  const entries: Array<{ provider: string; model: string }> = [];
-  for (const part of value.split(",")) {
-    const trimmed = part.trim();
-    if (!trimmed) continue;
-    const slashIdx = trimmed.indexOf("/");
-    if (slashIdx > 0 && slashIdx < trimmed.length - 1) {
-      const provider = trimmed.slice(0, slashIdx).trim();
-      const model = trimmed.slice(slashIdx + 1).trim();
-      if (provider && model) {
-        entries.push({ provider, model });
-      }
-    }
-  }
-  return entries.length > 0 ? entries : undefined;
-}
-
-/** Parse fallback providers from plugin config array (object items only). */
-function toFallbackProviderArray(value: unknown): Array<{ provider: string; model: string }> | undefined {
-  if (!Array.isArray(value)) return undefined;
-  const entries: Array<{ provider: string; model: string }> = [];
-  for (const item of value) {
-    if (item && typeof item === "object" && !Array.isArray(item)) {
-      const p = toStr((item as Record<string, unknown>).provider);
-      const m = toStr((item as Record<string, unknown>).model);
-      if (p && m) entries.push({ provider: p, model: m });
-    }
-  }
-  return entries.length > 0 ? entries : undefined;
-}
-
-/** Safely coerce an unknown value to a boolean, or return undefined. */
-function toBool(value: unknown): boolean | undefined {
-  if (typeof value === "boolean") return value;
-  if (value === "true") return true;
-  if (value === "false") return false;
-  return undefined;
-}
-
-/** Safely coerce an unknown value to a trimmed non-empty string, or return undefined. */
-function toStr(value: unknown): string | undefined {
-  if (typeof value === "string") {
-    const trimmed = value.trim();
-    return trimmed.length > 0 ? trimmed : undefined;
-  }
-  return undefined;
-}
-
-/** Coerce a plugin config value into a trimmed string array when possible. */
-function toStrArray(value: unknown): string[] | undefined {
-  if (Array.isArray(value)) {
-    const normalized = value
-      .map((entry) => toStr(entry))
-      .filter((entry): entry is string => typeof entry === "string");
-    return normalized.length > 0 ? normalized : [];
-  }
-  const single = toStr(value);
-  if (!single) {
-    return undefined;
-  }
-  return single
-    .split(",")
-    .map((entry) => entry.trim())
-    .filter(Boolean);
-}
-
-function toRecord(value: unknown): Record<string, unknown> | undefined {
-  return value && typeof value === "object" && !Array.isArray(value)
-    ? (value as Record<string, unknown>)
-    : undefined;
-}
-
-/**
- * Resolve LCM configuration with three-tier precedence:
- *   1. Environment variables (highest — backward compat)
- *   2. Plugin config object (from plugins.entries.lossless-claw.config)
- *   3. Hardcoded defaults (lowest)
- */
-export function resolveLcmConfig(
-  env: NodeJS.ProcessEnv = process.env,
-  pluginConfig?: Record<string, unknown>,
-): LcmConfig {
-  const pc = pluginConfig ?? {};
-  const cacheAwareCompaction = toRecord(pc.cacheAwareCompaction);
-  const dynamicLeafChunkTokens = toRecord(pc.dynamicLeafChunkTokens);
-  const resolvedLeafChunkTokens =
-    parseFiniteInt(env.LCM_LEAF_CHUNK_TOKENS)
-      ?? toNumber(pc.leafChunkTokens) ?? 20000;
-  const resolvedBootstrapMaxTokens =
-    parseFiniteInt(env.LCM_BOOTSTRAP_MAX_TOKENS)
-      ?? toNumber(pc.bootstrapMaxTokens)
-      ?? Math.max(6000, Math.floor(resolvedLeafChunkTokens * 0.3));
-  const envDelegationTimeoutMs =
-    env.LCM_DELEGATION_TIMEOUT_MS !== undefined
-      ? toNumber(env.LCM_DELEGATION_TIMEOUT_MS)
-      : undefined;
-  const resolvedDynamicLeafChunkMax = Math.max(
-    resolvedLeafChunkTokens,
-    parseFiniteInt(env.LCM_DYNAMIC_LEAF_CHUNK_TOKENS_MAX)
-      ?? toNumber(dynamicLeafChunkTokens?.max)
-      ?? Math.floor(resolvedLeafChunkTokens * 2),
-  );
-  const resolvedHotCachePressureFactor = Math.max(
-    1,
-    parseFiniteNumber(env.LCM_HOT_CACHE_PRESSURE_FACTOR)
-      ?? toNumber(cacheAwareCompaction?.hotCachePressureFactor)
-      ?? 4,
-  );
-  const resolvedHotCacheBudgetHeadroomRatio = Math.min(
-    0.95,
-    Math.max(
-      0,
-      parseFiniteNumber(env.LCM_HOT_CACHE_BUDGET_HEADROOM_RATIO)
-        ?? toNumber(cacheAwareCompaction?.hotCacheBudgetHeadroomRatio)
-        ?? 0.2,
-    ),
-  );
-
-  return {
-    enabled:
-      env.LCM_ENABLED !== undefined
-        ? env.LCM_ENABLED !== "false"
-        : toBool(pc.enabled) ?? true,
-    databasePath:
-      env.LCM_DATABASE_PATH
-      ?? toStr(pc.dbPath)
-      ?? toStr(pc.databasePath)
-      ?? join(homedir(), ".openclaw", "lcm.db"),
-    ignoreSessionPatterns:
-      env.LCM_IGNORE_SESSION_PATTERNS !== undefined
-        ? env.LCM_IGNORE_SESSION_PATTERNS
-          .split(",")
-          .map((entry) => entry.trim())
-          .filter(Boolean)
-        : toStrArray(pc.ignoreSessionPatterns) ?? [],
-    statelessSessionPatterns:
-      env.LCM_STATELESS_SESSION_PATTERNS !== undefined
-        ? env.LCM_STATELESS_SESSION_PATTERNS
-          .split(",")
-          .map((entry) => entry.trim())
-          .filter(Boolean)
-        : toStrArray(pc.statelessSessionPatterns) ?? [],
-    skipStatelessSessions:
-      env.LCM_SKIP_STATELESS_SESSIONS !== undefined
-        ? env.LCM_SKIP_STATELESS_SESSIONS === "true"
-        : toBool(pc.skipStatelessSessions) ?? true,
-    contextThreshold:
-      parseFiniteNumber(env.LCM_CONTEXT_THRESHOLD)
-        ?? toNumber(pc.contextThreshold) ?? 0.75,
-    freshTailCount:
-      parseFiniteInt(env.LCM_FRESH_TAIL_COUNT)
-        ?? toNumber(pc.freshTailCount) ?? 64,
-    newSessionRetainDepth:
-      parseFiniteInt(env.LCM_NEW_SESSION_RETAIN_DEPTH)
-        ?? toNumber(pc.newSessionRetainDepth) ?? 2,
-    leafMinFanout:
-      parseFiniteInt(env.LCM_LEAF_MIN_FANOUT)
-        ?? toNumber(pc.leafMinFanout) ?? 8,
-    condensedMinFanout:
-      parseFiniteInt(env.LCM_CONDENSED_MIN_FANOUT)
-        ?? toNumber(pc.condensedMinFanout) ?? 4,
-    condensedMinFanoutHard:
-      parseFiniteInt(env.LCM_CONDENSED_MIN_FANOUT_HARD)
-        ?? toNumber(pc.condensedMinFanoutHard) ?? 2,
-    incrementalMaxDepth:
-      parseFiniteInt(env.LCM_INCREMENTAL_MAX_DEPTH)
-        ?? toNumber(pc.incrementalMaxDepth) ?? 1,
-    leafChunkTokens: resolvedLeafChunkTokens,
-    bootstrapMaxTokens: resolvedBootstrapMaxTokens,
-    leafTargetTokens:
-      parseFiniteInt(env.LCM_LEAF_TARGET_TOKENS)
-        ?? toNumber(pc.leafTargetTokens) ?? 2400,
-    condensedTargetTokens:
-      parseFiniteInt(env.LCM_CONDENSED_TARGET_TOKENS)
-        ?? toNumber(pc.condensedTargetTokens) ?? 2000,
-    maxExpandTokens:
-      parseFiniteInt(env.LCM_MAX_EXPAND_TOKENS)
-        ?? toNumber(pc.maxExpandTokens) ?? 4000,
-    largeFileTokenThreshold:
-      parseFiniteInt(env.LCM_LARGE_FILE_TOKEN_THRESHOLD)
-        ?? toNumber(pc.largeFileThresholdTokens)
-        ?? toNumber(pc.largeFileTokenThreshold)
-        ?? 25000,
-    summaryProvider:
-      env.LCM_SUMMARY_PROVIDER?.trim() ?? toStr(pc.summaryProvider) ?? "",
-    summaryModel:
-      env.LCM_SUMMARY_MODEL?.trim() ?? toStr(pc.summaryModel) ?? "",
-    largeFileSummaryProvider:
-      env.LCM_LARGE_FILE_SUMMARY_PROVIDER?.trim() ?? toStr(pc.largeFileSummaryProvider) ?? "",
-    largeFileSummaryModel:
-      env.LCM_LARGE_FILE_SUMMARY_MODEL?.trim() ?? toStr(pc.largeFileSummaryModel) ?? "",
-    expansionProvider:
-      env.LCM_EXPANSION_PROVIDER?.trim() ?? toStr(pc.expansionProvider) ?? "",
-    expansionModel:
-      env.LCM_EXPANSION_MODEL?.trim() ?? toStr(pc.expansionModel) ?? "",
-    delegationTimeoutMs: envDelegationTimeoutMs ?? toNumber(pc.delegationTimeoutMs) ?? 120000,
-    summaryTimeoutMs:
-      parseFiniteInt(env.LCM_SUMMARY_TIMEOUT_MS)
-        ?? toNumber(pc.summaryTimeoutMs) ?? 60000,
-    timezone: env.TZ ?? toStr(pc.timezone) ?? Intl.DateTimeFormat().resolvedOptions().timeZone,
-    pruneHeartbeatOk:
-      env.LCM_PRUNE_HEARTBEAT_OK !== undefined
-        ? env.LCM_PRUNE_HEARTBEAT_OK === "true"
-        : toBool(pc.pruneHeartbeatOk) ?? false,
-    maxAssemblyTokenBudget:
-      parseFiniteInt(env.LCM_MAX_ASSEMBLY_TOKEN_BUDGET)
-        ?? toNumber(pc.maxAssemblyTokenBudget) ?? undefined,
-    summaryMaxOverageFactor:
-      parseFiniteNumber(env.LCM_SUMMARY_MAX_OVERAGE_FACTOR)
-        ?? toNumber(pc.summaryMaxOverageFactor) ?? 3,
-    customInstructions:
-      env.LCM_CUSTOM_INSTRUCTIONS?.trim() ?? toStr(pc.customInstructions) ?? "",
-    circuitBreakerThreshold:
-      parseFiniteInt(env.LCM_CIRCUIT_BREAKER_THRESHOLD)
-        ?? toNumber(pc.circuitBreakerThreshold) ?? 5,
-    circuitBreakerCooldownMs:
-      parseFiniteInt(env.LCM_CIRCUIT_BREAKER_COOLDOWN_MS)
-        ?? toNumber(pc.circuitBreakerCooldownMs) ?? 1_800_000,
-    fallbackProviders:
-      parseFallbackProviders(env.LCM_FALLBACK_PROVIDERS)
-        ?? toFallbackProviderArray(pc.fallbackProviders) ?? [],
-    cacheAwareCompaction: {
-      enabled:
-        env.LCM_CACHE_AWARE_COMPACTION_ENABLED !== undefined
-          ? env.LCM_CACHE_AWARE_COMPACTION_ENABLED !== "false"
-          : toBool(cacheAwareCompaction?.enabled) ?? true,
-      maxColdCacheCatchupPasses:
-        parseFiniteInt(env.LCM_MAX_COLD_CACHE_CATCHUP_PASSES)
-          ?? toNumber(cacheAwareCompaction?.maxColdCacheCatchupPasses)
-          ?? 2,
-      hotCachePressureFactor: resolvedHotCachePressureFactor,
-      hotCacheBudgetHeadroomRatio: resolvedHotCacheBudgetHeadroomRatio,
-    },
-    dynamicLeafChunkTokens: {
-      enabled:
-        env.LCM_DYNAMIC_LEAF_CHUNK_TOKENS_ENABLED !== undefined
-          ? env.LCM_DYNAMIC_LEAF_CHUNK_TOKENS_ENABLED === "true"
-          : toBool(dynamicLeafChunkTokens?.enabled) ?? true,
-      max: resolvedDynamicLeafChunkMax,
-    },
-  };
-}

package/src/db/connection.ts

@@ -1,141 +0,0 @@
-import { mkdirSync } from "node:fs";
-import { dirname, resolve } from "node:path";
-import { DatabaseSync } from "node:sqlite";
-
-type ConnectionKey = string;
-const SQLITE_BUSY_TIMEOUT_MS = 5_000;
-
-const connectionsByPath = new Map<ConnectionKey, Set<DatabaseSync>>();
-const connectionIndex = new Map<DatabaseSync, ConnectionKey>();
-
-function isInMemoryPath(dbPath: string): boolean {
-  const normalized = dbPath.trim();
-  return normalized === ":memory:" || normalized.startsWith("file::memory:");
-}
-
-export function normalizePath(dbPath: string): ConnectionKey {
-  if (isInMemoryPath(dbPath)) {
-    const trimmed = dbPath.trim();
-    return trimmed.length > 0 ? trimmed : ":memory:";
-  }
-  return resolve(dbPath);
-}
-
-function ensureDbDirectory(dbPath: string): void {
-  if (isInMemoryPath(dbPath)) {
-    return;
-  }
-  mkdirSync(dirname(dbPath), { recursive: true });
-}
-
-function configureConnection(db: DatabaseSync): DatabaseSync {
-  db.exec("PRAGMA journal_mode = WAL");
-  db.exec(`PRAGMA busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`);
-  db.exec("PRAGMA foreign_keys = ON");
-  // 64MB page cache (default 2MB is severely undersized for multi-GB databases
-  // with concurrent agents). Memory is demand-allocated, released on close.
-  db.exec("PRAGMA cache_size = -65536");
-  // NORMAL is officially recommended for WAL mode — crash-safe for app crashes,
-  // only risks data loss on power failure (OS/kernel crash). The bootstrap
-  // process re-ingests any lost transactions from session files.
-  db.exec("PRAGMA synchronous = NORMAL");
-  // Keep temp tables/indexes in RAM (helps ordinal resequencing).
-  db.exec("PRAGMA temp_store = MEMORY");
-  return db;
-}
-
-function trackConnection(dbPath: string, db: DatabaseSync): void {
-  const key = normalizePath(dbPath);
-  let entries = connectionsByPath.get(key);
-  if (!entries) {
-    entries = new Set();
-    connectionsByPath.set(key, entries);
-  }
-  entries.add(db);
-  connectionIndex.set(db, key);
-}
-
-function untrackConnection(db: DatabaseSync): void {
-  const key = connectionIndex.get(db);
-  if (!key) {
-    return;
-  }
-  const entries = connectionsByPath.get(key);
-  if (entries) {
-    entries.delete(db);
-    if (entries.size === 0) {
-      connectionsByPath.delete(key);
-    }
-  }
-  connectionIndex.delete(db);
-}
-
-function closeDatabase(db: DatabaseSync | undefined): void {
-  if (!db) {
-    return;
-  }
-  try {
-    // Update query planner statistics for tables that changed since last optimize.
-    // Separate try so a SQLITE_BUSY/SQLITE_READONLY from optimize doesn't skip close.
-    try { db.exec("PRAGMA optimize"); } catch { /* best-effort */ }
-    db.close();
-  } catch {
-    // Ignore close failures; callers are shutting down anyway.
-  } finally {
-    untrackConnection(db);
-  }
-}
-
-/**
- * Create a new SQLite connection for the given LCM database path.
- *
- * Connections are tracked so tests can close them by path via closeLcmConnection().
- */
-export function createLcmDatabaseConnection(dbPath: string): DatabaseSync {
-  ensureDbDirectory(dbPath);
-  const db = new DatabaseSync(dbPath);
-  try {
-    configureConnection(db);
-  } catch (err) {
-    try { db.close(); } catch { /* ignore cleanup failure */ }
-    throw err;
-  }
-  trackConnection(dbPath, db);
-  return db;
-}
-
-/**
- * Close tracked LCM connections.
- *
- * When a DatabaseSync instance is supplied, only that handle is closed.
- * When a path is supplied, all handles associated with the normalized path
- * are closed. When called with no arguments, all tracked connections are
- * closed. Intended primarily for tests.
- */
-export function closeLcmConnection(target?: string | DatabaseSync): void {
-  if (target && typeof target !== "string") {
-    closeDatabase(target);
-    return;
-  }
-
-  if (typeof target === "string") {
-    const key = normalizePath(target);
-    const entries = connectionsByPath.get(key);
-    if (!entries) {
-      return;
-    }
-    for (const db of [...entries]) {
-      closeDatabase(db);
-    }
-    connectionsByPath.delete(key);
-    return;
-  }
-
-  for (const db of [...connectionIndex.keys()]) {
-    closeDatabase(db);
-  }
-  connectionsByPath.clear();
-  connectionIndex.clear();
-}
-
-export const getLcmConnection = createLcmDatabaseConnection;

package/src/db/features.ts

@@ -1,42 +0,0 @@
-import type { DatabaseSync } from "node:sqlite";
-
-export type LcmDbFeatures = {
-  fts5Available: boolean;
-};
-
-const featureCache = new WeakMap<DatabaseSync, LcmDbFeatures>();
-
-function probeFts5(db: DatabaseSync): boolean {
-  try {
-    db.exec("DROP TABLE IF EXISTS temp.__lcm_fts5_probe");
-    db.exec("CREATE VIRTUAL TABLE temp.__lcm_fts5_probe USING fts5(content)");
-    db.exec("DROP TABLE temp.__lcm_fts5_probe");
-    return true;
-  } catch {
-    try {
-      db.exec("DROP TABLE IF EXISTS temp.__lcm_fts5_probe");
-    } catch {
-      // Ignore cleanup failures after a failed probe.
-    }
-    return false;
-  }
-}
-
-/**
- * Detect SQLite features exposed by the current Node runtime.
- *
- * The result is cached per DatabaseSync handle because the probe is runtime-
- * specific, not database-file-specific.
- */
-export function getLcmDbFeatures(db: DatabaseSync): LcmDbFeatures {
-  const cached = featureCache.get(db);
-  if (cached) {
-    return cached;
-  }
-
-  const detected: LcmDbFeatures = {
-    fts5Available: probeFts5(db),
-  };
-  featureCache.set(db, detected);
-  return detected;
-}