lazyclaw 4.3.0 → 5.0.1

package/mas/confidence.mjs

@@ -0,0 +1,108 @@
+// Confidence calculator for v5 skills — spec §0.1 H2, §3.5.
+//
+// Pure functions, no I/O. Used by skill_synth v2 to stamp frontmatter
+// and by trajectory_store recall ranking to weight near-duplicates.
+//
+//   wilsonLowerBound(s, n)           — 95% Wilson lower bound on success rate.
+//   crossCliDampen(score, trainer, provider, factor?)
+//                                    — multiply by `factor` (default 0.85)
+//                                      when trainer provider differs from
+//                                      worker provider (canonical decision
+//                                      §0.1 H2). Phase H2 makes the factor
+//                                      tunable; same-family scores are
+//                                      always returned unchanged.
+//   recencyDecay(ageMs, halfLifeMs)  — exponential decay weight (0..1].
+//   computeConfidence({successes, trials, ageMs, trainer, provider, dampenFactor?})
+//                                    — composed score in [0, 1].
+//   resolveDampenFactor(cfg)         — pull `crossCliDampenFactor` out of
+//                                      cfg.orchestra.learning (or legacy
+//                                      cfg.orchestrator.learning); clamps
+//                                      to [0,1]; defaults to 0.85 on miss.
+
+const Z = 1.96; // 95% confidence two-sided
+
+export const DEFAULT_CROSS_CLI_DAMPEN = 0.85;
+
+export function wilsonLowerBound(successes, trials) {
+  const s = Number(successes) || 0;
+  const n = Number(trials) || 0;
+  if (n <= 0) return 0;
+  const phat = s / n;
+  const z2 = Z * Z;
+  const denom = 1 + z2 / n;
+  const center = phat + z2 / (2 * n);
+  const margin = Z * Math.sqrt((phat * (1 - phat) + z2 / (4 * n)) / n);
+  const lb = (center - margin) / denom;
+  return Math.max(0, Math.min(1, lb));
+}
+
+const PROVIDER_FAMILY = {
+  'claude-cli': 'anthropic',
+  'anthropic': 'anthropic',
+  'codex-cli': 'openai',
+  'openai': 'openai',
+  'gemini-cli': 'gemini',
+  'gemini': 'gemini',
+  'ollama': 'ollama',
+};
+
+export function sameFamily(a, b) {
+  if (!a || !b) return false;
+  return PROVIDER_FAMILY[a] === PROVIDER_FAMILY[b];
+}
+
+// Normalise a caller-supplied dampening factor. Anything non-finite (NaN,
+// non-numeric strings, undefined) falls back to the canonical 0.85 so a
+// typo in config does not silently zero out every cross-CLI skill.
+function _normalizeFactor(factor) {
+  if (factor === null || factor === undefined) return DEFAULT_CROSS_CLI_DAMPEN;
+  const n = Number(factor);
+  if (!Number.isFinite(n)) return DEFAULT_CROSS_CLI_DAMPEN;
+  if (n < 0) return 0;
+  if (n > 1) return 1;
+  return n;
+}
+
+export function crossCliDampen(score, trainerProvider, workerProvider, factor) {
+  if (!trainerProvider || !workerProvider) return score;
+  if (sameFamily(trainerProvider, workerProvider)) return score;
+  return score * _normalizeFactor(factor);
+}
+
+export function recencyDecay(ageMs, halfLifeMs = 30 * 24 * 60 * 60 * 1000) {
+  const t = Math.max(0, Number(ageMs) || 0);
+  const hl = Math.max(1, Number(halfLifeMs) || 1);
+  return Math.pow(0.5, t / hl);
+}
+
+export function computeConfidence({
+  successes = 0,
+  trials = 0,
+  ageMs = 0,
+  trainerProvider = null,
+  workerProvider = null,
+  halfLifeMs,
+  dampenFactor,
+} = {}) {
+  const base = wilsonLowerBound(successes, trials);
+  const decayed = base * recencyDecay(ageMs, halfLifeMs);
+  const dampened = crossCliDampen(decayed, trainerProvider, workerProvider, dampenFactor);
+  return Math.max(0, Math.min(1, dampened));
+}
+
+// Pull the tunable factor out of config. Honours both the canonical
+// `cfg.orchestra.learning.crossCliDampenFactor` shape and the legacy
+// `cfg.orchestrator.learning.crossCliDampenFactor` (v4 callers). Returns
+// 0.85 when missing/invalid so callers never have to gate on undefined.
+export function resolveDampenFactor(cfg) {
+  if (!cfg || typeof cfg !== 'object') return DEFAULT_CROSS_CLI_DAMPEN;
+  const orchestra = cfg.orchestra || cfg.orchestrator || {};
+  const learning = orchestra.learning || {};
+  const raw = learning.crossCliDampenFactor;
+  if (raw === undefined || raw === null) return DEFAULT_CROSS_CLI_DAMPEN;
+  const n = Number(raw);
+  if (!Number.isFinite(n)) return DEFAULT_CROSS_CLI_DAMPEN;
+  if (n < 0) return 0;
+  if (n > 1) return 1;
+  return n;
+}

package/mas/index_db.mjs

@@ -0,0 +1,242 @@
+// mas/index_db.mjs — Phase A.
+//
+// Single SQLite handle per configDir backing four FTS5 virtual tables
+// (spec §4.3). The daemon opens the db at boot; CLI subcommands open
+// on demand. WAL mode lets many readers coexist with the one writer.
+//
+// Index failure NEVER propagates — see spec §4.4: write-through hooks
+// log and swallow so a corrupt index can't break the session-write
+// path. Recovery is via `lazyclaw index rebuild`.
+
+import Database from 'better-sqlite3';
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { redactSecrets } from './redact.mjs';
+
+const SCHEMA_VERSION = 1;
+const _handles = new Map();   // configDir → { db, stmts }
+
+function defaultConfigDir() {
+  return process.env.LAZYCLAW_CONFIG_DIR || path.join(os.homedir(), '.lazyclaw');
+}
+
+function dbPath(configDir) {
+  return path.join(configDir, 'index.db');
+}
+
+function ensureSchema(db) {
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS fts_sessions USING fts5(
+      content,
+      session_id UNINDEXED, turn_idx UNINDEXED, role UNINDEXED, ts UNINDEXED
+    );
+    CREATE VIRTUAL TABLE IF NOT EXISTS fts_skills USING fts5(
+      content,
+      skill_name UNINDEXED, trained_by UNINDEXED, group_name UNINDEXED
+    );
+    CREATE VIRTUAL TABLE IF NOT EXISTS fts_trajectories USING fts5(
+      content,
+      trajectory_id UNINDEXED, agent UNINDEXED, outcome UNINDEXED
+    );
+    CREATE VIRTUAL TABLE IF NOT EXISTS fts_memories USING fts5(
+      content,
+      topic UNINDEXED, kind UNINDEXED
+    );
+    CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
+  `);
+  const cur = db.prepare("SELECT value FROM meta WHERE key='schema_version'").get();
+  if (!cur) {
+    db.prepare("INSERT INTO meta(key,value) VALUES('schema_version', ?)").run(String(SCHEMA_VERSION));
+  }
+}
+
+function prepareStatements(db) {
+  return {
+    insertSession: db.prepare(
+      `INSERT INTO fts_sessions(content, session_id, turn_idx, role, ts)
+       VALUES (?, ?, ?, ?, ?)`),
+    insertSkill: db.prepare(
+      `INSERT INTO fts_skills(content, skill_name, trained_by, group_name)
+       VALUES (?, ?, ?, ?)`),
+    insertTrajectory: db.prepare(
+      `INSERT INTO fts_trajectories(content, trajectory_id, agent, outcome)
+       VALUES (?, ?, ?, ?)`),
+    insertMemory: db.prepare(
+      `INSERT INTO fts_memories(content, topic, kind)
+       VALUES (?, ?, ?)`),
+    queries: {
+      sessions: db.prepare(
+        `SELECT 'sessions' AS scope, bm25(fts_sessions) AS bm25,
+                snippet(fts_sessions, 0, '<mark>', '</mark>', '...', 16) AS snippet,
+                session_id, turn_idx, role, ts
+           FROM fts_sessions WHERE content MATCH ? ORDER BY bm25 LIMIT ?`),
+      skills: db.prepare(
+        `SELECT 'skills' AS scope, bm25(fts_skills) AS bm25,
+                snippet(fts_skills, 0, '<mark>', '</mark>', '...', 16) AS snippet,
+                skill_name, trained_by, group_name
+           FROM fts_skills WHERE content MATCH ? ORDER BY bm25 LIMIT ?`),
+      trajectories: db.prepare(
+        `SELECT 'trajectories' AS scope, bm25(fts_trajectories) AS bm25,
+                snippet(fts_trajectories, 0, '<mark>', '</mark>', '...', 16) AS snippet,
+                trajectory_id, agent, outcome
+           FROM fts_trajectories WHERE content MATCH ? ORDER BY bm25 LIMIT ?`),
+      memories: db.prepare(
+        `SELECT 'memories' AS scope, bm25(fts_memories) AS bm25,
+                snippet(fts_memories, 0, '<mark>', '</mark>', '...', 16) AS snippet,
+                topic, kind
+           FROM fts_memories WHERE content MATCH ? ORDER BY bm25 LIMIT ?`),
+    },
+  };
+}
+
+export function openIndex(configDir = defaultConfigDir(), opts = {}) {
+  const dir = configDir;
+  if (_handles.has(dir)) return _handles.get(dir).db;
+  fs.mkdirSync(dir, { recursive: true });
+  const db = new Database(dbPath(dir));
+  db.pragma('journal_mode = WAL');
+  db.pragma('synchronous = NORMAL');
+  ensureSchema(db);
+  if (opts.runIntegrityCheck !== false) {
+    const r = db.pragma('integrity_check', { simple: true });
+    if (r !== 'ok') {
+      // eslint-disable-next-line no-console
+      console.warn(`[index_db] integrity_check returned ${r} for ${dbPath(dir)}`);
+    }
+  }
+  const stmts = prepareStatements(db);
+  _handles.set(dir, { db, stmts });
+  return db;
+}
+
+export function closeIndex(configDir = defaultConfigDir()) {
+  const h = _handles.get(configDir);
+  if (!h) return;
+  try { h.db.close(); } catch { /* ignore */ }
+  _handles.delete(configDir);
+}
+
+function _stmts(configDir) {
+  if (!_handles.has(configDir)) openIndex(configDir);
+  return _handles.get(configDir).stmts;
+}
+
+export function indexSessionTurn(row, configDir = defaultConfigDir()) {
+  try {
+    const s = _stmts(configDir);
+    s.insertSession.run(
+      redactSecrets(String(row.content || '')),
+      String(row.session_id || ''), Number(row.turn_idx || 0),
+      String(row.role || ''), Number(row.ts || Date.now()),
+    );
+  } catch (e) {
+    // eslint-disable-next-line no-console
+    console.warn('[index_db] indexSessionTurn failed:', e.message);
+  }
+}
+
+export function indexSkill(row, configDir = defaultConfigDir()) {
+  try {
+    const s = _stmts(configDir);
+    s.insertSkill.run(
+      redactSecrets(String(row.content || '')),
+      String(row.skill_name || ''), String(row.trained_by || ''),
+      String(row.group_name || ''),
+    );
+  } catch (e) {
+    // eslint-disable-next-line no-console
+    console.warn('[index_db] indexSkill failed:', e.message);
+  }
+}
+
+export function indexTrajectory(row, configDir = defaultConfigDir()) {
+  try {
+    const s = _stmts(configDir);
+    s.insertTrajectory.run(
+      redactSecrets(String(row.content || '')),
+      String(row.trajectory_id || ''), String(row.agent || ''),
+      String(row.outcome || ''),
+    );
+  } catch (e) {
+    // eslint-disable-next-line no-console
+    console.warn('[index_db] indexTrajectory failed:', e.message);
+  }
+}
+
+export function indexMemory(row, configDir = defaultConfigDir()) {
+  try {
+    const s = _stmts(configDir);
+    s.insertMemory.run(
+      redactSecrets(String(row.content || '')),
+      String(row.topic || ''), String(row.kind || ''),
+    );
+  } catch (e) {
+    // eslint-disable-next-line no-console
+    console.warn('[index_db] indexMemory failed:', e.message);
+  }
+}
+
+// FTS5 query parser treats `-` as unary NOT and `:` as a column filter, so
+// raw user phrases like "write-through" turn into a NOT op against a
+// non-existent column. Strip FTS5 operators down to whitespace before the
+// MATCH so a bareword recall behaves like content search. Quoted phrases
+// from the caller are left alone (one or more `"` survives).
+function sanitizeFtsQuery(q) {
+  const s = String(q ?? '').trim();
+  if (!s) return s;
+  // Preserve quoted phrases verbatim; rewrite only unquoted runs.
+  if (/["()*^]/.test(s)) return s;
+  return s.replace(/[-:+]/g, ' ').replace(/\s+/g, ' ').trim();
+}
+
+export function recall(query, opts = {}) {
+  const t0 = process.hrtime.bigint();
+  const configDir = opts.configDir || defaultConfigDir();
+  const scope = opts.scope || ['sessions', 'skills', 'trajectories', 'memories'];
+  const k = Math.min(Math.max(Number(opts.k) || 10, 1), 50);
+  const s = _stmts(configDir);
+  const safeQuery = sanitizeFtsQuery(query);
+  const hits = [];
+  for (const sc of scope) {
+    const stmt = s.queries[sc];
+    if (!stmt) continue;
+    try {
+      const rows = stmt.all(safeQuery, k);
+      for (const r of rows) {
+        const { scope: sc2, bm25, snippet, ...metadata } = r;
+        hits.push({ scope: sc2, rank: hits.length, bm25, snippet, metadata });
+      }
+    } catch (e) {
+      // FTS5 MATCH syntax errors are caller mistakes; skip silently.
+      // "no such column" arises from the FTS5 column-filter syntax when
+      // a stray colon/operator survives sanitisation — also benign.
+      if (!/syntax error|no such column/i.test(e.message)) throw e;
+    }
+  }
+  hits.sort((a, b) => a.bm25 - b.bm25);
+  const trimmed = hits.slice(0, k);
+  for (let i = 0; i < trimmed.length; i++) trimmed[i].rank = i;
+  const elapsedNs = process.hrtime.bigint() - t0;
+  return { query, hits: trimmed, latencyMs: Number(elapsedNs) / 1e6 };
+}
+
+export function integrityCheck(configDir = defaultConfigDir()) {
+  const db = openIndex(configDir);
+  const r = db.pragma('integrity_check', { simple: true });
+  return { ok: r === 'ok', result: r };
+}
+
+export function rebuild(configDir = defaultConfigDir()) {
+  closeIndex(configDir);
+  const p = dbPath(configDir);
+  if (fs.existsSync(p)) {
+    fs.unlinkSync(p);
+    // WAL sidecar files
+    for (const ext of ['-wal', '-shm']) {
+      const side = p + ext;
+      if (fs.existsSync(side)) fs.unlinkSync(side);
+    }
+  }
+  openIndex(configDir);
+}

package/mas/nudge.mjs

@@ -0,0 +1,97 @@
+// Nudge loop — Phase B (v5 §3.6).
+//
+// Periodically scans the tail of memory/recent.jsonl, clusters
+// repeated user prompts, and (when a cluster crosses minCount) emits
+// a `nudge.suggest_skill` event into the daemon's SSE bus so the
+// curator UI can suggest "should I turn this into a skill?".
+//
+// v5.0 scope: emit only. Cross-channel push (Slack/Telegram) lands in
+// v5.1 per spec §0.2.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+
+const DEFAULT_TAIL = 200;
+const DEFAULT_INTERVAL_MS = 5 * 60 * 1000;   // 5 min
+const DEFAULT_MIN_COUNT = 3;
+
+export function defaultConfigDir() {
+  return process.env.LAZYCLAW_CONFIG_DIR || path.join(os.homedir(), '.lazyclaw');
+}
+
+function recentPath(configDir) {
+  return path.join(configDir, 'memory', 'recent.jsonl');
+}
+
+export function readRecent(configDir = defaultConfigDir(), n = DEFAULT_TAIL) {
+  const p = recentPath(configDir);
+  let raw;
+  try { raw = fs.readFileSync(p, 'utf8'); } catch { return []; }
+  const lines = raw.split('\n').filter(Boolean).slice(-n);
+  const out = [];
+  for (const line of lines) {
+    try { out.push(JSON.parse(line)); } catch { /* skip */ }
+  }
+  return out;
+}
+
+function normalise(text) {
+  return String(text || '')
+    .toLowerCase()
+    .replace(/[^a-z0-9\s]+/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+export function clusterRecent(entries, { minCount = DEFAULT_MIN_COUNT } = {}) {
+  const byKey = new Map();
+  for (const e of entries || []) {
+    if (e.role && e.role !== 'user') continue;
+    const key = normalise(e.content);
+    if (!key) continue;
+    if (!byKey.has(key)) byKey.set(key, { key, count: 0, sample: e.content, firstTs: e.ts, lastTs: e.ts });
+    const c = byKey.get(key);
+    c.count += 1;
+    c.lastTs = e.ts;
+  }
+  return [...byKey.values()].filter((c) => c.count >= minCount).sort((a, b) => b.count - a.count);
+}
+
+export function makeNudgeEvent({ cluster, ts = Date.now() } = {}) {
+  return {
+    kind: 'nudge.suggest_skill',
+    ts,
+    cluster: {
+      count: cluster.count,
+      sample: cluster.sample,
+      firstTs: cluster.firstTs,
+      lastTs: cluster.lastTs,
+    },
+    suggestion: `Repeated prompt: "${String(cluster.sample).slice(0, 80)}" (${cluster.count}×). Consider /skill create.`,
+  };
+}
+
+export function startNudgeLoop({ configDir = defaultConfigDir(), intervalMs = DEFAULT_INTERVAL_MS, minCount = DEFAULT_MIN_COUNT, emit, logger } = {}) {
+  if (typeof emit !== 'function') throw new Error('startNudgeLoop: emit(event) is required');
+  let timer = null;
+  let stopped = false;
+
+  function tick() {
+    if (stopped) return;
+    try {
+      const entries = readRecent(configDir);
+      const clusters = clusterRecent(entries, { minCount });
+      for (const c of clusters) emit(makeNudgeEvent({ cluster: c }));
+    } catch (err) {
+      try { logger?.warn?.('nudge_tick_failed', { err: err.message }); } catch { /* ignore */ }
+    }
+  }
+
+  timer = setInterval(tick, intervalMs);
+  if (typeof timer.unref === 'function') timer.unref();
+  return {
+    stop() { stopped = true; if (timer) clearInterval(timer); timer = null; },
+    runOnce: tick,
+  };
+}

package/mas/prompt_stack.mjs

@@ -0,0 +1,80 @@
+// mas/prompt_stack.mjs
+// 8-layer system-prompt composer (v5.0 spec §9.3, canonical C10).
+// Layers (top-to-bottom in the system prompt):
+//   1.  Global  SOUL.md          <configDir>/SOUL.md
+//   1.5 Workspace SOUL.md        <configDir>/workspaces/<name>/SOUL.md      (C10)
+//   2.  Personality              <configDir>/personalities/<name>.md         (C7)
+//   3.  agent.role               from agent record
+//   4.  USER.md                  <configDir>/memory/USER.md                  (C6)
+//   5.  Skill index              skills.skillsIndex(cfgDir)
+//   6.  Memory (core.md)         memory.loadCore(cfgDir)
+//   7.  Trajectory tail          last recent.jsonl entry (best-effort)
+//
+// Missing layers are silently skipped. Never throws. Result is a single
+// newline-joined string suitable for prepending to the agent system
+// prompt. Caller decides whether to further sandwich it with task input.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { skillsIndex } from '../skills.mjs';
+import { loadCore, recentPath, defaultConfigDir } from '../memory.mjs';
+
+function readOpt(p) {
+  try { return fs.readFileSync(p, 'utf8').trim(); }
+  catch { return ''; }
+}
+
+function lastRecentLine(cfgDir) {
+  try {
+    const p = recentPath(cfgDir);
+    if (!fs.existsSync(p)) return '';
+    const txt = fs.readFileSync(p, 'utf8');
+    const lines = txt.split('\n').filter(Boolean);
+    if (!lines.length) return '';
+    const parsed = JSON.parse(lines[lines.length - 1]);
+    return `${parsed.role || 'user'}: ${String(parsed.content || '').slice(0, 240)}`;
+  } catch { return ''; }
+}
+
+export function composePromptStack({ cfgDir, agent, workspace, sessionId } = {}) {
+  const dir = cfgDir || defaultConfigDir();
+  const a = agent || {};
+  const parts = [];
+
+  // 1. global SOUL
+  const globalSoul = readOpt(path.join(dir, 'SOUL.md'));
+  if (globalSoul) parts.push(`## SOUL\n${globalSoul}`);
+
+  // 1.5 workspace SOUL (C10)
+  if (workspace) {
+    const wsSoul = readOpt(path.join(dir, 'workspaces', workspace, 'SOUL.md'));
+    if (wsSoul) parts.push(`## Workspace SOUL (${workspace})\n${wsSoul}`);
+  }
+
+  // 2. personality (C7)
+  if (a.personality) {
+    const p = readOpt(path.join(dir, 'personalities', `${a.personality}.md`));
+    if (p) parts.push(`## Personality (${a.personality})\n${p}`);
+  }
+
+  // 3. agent.role
+  if (a.role) parts.push(`## Role (${a.name || 'agent'})\n${a.role}`);
+
+  // 4. USER.md (C6)
+  const userMd = readOpt(path.join(dir, 'memory', 'USER.md'));
+  if (userMd) parts.push(`## What the user has told you before\n${userMd}`);
+
+  // 5. skill index
+  const idx = skillsIndex(dir);
+  if (idx) parts.push(`## Available skills\n${idx}`);
+
+  // 6. memory core.md
+  const core = loadCore(dir);
+  if (core && core.trim()) parts.push(`## Long-term memory\n${core.trim()}`);
+
+  // 7. trajectory tail (sessionId may be ignored — recent.jsonl is global)
+  const tail = lastRecentLine(dir);
+  if (tail) parts.push(`## Most-recent turn\n${tail}`);
+
+  return parts.join('\n\n');
+}