lazyclaw 4.3.0 → 5.0.0

package/mas/trajectory_store.mjs

@@ -0,0 +1,179 @@
+// mas/trajectory_store.mjs — Phase A.
+//
+// Persists TrajectoryRecord (spec §3.3) to JSONL on disk plus an
+// in-memory cache. Storage layout:
+//   <configDir>/trajectories/<YYYY-MM-DD>/<id>.jsonl
+// One file per trajectory id (a ULID) so concurrent writers never
+// contend. A single in-memory Map<id, record> serves hot reads; cold
+// reads stream the file back through JSON.parse.
+//
+// Phase A scope: write, read, list-by-task. Recall by full-text query
+// lives in mas/index_db.mjs (FTS5). The two stores share the same
+// record but the FTS5 mirror is best-effort — disk JSONL is the
+// source of truth.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import crypto from 'node:crypto';
+import { redactSecrets } from './redact.mjs';
+import { indexTrajectory as _indexTrajectory } from './index_db.mjs';
+
+export const OUTCOME_ENUM = Object.freeze(['done', 'failed', 'abandoned']);
+
+const _cache = new Map();   // id → record (capped at CACHE_MAX entries)
+const CACHE_MAX = 256;
+
+function defaultConfigDir() {
+  return process.env.LAZYCLAW_CONFIG_DIR || path.join(os.homedir(), '.lazyclaw');
+}
+
+function trajectoriesDir(configDir) {
+  return path.join(configDir, 'trajectories');
+}
+
+// Crockford-base32 ULID generator. Monotonic within a single ms by
+// appending a counter — same-millisecond puts stay sortable.
+const ULID_ALPHABET = '0123456789ABCDEFGHJKMNPQRSTVWXYZ';
+let _ulidLastMs = 0;
+let _ulidCounter = 0;
+let _ulidLastRandPrefix = '';
+function ulid() {
+  let now = Date.now();
+  let sameMs = (now === _ulidLastMs);
+  if (sameMs) _ulidCounter++;
+  else { _ulidLastMs = now; _ulidCounter = 0; }
+  let timePart = '';
+  let t = now;
+  for (let i = 0; i < 10; i++) {
+    timePart = ULID_ALPHABET[t % 32] + timePart;
+    t = Math.floor(t / 32);
+  }
+  // Hold the random prefix stable within a single ms so the counter
+  // suffix is the ONLY thing that changes — that's what keeps same-ms
+  // ULIDs lexicographically sortable (canonical ULID monotonicity).
+  if (!sameMs) {
+    const rand = crypto.randomBytes(10);
+    let randPart = '';
+    for (let i = 0; i < 16; i++) {
+      randPart += ULID_ALPHABET[rand[i % 10] % 32];
+    }
+    _ulidLastRandPrefix = randPart.slice(0, 14);
+  }
+  // Counter suffix (last 2 chars) keeps monotonicity intra-ms.
+  const ctr = ULID_ALPHABET[(_ulidCounter >> 5) % 32]
+            + ULID_ALPHABET[_ulidCounter % 32];
+  return timePart + _ulidLastRandPrefix + ctr;
+}
+
+function redactTurns(turns) {
+  return (turns || []).map(t => ({
+    ...t,
+    content: typeof t.content === 'string' ? redactSecrets(t.content) : t.content,
+    thinking: typeof t.thinking === 'string' ? redactSecrets(t.thinking) : t.thinking,
+    toolCalls: (t.toolCalls || []).map(c => ({
+      ...c,
+      result: typeof c.result === 'string' ? redactSecrets(c.result) : c.result,
+    })),
+  }));
+}
+
+function dateBucket(ms) {
+  return new Date(ms).toISOString().slice(0, 10);   // YYYY-MM-DD
+}
+
+function recordPath(configDir, bucket, id) {
+  return path.join(trajectoriesDir(configDir), bucket, `${id}.jsonl`);
+}
+
+function cachePush(id, rec) {
+  _cache.set(id, rec);
+  if (_cache.size > CACHE_MAX) {
+    const oldest = _cache.keys().next().value;
+    _cache.delete(oldest);
+  }
+}
+
+export async function put(record, opts = {}) {
+  if (!record || typeof record !== 'object') {
+    throw new TypeError('trajectory_store.put: record must be an object');
+  }
+  if (!OUTCOME_ENUM.includes(record.outcome)) {
+    throw new Error(
+      `outcome must be one of ${OUTCOME_ENUM.join('|')}, got ${record.outcome}`,
+    );
+  }
+  const configDir = opts.configDir || defaultConfigDir();
+  const id = record.id || ulid();
+  const stored = {
+    ...record,
+    id,
+    systemPrompt: typeof record.systemPrompt === 'string'
+      ? redactSecrets(record.systemPrompt) : '',
+    userMessages: (record.userMessages || []).map(m =>
+      typeof m === 'string' ? redactSecrets(m) : m),
+    turns: redactTurns(record.turns),
+    finalAnswer: typeof record.finalAnswer === 'string'
+      ? redactSecrets(record.finalAnswer) : '',
+  };
+  const bucket = dateBucket(stored.startedAt || Date.now());
+  const file = recordPath(configDir, bucket, id);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, JSON.stringify(stored) + '\n');
+  // Phase A: FTS5 mirror (spec §4.4). Content is the concatenation of
+  // the final answer plus every turn's textual content so a single
+  // recall() can surface trajectories by either signal.
+  try {
+    const ftsContent = [
+      stored.finalAnswer || '',
+      ...(stored.turns || []).map(t => String(t.content || '')),
+    ].filter(Boolean).join('\n');
+    _indexTrajectory({
+      trajectory_id: id,
+      agent: stored.agentName || '',
+      outcome: stored.outcome,
+      content: ftsContent,
+    }, configDir);
+  } catch { /* swallow */ }
+  cachePush(id, stored);
+  return stored;
+}
+
+export async function get(id, opts = {}) {
+  if (_cache.has(id)) return _cache.get(id);
+  const configDir = opts.configDir || defaultConfigDir();
+  const root = trajectoriesDir(configDir);
+  if (!fs.existsSync(root)) return null;
+  for (const bucket of fs.readdirSync(root)) {
+    const file = recordPath(configDir, bucket, id);
+    if (fs.existsSync(file)) {
+      const raw = fs.readFileSync(file, 'utf8').trim();
+      const rec = JSON.parse(raw);
+      cachePush(id, rec);
+      return rec;
+    }
+  }
+  return null;
+}
+
+export async function listByTaskId(taskId, opts = {}) {
+  const configDir = opts.configDir || defaultConfigDir();
+  const root = trajectoriesDir(configDir);
+  if (!fs.existsSync(root)) return [];
+  const matches = [];
+  for (const bucket of fs.readdirSync(root).sort()) {
+    const bdir = path.join(root, bucket);
+    if (!fs.statSync(bdir).isDirectory()) continue;
+    for (const f of fs.readdirSync(bdir).sort()) {
+      if (!f.endsWith('.jsonl')) continue;
+      try {
+        const rec = JSON.parse(fs.readFileSync(path.join(bdir, f), 'utf8'));
+        if (rec.taskId === taskId) matches.push(rec);
+      } catch { /* skip corrupt */ }
+    }
+  }
+  return matches;
+}
+
+// Test/maintenance hook.
+export function _resetCache() { _cache.clear(); }

package/mas/user_modeler.mjs

@@ -0,0 +1,108 @@
+// User modeler — Phase B (v5 §4.10, §9.2, §0.1 C6).
+//
+// Honcho-equivalent. At session end, take the session's turns and ask
+// the trainer to produce a dialectic update for ~/.lazyclaw/memory/USER.md:
+//
+//   ## Thesis      — durable facts the user just confirmed
+//   ## Antithesis  — contradictions to prior model (if any)
+//   ## Synthesis   — the reconciled, persisted summary
+//
+// The synthesis block is also fed to mas/index_db.mjs as a row of
+// fts_memories with kind='user_model' so recall() can pull user facts
+// at prompt assembly time.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+
+import { runTextCompletion } from './provider_adapters.mjs';
+import { redactSecrets, neutralizeRoleLabels } from './redact.mjs';
+
+const USER_MD_REL = path.join('memory', 'USER.md');
+const MAX_TRANSCRIPT_CHARS = 16 * 1024;
+const MAX_USER_MD_BYTES = 32 * 1024;
+
+export function defaultConfigDir() {
+  return process.env.LAZYCLAW_CONFIG_DIR || path.join(os.homedir(), '.lazyclaw');
+}
+
+export function userModelPath(configDir = defaultConfigDir()) {
+  return path.join(configDir, USER_MD_REL);
+}
+
+export function readUserModel(configDir = defaultConfigDir()) {
+  const p = userModelPath(configDir);
+  try { return fs.readFileSync(p, 'utf8'); } catch { return ''; }
+}
+
+function flattenTurns(turns) {
+  if (!Array.isArray(turns) || turns.length === 0) return '';
+  const text = turns
+    .map((t) => {
+      const who = t.role === 'user' ? 'User' : t.role === 'assistant' ? 'Assistant' : (t.role || 'unknown');
+      return `[${who}] ${neutralizeRoleLabels(String(t.content || ''))}`;
+    })
+    .join('\n\n');
+  return redactSecrets(text).slice(0, MAX_TRANSCRIPT_CHARS);
+}
+
+export async function updateUserModel({
+  sessionTurns,
+  provider,
+  model,
+  apiKey,
+  baseUrl,
+  fetchImpl,
+  configDir = defaultConfigDir(),
+  ts = new Date(),
+} = {}) {
+  const transcript = flattenTurns(sessionTurns);
+  if (!transcript.trim()) return null;
+
+  const prior = readUserModel(configDir).slice(-8 * 1024);
+  const userMessage =
+    `Below is a recent session transcript and the current USER model. ` +
+    `Update the model using a dialectic structure. Reply in EXACTLY this format:\n\n` +
+    `## Thesis\n<bullets of new durable facts about the user>\n\n` +
+    `## Antithesis\n<bullets of contradictions with the prior model, if any; "(none)" if none>\n\n` +
+    `## Synthesis\n<the reconciled model, ≤ 20 bullets, suitable for permanent storage>\n\n` +
+    `Prior USER model (may be empty):\n\n` + (prior || '(empty)') + `\n\n` +
+    `Session transcript:\n\n` + transcript;
+
+  let raw;
+  try {
+    raw = await runTextCompletion({
+      provider, model, system: 'You maintain a durable user model.',
+      userMessage, apiKey, baseUrl, fetchImpl,
+    });
+  } catch (err) {
+    return { path: userModelPath(configDir), error: String(err?.message || err) };
+  }
+  const cleaned = redactSecrets(String(raw || '')).trim();
+  if (!cleaned || !/##\s*Synthesis/i.test(cleaned)) return null;
+
+  const date = (ts instanceof Date ? ts : new Date(ts)).toISOString().slice(0, 10);
+  const header = `# USER\n\n_Last updated ${date}_\n\n`;
+  let body = header + cleaned + '\n';
+  if (Buffer.byteLength(body, 'utf8') > MAX_USER_MD_BYTES) {
+    body = Buffer.from(body, 'utf8').subarray(0, MAX_USER_MD_BYTES).toString('utf8') + '\n…[truncated]\n';
+  }
+
+  const p = userModelPath(configDir);
+  fs.mkdirSync(path.dirname(p), { recursive: true });
+  const tmp = p + '.tmp';
+  fs.writeFileSync(tmp, body);
+  fs.renameSync(tmp, p);
+
+  // Best-effort FTS5 mirror — only the Synthesis section is indexed.
+  try {
+    const synth = (cleaned.match(/##\s*Synthesis\s*\n([\s\S]*?)(?=\n##\s|$)/i) || [, ''])[1].trim();
+    if (synth) {
+      const idx = await import('./index_db.mjs');
+      idx.openIndex(configDir);
+      idx.indexMemory({ topic: 'USER', kind: 'user_model', content: synth }, configDir);
+    }
+  } catch { /* non-fatal */ }
+
+  return { path: p, body };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lazyclaw",
-  "version": "4.3.0",
+  "version": "5.0.0",
   "description": "Lazy, elegant terminal CLI for chatting with Claude / OpenAI / Gemini / Ollama, orchestrating multi-step LLM workflows, and running multi-agent Slack teams with cross-task memory. Banner-on-launch, slash-command ghost autocomplete, persistent sessions, local HTTP gateway.",
   "keywords": [
     "claude",
@@ -31,8 +31,12 @@
     "lazyclaw": "cli.mjs"
   },
   "scripts": {
-    "test": "playwright test",
-    "test:bench": "node scripts/bench-providers.mjs"
+    "test": "node --test tests/phaseC-*.test.mjs && playwright test",
+    "test:bench": "node scripts/bench-providers.mjs",
+    "test:bench:index": "node --test tests/index_store.bench.mjs",
+    "test:perf": "node --test tests/phaseH-perf.test.mjs tests/index_store.bench.mjs",
+    "migrate:v5": "node scripts/migrate-v5.mjs",
+    "build:splash": "node scripts/build-splash.mjs"
   },
   "files": [
     "cli.mjs",
@@ -47,6 +51,7 @@
     "workspace.mjs",
     "browse.mjs",
     "sandbox.mjs",
+    "sandbox/",
     "skills_install.mjs",
     "skills_curator.mjs",
     "cron.mjs",
@@ -65,12 +70,24 @@
     "gateway/",
     "docs/multi-agent.md",
     "scripts/loop-worker.mjs",
+    "scripts/migrate-v5.mjs",
+    "scripts/hermes-import.mjs",
+    "scripts/openclaw-import.mjs",
     "README.md",
     "LICENSE"
   ],
   "engines": {
     "node": ">=18"
   },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.18.0",
+    "better-sqlite3": "^11.10.0",
+    "chalk": "^5.3.0",
+    "ink": "^5.0.1",
+    "node-ssh": "^13.2.0",
+    "string-width": "^7.2.0",
+    "undici": "^6.21.0"
+  },
   "devDependencies": {
     "@playwright/test": "^1.59.1",
     "@types/node": "^25.6.0",

package/providers/codex_cli.mjs

@@ -0,0 +1,200 @@
+// Codex subscription/CLI provider (no API key).
+//
+// Spawns the local `codex` CLI binary (OpenAI's Codex CLI shipped from
+// npm: @openai/codex) in non-interactive mode:
+//
+//   codex exec --skip-git-repo-check --json [-m model] "<prompt>"
+//
+// Auth is whatever `codex` is signed into — a ChatGPT Plus/Pro/Business
+// session bound to ~/.codex/auth — so no OPENAI_API_KEY is required and
+// no key lands in the lazyclaw config. This is the CLI counterpart to
+// providers/openai.mjs (which talks to api.openai.com and needs an API
+// key).
+//
+// Output protocol (NDJSON on stdout, one JSON object per line):
+//   {"type":"thread.started", "thread_id": "..."}
+//   {"type":"turn.started"}
+//   {"type":"item.completed", "item": {"type":"agent_message","text":"..."}}
+//   {"type":"turn.completed", "usage": {...}}
+//
+// We stream-parse line-by-line and yield the `text` field of every
+// agent_message item. Reasoning items (`item.type === 'reasoning'`)
+// are skipped — they're internal thought summaries, not the visible
+// answer. Usage from turn.completed is surfaced via onUsage.
+//
+// Why `--skip-git-repo-check` is hard-coded: lazyclaw orchestrator
+// runs workers from scratch dirs (often /tmp) that aren't git repos,
+// and codex's default git-repo gate would reject every such spawn.
+// The flag is a UX shim for headless invocation, not a security
+// downgrade — we're handing codex our own prompt, not user-trusted
+// shell input.
+
+import { spawnSandboxed } from '../sandbox.mjs';
+
+class AbortError extends Error {
+  constructor(message = 'aborted') {
+    super(message);
+    this.name = 'AbortError';
+    this.code = 'ABORT';
+  }
+}
+
+class CliMissingError extends Error {
+  constructor() {
+    super('codex CLI not found in PATH — install @openai/codex or use the openai API provider');
+    this.name = 'CodexCliMissingError';
+    this.code = 'CLI_MISSING';
+  }
+}
+
+class CliExitError extends Error {
+  constructor(code, signal, stderr) {
+    super(`codex CLI exited ${code ?? signal}: ${String(stderr).slice(0, 400)}`);
+    this.name = 'CodexCliExitError';
+    this.code = 'CLI_EXIT';
+    this.exitCode = code;
+    this.signal = signal;
+    this.stderr = stderr;
+  }
+}
+
+// Map a few friendly aliases. Codex CLI accepts the literal model id
+// (e.g. gpt-5-codex) directly, so unknown inputs pass through.
+const _ALIASES = {
+  codex: 'gpt-5-codex',
+  'gpt-codex': 'gpt-5-codex',
+};
+
+// Drop cross-vendor model ids (claude-*, gemini-*) silently so the
+// CLI falls back to its own default. The orchestrator workflow forwards
+// cfg.model verbatim to every worker, and `providers test` does the
+// same — both would otherwise crash here when cfg.model is Anthropic
+// or Google.
+function resolveModel(model) {
+  if (!model) return '';
+  const lower = String(model).toLowerCase();
+  if (_ALIASES[lower]) return _ALIASES[lower];
+  if (
+    lower.startsWith('gpt-') ||
+    lower.startsWith('o1') ||
+    lower.startsWith('o3') ||
+    lower.startsWith('o4')
+  ) return String(model);
+  return '';
+}
+
+function buildPrompt(messages, system) {
+  const parts = [];
+  if (system) parts.push(`[System instructions: ${system}]`);
+  for (const m of messages) {
+    if (!m || !m.content) continue;
+    if (m.role === 'system' && !system) parts.push(`[System instructions: ${m.content}]`);
+    else if (m.role === 'user') parts.push(`User: ${m.content}`);
+    else if (m.role === 'assistant') parts.push(`Assistant: ${m.content}`);
+  }
+  return parts.length ? parts.join('\n') + '\n\nAssistant:' : '';
+}
+
+// Pull text out of a single NDJSON event. We only surface agent_message
+// items (the visible answer). Reasoning summaries are intentionally
+// dropped so the orchestrator's planner doesn't see the model's
+// internal deliberation as part of "the worker's answer".
+function extractEventText(obj) {
+  if (!obj || typeof obj !== 'object') return '';
+  if (obj.type !== 'item.completed') return '';
+  const item = obj.item || {};
+  if (item.type === 'agent_message' && typeof item.text === 'string') return item.text;
+  return '';
+}
+
+function extractUsage(obj) {
+  if (!obj || typeof obj !== 'object' || obj.type !== 'turn.completed') return null;
+  const u = obj.usage || {};
+  const input = (u.input_tokens ?? 0) + (u.cached_input_tokens ?? 0);
+  const output = (u.output_tokens ?? 0) + (u.reasoning_output_tokens ?? 0);
+  if (!input && !output) return null;
+  return { inputTokens: input, outputTokens: output, totalCostUsd: 0 };
+}
+
+export const codexCliProvider = {
+  name: 'codex-cli',
+  async *sendMessage(messages, opts = {}) {
+    const bin = opts.bin || 'codex';
+    const prompt = buildPrompt(messages, opts.system || messages.find(m => m.role === 'system')?.content);
+    if (!prompt) return;
+
+    const args = ['exec', '--skip-git-repo-check', '--json'];
+    const model = resolveModel(opts.model);
+    if (model) args.push('-m', model);
+    args.push(prompt);
+
+    if (opts.signal?.aborted) throw new AbortError('aborted before spawn');
+
+    let proc;
+    try {
+      proc = spawnSandboxed(opts.sandbox || null, bin, args, {
+        cwd: opts.cwd || process.cwd(),
+        stdio: ['ignore', 'pipe', 'pipe'],
+      });
+    } catch (err) {
+      if (err && err.code === 'ENOENT') throw new CliMissingError();
+      throw err;
+    }
+
+    const onAbort = () => { try { proc.kill('SIGTERM'); } catch (_) { /* ignore */ } };
+    if (opts.signal) opts.signal.addEventListener('abort', onAbort);
+
+    let stderr = '';
+    proc.stderr.setEncoding('utf8');
+    proc.stderr.on('data', (chunk) => { stderr += chunk; });
+
+    proc.stdout.setEncoding('utf8');
+    let buffer = '';
+    let exitInfo = null;
+    const exitPromise = new Promise((resolve) => {
+      proc.on('close', (code, signal) => {
+        exitInfo = { code, signal };
+        resolve();
+      });
+    });
+
+    try {
+      for await (const chunk of proc.stdout) {
+        if (opts.signal?.aborted) throw new AbortError('aborted mid-stream');
+        buffer += chunk;
+        let nl;
+        while ((nl = buffer.indexOf('\n')) >= 0) {
+          const line = buffer.slice(0, nl).trim();
+          buffer = buffer.slice(nl + 1);
+          if (!line) continue;
+          let obj;
+          try { obj = JSON.parse(line); } catch { continue; }
+          const text = extractEventText(obj);
+          if (text) yield text;
+          const usage = extractUsage(obj);
+          if (usage && typeof opts.onUsage === 'function') {
+            try { opts.onUsage(usage); } catch (_) { /* never break stream on usage */ }
+          }
+        }
+      }
+      if (buffer.trim()) {
+        try {
+          const obj = JSON.parse(buffer.trim());
+          const text = extractEventText(obj);
+          if (text) yield text;
+        } catch (_) { /* incomplete tail — drop */ }
+      }
+      await exitPromise;
+      if (exitInfo && exitInfo.code !== 0 && !opts.signal?.aborted) {
+        throw new CliExitError(exitInfo.code, exitInfo.signal, stderr);
+      }
+    } finally {
+      if (opts.signal) opts.signal.removeEventListener('abort', onAbort);
+      if (!proc.killed && exitInfo === null) {
+        try { proc.kill('SIGTERM'); } catch (_) { /* ignore */ }
+      }
+    }
+  },
+};
+
+export { CliMissingError, CliExitError, AbortError, resolveModel, buildPrompt, extractEventText, extractUsage };