lazyclaw 4.2.2 → 5.0.0

package/mas/toolsets.mjs

@@ -0,0 +1,64 @@
+// toolsets — named bundles of tool names that an agent can be assigned via
+// `lazyclaw agent edit <name> --toolset coding-min`. Built-ins ship in
+// code; user-defined sets live in <configDir>/toolsets.json.
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+const BUILTIN = {
+  'coding-min':  { tools: ['bash', 'read', 'write', 'edit', 'patch', 'grep', 'git_status', 'git_diff'] },
+  'web-research':{ tools: ['web_fetch', 'web_search', 'url_extract', 'read', 'write', 'recall'] },
+  'devops':      { tools: ['bash', 'git_status', 'git_diff', 'git_log', 'git_commit', 'cron_add', 'cron_list', 'http_request'] },
+  'learning':    { tools: ['recall', 'skill_view', 'skill_create', 'skill_edit', 'memory_read', 'memory_write', 'user_view', 'user_update'] },
+  'media':       { tools: ['image_describe', 'image_generate', 'transcribe'] },
+  'agentic':     { tools: ['task_spawn', 'delegate', 'clarify', 'recall', 'skill_view'] },
+};
+
+function configFile(opts) {
+  const dir = opts?.configDir || process.env.LAZYCLAW_CONFIG_DIR || path.join(process.env.HOME || '.', '.lazyclaw');
+  fs.mkdirSync(dir, { recursive: true });
+  return path.join(dir, 'toolsets.json');
+}
+
+function readUser(opts) {
+  const f = configFile(opts);
+  if (!fs.existsSync(f)) return {};
+  try { return JSON.parse(fs.readFileSync(f, 'utf8')); }
+  catch { return {}; }
+}
+
+function writeUser(data, opts) {
+  fs.writeFileSync(configFile(opts), JSON.stringify(data, null, 2));
+}
+
+export function listToolsets(opts) {
+  const user = readUser(opts);
+  const out = [];
+  for (const [name, t] of Object.entries(BUILTIN)) out.push({ name, ...t, source: 'builtin' });
+  for (const [name, t] of Object.entries(user))    out.push({ name, ...t, source: 'user' });
+  return out;
+}
+
+export function resolveToolset(name, opts) {
+  const user = readUser(opts);
+  const t = user[name] || BUILTIN[name];
+  if (!t || !Array.isArray(t.tools)) throw new Error(`toolset "${name}" not found`);
+  return [...t.tools];
+}
+
+export function addToolset({ name, tools }, opts) {
+  if (!name || !Array.isArray(tools)) throw new Error('addToolset: name + tools[] required');
+  if (BUILTIN[name]) throw new Error(`toolset "${name}" is built-in; pick a different name`);
+  const data = readUser(opts);
+  data[name] = { tools };
+  writeUser(data, opts);
+  return data[name];
+}
+
+export function removeToolset(name, opts) {
+  if (BUILTIN[name]) throw new Error(`cannot remove built-in toolset "${name}"`);
+  const data = readUser(opts);
+  delete data[name];
+  writeUser(data, opts);
+  return true;
+}

package/mas/trajectory_export.mjs

@@ -0,0 +1,169 @@
+// mas/trajectory_export.mjs — Phase H1.
+//
+// Trajectory exporter (spec §2.7). Read-only serialiser — never spawns
+// a trainer, never touches weights. Reads JSONL records produced by
+// mas/trajectory_store.mjs and emits one of four downstream formats:
+//
+//   jsonl     — raw transcripts (record verbatim, one per line)
+//   atropos   — NousResearch/atropos: {messages, reward, metadata}
+//   axolotl   — Axolotl ShareGPT: {conversations: [{from, value}]}
+//   openai-ft — OpenAI fine-tune: {messages: [{role, content}]}
+//
+// Filters: --since <Nd|Nh|Nm> (relative window), --outcome <done|failed|abandoned>.
+// `reward` for atropos is null by default per spec Appendix B.6 #22
+// (reward signal undefined for v5.0 — `--reward none` is the default).
+
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+
+export const FORMATS = Object.freeze(['atropos', 'axolotl', 'openai-ft', 'jsonl']);
+
+function defaultConfigDir() {
+  return process.env.LAZYCLAW_CONFIG_DIR || path.join(os.homedir(), '.lazyclaw');
+}
+
+function trajectoriesDir(configDir) {
+  return path.join(configDir, 'trajectories');
+}
+
+// Parse "7d" / "12h" / "30m" into a millisecond window relative to now.
+// Returns null if input is empty/undefined (no filter).
+function parseSince(spec) {
+  if (!spec) return null;
+  const m = String(spec).match(/^(\d+)\s*([dhm])$/i);
+  if (!m) throw new Error(`invalid --since: ${spec} (expected Nd|Nh|Nm)`);
+  const n = parseInt(m[1], 10);
+  const unit = m[2].toLowerCase();
+  const mult = unit === 'd' ? 86400_000 : unit === 'h' ? 3600_000 : 60_000;
+  return Date.now() - n * mult;
+}
+
+function* iterRecords(configDir) {
+  const root = trajectoriesDir(configDir);
+  if (!fs.existsSync(root)) return;
+  const buckets = fs.readdirSync(root).sort();
+  for (const bucket of buckets) {
+    const bdir = path.join(root, bucket);
+    let stat; try { stat = fs.statSync(bdir); } catch { continue; }
+    if (!stat.isDirectory()) continue;
+    for (const f of fs.readdirSync(bdir).sort()) {
+      if (!f.endsWith('.jsonl')) continue;
+      let raw;
+      try { raw = fs.readFileSync(path.join(bdir, f), 'utf8'); } catch { continue; }
+      for (const line of raw.split('\n')) {
+        const s = line.trim();
+        if (!s) continue;
+        try { yield JSON.parse(s); } catch { /* skip corrupt */ }
+      }
+    }
+  }
+}
+
+// Build a uniform message list from a TrajectoryRecord. The store's
+// canonical shape carries systemPrompt + userMessages + turns; downstream
+// formats all want a chronological role+content sequence so we synthesise
+// one here.
+function toMessages(rec) {
+  const out = [];
+  if (rec.systemPrompt) out.push({ role: 'system', content: String(rec.systemPrompt) });
+  for (const u of rec.userMessages || []) {
+    out.push({ role: 'user', content: String(u) });
+  }
+  for (const t of rec.turns || []) {
+    if (!t || !t.role) continue;
+    // turns may already include the user echo; keep them — downstream
+    // trainers expect duplicates filtered at their own dedupe stage.
+    out.push({ role: t.role, content: String(t.content || '') });
+  }
+  return out;
+}
+
+function formatAtropos(rec) {
+  return {
+    messages: toMessages(rec),
+    reward: null,                     // §B.6 #22 default
+    metadata: {
+      id: rec.id,
+      taskId: rec.taskId,
+      agentName: rec.agentName,
+      workerProvider: rec.workerProvider,
+      workerModel: rec.workerModel,
+      outcome: rec.outcome,
+      startedAt: rec.startedAt,
+      endedAt: rec.endedAt,
+    },
+  };
+}
+
+function formatAxolotl(rec) {
+  // ShareGPT role mapping: system→system, user→human, assistant→gpt,
+  // tool→tool. Keeps the channel a downstream Axolotl recipe can filter.
+  const FROM = { system: 'system', user: 'human', assistant: 'gpt', tool: 'tool' };
+  return {
+    conversations: toMessages(rec).map(m => ({
+      from: FROM[m.role] || m.role,
+      value: m.content,
+    })),
+  };
+}
+
+function formatOpenAIFT(rec) {
+  return { messages: toMessages(rec) };
+}
+
+function serialise(rec, format) {
+  switch (format) {
+    case 'jsonl':     return JSON.stringify(rec);
+    case 'atropos':   return JSON.stringify(formatAtropos(rec));
+    case 'axolotl':   return JSON.stringify(formatAxolotl(rec));
+    case 'openai-ft': return JSON.stringify(formatOpenAIFT(rec));
+    default: throw new Error(`unknown format: ${format}`);
+  }
+}
+
+// Public API. Returns { count, outFile, format }. Filesystem side effect
+// is a single .jsonl under <outDir> named `trajectories-<format>-<ts>.jsonl`.
+export async function exportTrajectories({
+  format,
+  configDir,
+  outDir,
+  since,
+  filter = {},
+} = {}) {
+  if (!FORMATS.includes(format)) throw new Error(`unknown format: ${format}`);
+  const cfgDir = configDir || defaultConfigDir();
+  const dest = outDir || path.join(cfgDir, 'exports');
+  fs.mkdirSync(dest, { recursive: true });
+
+  const sinceMs = parseSince(since);
+  const outcomeFilter = filter && filter.outcome;
+
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const outFile = path.join(dest, `trajectories-${format}-${stamp}.jsonl`);
+  const fd = fs.openSync(outFile, 'w');
+  let count = 0;
+  try {
+    for (const rec of iterRecords(cfgDir)) {
+      if (sinceMs !== null && (rec.startedAt || 0) < sinceMs) continue;
+      if (outcomeFilter && rec.outcome !== outcomeFilter) continue;
+      fs.writeSync(fd, serialise(rec, format) + '\n');
+      count++;
+    }
+  } finally {
+    fs.closeSync(fd);
+  }
+  return { count, outFile, format };
+}
+
+// Parse `outcome=done` style key=value filter strings into a plain object.
+// Exported so the CLI can share the parser.
+export function parseFilterArg(str) {
+  if (!str) return {};
+  const out = {};
+  for (const part of String(str).split(',')) {
+    const [k, v] = part.split('=').map(s => s && s.trim());
+    if (k && v) out[k] = v;
+  }
+  return out;
+}

package/mas/trajectory_store.mjs

@@ -0,0 +1,179 @@
+// mas/trajectory_store.mjs — Phase A.
+//
+// Persists TrajectoryRecord (spec §3.3) to JSONL on disk plus an
+// in-memory cache. Storage layout:
+//   <configDir>/trajectories/<YYYY-MM-DD>/<id>.jsonl
+// One file per trajectory id (a ULID) so concurrent writers never
+// contend. A single in-memory Map<id, record> serves hot reads; cold
+// reads stream the file back through JSON.parse.
+//
+// Phase A scope: write, read, list-by-task. Recall by full-text query
+// lives in mas/index_db.mjs (FTS5). The two stores share the same
+// record but the FTS5 mirror is best-effort — disk JSONL is the
+// source of truth.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import crypto from 'node:crypto';
+import { redactSecrets } from './redact.mjs';
+import { indexTrajectory as _indexTrajectory } from './index_db.mjs';
+
+export const OUTCOME_ENUM = Object.freeze(['done', 'failed', 'abandoned']);
+
+const _cache = new Map();   // id → record (capped at CACHE_MAX entries)
+const CACHE_MAX = 256;
+
+function defaultConfigDir() {
+  return process.env.LAZYCLAW_CONFIG_DIR || path.join(os.homedir(), '.lazyclaw');
+}
+
+function trajectoriesDir(configDir) {
+  return path.join(configDir, 'trajectories');
+}
+
+// Crockford-base32 ULID generator. Monotonic within a single ms by
+// appending a counter — same-millisecond puts stay sortable.
+const ULID_ALPHABET = '0123456789ABCDEFGHJKMNPQRSTVWXYZ';
+let _ulidLastMs = 0;
+let _ulidCounter = 0;
+let _ulidLastRandPrefix = '';
+function ulid() {
+  let now = Date.now();
+  let sameMs = (now === _ulidLastMs);
+  if (sameMs) _ulidCounter++;
+  else { _ulidLastMs = now; _ulidCounter = 0; }
+  let timePart = '';
+  let t = now;
+  for (let i = 0; i < 10; i++) {
+    timePart = ULID_ALPHABET[t % 32] + timePart;
+    t = Math.floor(t / 32);
+  }
+  // Hold the random prefix stable within a single ms so the counter
+  // suffix is the ONLY thing that changes — that's what keeps same-ms
+  // ULIDs lexicographically sortable (canonical ULID monotonicity).
+  if (!sameMs) {
+    const rand = crypto.randomBytes(10);
+    let randPart = '';
+    for (let i = 0; i < 16; i++) {
+      randPart += ULID_ALPHABET[rand[i % 10] % 32];
+    }
+    _ulidLastRandPrefix = randPart.slice(0, 14);
+  }
+  // Counter suffix (last 2 chars) keeps monotonicity intra-ms.
+  const ctr = ULID_ALPHABET[(_ulidCounter >> 5) % 32]
+            + ULID_ALPHABET[_ulidCounter % 32];
+  return timePart + _ulidLastRandPrefix + ctr;
+}
+
+function redactTurns(turns) {
+  return (turns || []).map(t => ({
+    ...t,
+    content: typeof t.content === 'string' ? redactSecrets(t.content) : t.content,
+    thinking: typeof t.thinking === 'string' ? redactSecrets(t.thinking) : t.thinking,
+    toolCalls: (t.toolCalls || []).map(c => ({
+      ...c,
+      result: typeof c.result === 'string' ? redactSecrets(c.result) : c.result,
+    })),
+  }));
+}
+
+function dateBucket(ms) {
+  return new Date(ms).toISOString().slice(0, 10);   // YYYY-MM-DD
+}
+
+function recordPath(configDir, bucket, id) {
+  return path.join(trajectoriesDir(configDir), bucket, `${id}.jsonl`);
+}
+
+function cachePush(id, rec) {
+  _cache.set(id, rec);
+  if (_cache.size > CACHE_MAX) {
+    const oldest = _cache.keys().next().value;
+    _cache.delete(oldest);
+  }
+}
+
+export async function put(record, opts = {}) {
+  if (!record || typeof record !== 'object') {
+    throw new TypeError('trajectory_store.put: record must be an object');
+  }
+  if (!OUTCOME_ENUM.includes(record.outcome)) {
+    throw new Error(
+      `outcome must be one of ${OUTCOME_ENUM.join('|')}, got ${record.outcome}`,
+    );
+  }
+  const configDir = opts.configDir || defaultConfigDir();
+  const id = record.id || ulid();
+  const stored = {
+    ...record,
+    id,
+    systemPrompt: typeof record.systemPrompt === 'string'
+      ? redactSecrets(record.systemPrompt) : '',
+    userMessages: (record.userMessages || []).map(m =>
+      typeof m === 'string' ? redactSecrets(m) : m),
+    turns: redactTurns(record.turns),
+    finalAnswer: typeof record.finalAnswer === 'string'
+      ? redactSecrets(record.finalAnswer) : '',
+  };
+  const bucket = dateBucket(stored.startedAt || Date.now());
+  const file = recordPath(configDir, bucket, id);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, JSON.stringify(stored) + '\n');
+  // Phase A: FTS5 mirror (spec §4.4). Content is the concatenation of
+  // the final answer plus every turn's textual content so a single
+  // recall() can surface trajectories by either signal.
+  try {
+    const ftsContent = [
+      stored.finalAnswer || '',
+      ...(stored.turns || []).map(t => String(t.content || '')),
+    ].filter(Boolean).join('\n');
+    _indexTrajectory({
+      trajectory_id: id,
+      agent: stored.agentName || '',
+      outcome: stored.outcome,
+      content: ftsContent,
+    }, configDir);
+  } catch { /* swallow */ }
+  cachePush(id, stored);
+  return stored;
+}
+
+export async function get(id, opts = {}) {
+  if (_cache.has(id)) return _cache.get(id);
+  const configDir = opts.configDir || defaultConfigDir();
+  const root = trajectoriesDir(configDir);
+  if (!fs.existsSync(root)) return null;
+  for (const bucket of fs.readdirSync(root)) {
+    const file = recordPath(configDir, bucket, id);
+    if (fs.existsSync(file)) {
+      const raw = fs.readFileSync(file, 'utf8').trim();
+      const rec = JSON.parse(raw);
+      cachePush(id, rec);
+      return rec;
+    }
+  }
+  return null;
+}
+
+export async function listByTaskId(taskId, opts = {}) {
+  const configDir = opts.configDir || defaultConfigDir();
+  const root = trajectoriesDir(configDir);
+  if (!fs.existsSync(root)) return [];
+  const matches = [];
+  for (const bucket of fs.readdirSync(root).sort()) {
+    const bdir = path.join(root, bucket);
+    if (!fs.statSync(bdir).isDirectory()) continue;
+    for (const f of fs.readdirSync(bdir).sort()) {
+      if (!f.endsWith('.jsonl')) continue;
+      try {
+        const rec = JSON.parse(fs.readFileSync(path.join(bdir, f), 'utf8'));
+        if (rec.taskId === taskId) matches.push(rec);
+      } catch { /* skip corrupt */ }
+    }
+  }
+  return matches;
+}
+
+// Test/maintenance hook.
+export function _resetCache() { _cache.clear(); }

package/mas/user_modeler.mjs

@@ -0,0 +1,108 @@
+// User modeler — Phase B (v5 §4.10, §9.2, §0.1 C6).
+//
+// Honcho-equivalent. At session end, take the session's turns and ask
+// the trainer to produce a dialectic update for ~/.lazyclaw/memory/USER.md:
+//
+//   ## Thesis      — durable facts the user just confirmed
+//   ## Antithesis  — contradictions to prior model (if any)
+//   ## Synthesis   — the reconciled, persisted summary
+//
+// The synthesis block is also fed to mas/index_db.mjs as a row of
+// fts_memories with kind='user_model' so recall() can pull user facts
+// at prompt assembly time.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+
+import { runTextCompletion } from './provider_adapters.mjs';
+import { redactSecrets, neutralizeRoleLabels } from './redact.mjs';
+
+const USER_MD_REL = path.join('memory', 'USER.md');
+const MAX_TRANSCRIPT_CHARS = 16 * 1024;
+const MAX_USER_MD_BYTES = 32 * 1024;
+
+export function defaultConfigDir() {
+  return process.env.LAZYCLAW_CONFIG_DIR || path.join(os.homedir(), '.lazyclaw');
+}
+
+export function userModelPath(configDir = defaultConfigDir()) {
+  return path.join(configDir, USER_MD_REL);
+}
+
+export function readUserModel(configDir = defaultConfigDir()) {
+  const p = userModelPath(configDir);
+  try { return fs.readFileSync(p, 'utf8'); } catch { return ''; }
+}
+
+function flattenTurns(turns) {
+  if (!Array.isArray(turns) || turns.length === 0) return '';
+  const text = turns
+    .map((t) => {
+      const who = t.role === 'user' ? 'User' : t.role === 'assistant' ? 'Assistant' : (t.role || 'unknown');
+      return `[${who}] ${neutralizeRoleLabels(String(t.content || ''))}`;
+    })
+    .join('\n\n');
+  return redactSecrets(text).slice(0, MAX_TRANSCRIPT_CHARS);
+}
+
+export async function updateUserModel({
+  sessionTurns,
+  provider,
+  model,
+  apiKey,
+  baseUrl,
+  fetchImpl,
+  configDir = defaultConfigDir(),
+  ts = new Date(),
+} = {}) {
+  const transcript = flattenTurns(sessionTurns);
+  if (!transcript.trim()) return null;
+
+  const prior = readUserModel(configDir).slice(-8 * 1024);
+  const userMessage =
+    `Below is a recent session transcript and the current USER model. ` +
+    `Update the model using a dialectic structure. Reply in EXACTLY this format:\n\n` +
+    `## Thesis\n<bullets of new durable facts about the user>\n\n` +
+    `## Antithesis\n<bullets of contradictions with the prior model, if any; "(none)" if none>\n\n` +
+    `## Synthesis\n<the reconciled model, ≤ 20 bullets, suitable for permanent storage>\n\n` +
+    `Prior USER model (may be empty):\n\n` + (prior || '(empty)') + `\n\n` +
+    `Session transcript:\n\n` + transcript;
+
+  let raw;
+  try {
+    raw = await runTextCompletion({
+      provider, model, system: 'You maintain a durable user model.',
+      userMessage, apiKey, baseUrl, fetchImpl,
+    });
+  } catch (err) {
+    return { path: userModelPath(configDir), error: String(err?.message || err) };
+  }
+  const cleaned = redactSecrets(String(raw || '')).trim();
+  if (!cleaned || !/##\s*Synthesis/i.test(cleaned)) return null;
+
+  const date = (ts instanceof Date ? ts : new Date(ts)).toISOString().slice(0, 10);
+  const header = `# USER\n\n_Last updated ${date}_\n\n`;
+  let body = header + cleaned + '\n';
+  if (Buffer.byteLength(body, 'utf8') > MAX_USER_MD_BYTES) {
+    body = Buffer.from(body, 'utf8').subarray(0, MAX_USER_MD_BYTES).toString('utf8') + '\n…[truncated]\n';
+  }
+
+  const p = userModelPath(configDir);
+  fs.mkdirSync(path.dirname(p), { recursive: true });
+  const tmp = p + '.tmp';
+  fs.writeFileSync(tmp, body);
+  fs.renameSync(tmp, p);
+
+  // Best-effort FTS5 mirror — only the Synthesis section is indexed.
+  try {
+    const synth = (cleaned.match(/##\s*Synthesis\s*\n([\s\S]*?)(?=\n##\s|$)/i) || [, ''])[1].trim();
+    if (synth) {
+      const idx = await import('./index_db.mjs');
+      idx.openIndex(configDir);
+      idx.indexMemory({ topic: 'USER', kind: 'user_model', content: synth }, configDir);
+    }
+  } catch { /* non-fatal */ }
+
+  return { path: p, body };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lazyclaw",
-  "version": "4.2.2",
+  "version": "5.0.0",
   "description": "Lazy, elegant terminal CLI for chatting with Claude / OpenAI / Gemini / Ollama, orchestrating multi-step LLM workflows, and running multi-agent Slack teams with cross-task memory. Banner-on-launch, slash-command ghost autocomplete, persistent sessions, local HTTP gateway.",
   "keywords": [
     "claude",
@@ -31,8 +31,12 @@
     "lazyclaw": "cli.mjs"
   },
   "scripts": {
-    "test": "playwright test",
-    "test:bench": "node scripts/bench-providers.mjs"
+    "test": "node --test tests/phaseC-*.test.mjs && playwright test",
+    "test:bench": "node scripts/bench-providers.mjs",
+    "test:bench:index": "node --test tests/index_store.bench.mjs",
+    "test:perf": "node --test tests/phaseH-perf.test.mjs tests/index_store.bench.mjs",
+    "migrate:v5": "node scripts/migrate-v5.mjs",
+    "build:splash": "node scripts/build-splash.mjs"
   },
   "files": [
     "cli.mjs",
@@ -47,7 +51,9 @@
     "workspace.mjs",
     "browse.mjs",
     "sandbox.mjs",
+    "sandbox/",
     "skills_install.mjs",
+    "skills_curator.mjs",
     "cron.mjs",
     "loop-engine.mjs",
     "loops.mjs",
@@ -61,14 +67,27 @@
     "workflow/",
     "web/",
     "mas/",
+    "gateway/",
     "docs/multi-agent.md",
     "scripts/loop-worker.mjs",
+    "scripts/migrate-v5.mjs",
+    "scripts/hermes-import.mjs",
+    "scripts/openclaw-import.mjs",
     "README.md",
     "LICENSE"
   ],
   "engines": {
     "node": ">=18"
   },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.18.0",
+    "better-sqlite3": "^11.10.0",
+    "chalk": "^5.3.0",
+    "ink": "^5.0.1",
+    "node-ssh": "^13.2.0",
+    "string-width": "^7.2.0",
+    "undici": "^6.21.0"
+  },
   "devDependencies": {
     "@playwright/test": "^1.59.1",
     "@types/node": "^25.6.0",