create-walle 0.9.13 → 0.9.15

package/template/wall-e/brain.js

@@ -431,6 +431,7 @@ function _detectUpgrade(previousSchemaVersion) {
           migrations_applied: Math.max(0, SCHEMA_VERSION - previousSchemaVersion),
           first_tracked: true,
         });
+        telemetry.trackFunnelStep?.('upgrade_completed');
         setKv('last_upgrade_at', new Date().toISOString());
       } else {
         setKv('first_installed_at', new Date().toISOString());
@@ -444,6 +445,7 @@ function _detectUpgrade(previousSchemaVersion) {
         schema_to: SCHEMA_VERSION,
         migrations_applied: SCHEMA_VERSION - previousSchemaVersion,
       });
+      telemetry.trackFunnelStep?.('upgrade_completed');
       setKv('installed_version', pkgVersion);
       setKv('last_upgrade_at', new Date().toISOString());
     }
@@ -495,6 +497,27 @@ function getDbPath() {
   return currentDbPath || DEFAULT_DB_PATH;
 }
 
+const _VALID_CHECKPOINT_MODES = new Set(['PASSIVE', 'FULL', 'RESTART', 'TRUNCATE']);
+function checkpointWalOrThrow(mode) {
+  const d = getDb();
+  const m = (mode || 'PASSIVE').toUpperCase();
+  if (!_VALID_CHECKPOINT_MODES.has(m)) throw new Error(`Invalid WAL checkpoint mode: ${mode}`);
+  const rows = d.pragma(`wal_checkpoint(${m})`);
+  const row = Array.isArray(rows) ? rows[0] : rows;
+  const busy = Number(row?.busy ?? row?.[0] ?? 0);
+  if (busy > 0) {
+    const log = row?.log ?? row?.[1] ?? 'unknown';
+    const checkpointed = row?.checkpointed ?? row?.[2] ?? 'unknown';
+    throw new Error(`WAL checkpoint ${m} could not complete: busy=${busy}, log=${log}, checkpointed=${checkpointed}`);
+  }
+  return rows;
+}
+
+function runImmediateTransaction(d, fn, ...args) {
+  const txn = d.transaction(fn);
+  return typeof txn.immediate === 'function' ? txn.immediate(...args) : txn(...args);
+}
+
 function initDb(dbPath) {
   dbPath = dbPath || DEFAULT_DB_PATH;
   currentDbPath = dbPath;
@@ -936,6 +959,21 @@ function createTables() {
     );
     CREATE INDEX IF NOT EXISTS idx_slack_threads_active ON slack_threads(last_activity);
 
+    CREATE TABLE IF NOT EXISTS slack_inbound_events (
+      id TEXT PRIMARY KEY,
+      channel_id TEXT NOT NULL,
+      message_ts TEXT NOT NULL,
+      thread_ts TEXT,
+      source TEXT,
+      status TEXT DEFAULT 'claimed',
+      payload_hash TEXT,
+      error TEXT,
+      created_at TEXT DEFAULT (datetime('now')),
+      updated_at TEXT DEFAULT (datetime('now'))
+    );
+    CREATE INDEX IF NOT EXISTS idx_slack_inbound_events_updated ON slack_inbound_events(updated_at);
+    CREATE INDEX IF NOT EXISTS idx_slack_inbound_events_thread ON slack_inbound_events(channel_id, thread_ts);
+
     -- Model management tables
     CREATE TABLE IF NOT EXISTS model_providers (
       id TEXT PRIMARY KEY,
@@ -1397,11 +1435,27 @@ function listMemories({ source, since, extractionStatus, limit } = {}) {
   return getDb().prepare(sql).all(...params);
 }
 
-function searchMemories({ query, limit } = {}) {
+function searchMemories({ query, limit, source, memory_type, since, until } = {}) {
   if (!query || !query.trim()) return [];
   const terms = query.trim().split(/\s+/).filter(Boolean);
   const conditions = terms.map(() => "content LIKE ? ESCAPE '\\'");
   const params = terms.map(t => '%' + t.replace(/[%_\\]/g, '\\$&') + '%');
+  if (source) {
+    conditions.push('source = ?');
+    params.push(source);
+  }
+  if (memory_type) {
+    conditions.push('memory_type = ?');
+    params.push(memory_type);
+  }
+  if (since) {
+    conditions.push('timestamp >= ?');
+    params.push(since);
+  }
+  if (until) {
+    conditions.push('timestamp <= ?');
+    params.push(until);
+  }
   const lim = Math.min(Math.max(limit || 50, 1), 200);
   params.push(lim);
   return getDb().prepare(
@@ -1633,7 +1687,7 @@ let backupIntervalId = null;
 
 function createBackup(label) {
   getDb(); // throws if not initialized
-  db.pragma('wal_checkpoint(TRUNCATE)');
+  checkpointWalOrThrow('TRUNCATE');
 
   // Use backup dir relative to the current DB, not the module-level constant.
   // This prevents test runs from writing backups to the production backup dir.
@@ -1842,18 +1896,20 @@ function _editDistance(a, b) {
 
 function mergeEntities(keepId, removeId) {
   const db = getDb();
-  const keep = getEntity(keepId);
-  const remove = getEntity(removeId);
-  if (!keep || !remove) throw new Error('Both entities must exist');
-  let keepAliases = [], removeAliases = [];
-  try { keepAliases = JSON.parse(keep.aliases || '[]'); } catch {}
-  try { removeAliases = JSON.parse(remove.aliases || '[]'); } catch {}
-  const mergedAliases = [...new Set([...keepAliases, ...removeAliases, remove.canonical_name])];
-  db.prepare("UPDATE entities SET aliases = ?, updated_at = datetime('now') WHERE id = ?").run(JSON.stringify(mergedAliases), keepId);
-  db.prepare('UPDATE knowledge SET subject_entity_id = ? WHERE subject_entity_id = ?').run(keepId, removeId);
-  db.prepare('UPDATE knowledge SET object_entity_id = ? WHERE object_entity_id = ?').run(keepId, removeId);
-  db.prepare('DELETE FROM entities WHERE id = ?').run(removeId);
-  return { merged: true, kept: keepId, removed: removeId, aliases: mergedAliases };
+  return runImmediateTransaction(db, (targetKeepId, targetRemoveId) => {
+    const keep = db.prepare('SELECT * FROM entities WHERE id = ?').get(targetKeepId);
+    const remove = db.prepare('SELECT * FROM entities WHERE id = ?').get(targetRemoveId);
+    if (!keep || !remove) throw new Error('Both entities must exist');
+    let keepAliases = [], removeAliases = [];
+    try { keepAliases = JSON.parse(keep.aliases || '[]'); } catch {}
+    try { removeAliases = JSON.parse(remove.aliases || '[]'); } catch {}
+    const mergedAliases = [...new Set([...keepAliases, ...removeAliases, remove.canonical_name])];
+    db.prepare("UPDATE entities SET aliases = ?, updated_at = datetime('now') WHERE id = ?").run(JSON.stringify(mergedAliases), targetKeepId);
+    db.prepare('UPDATE knowledge SET subject_entity_id = ? WHERE subject_entity_id = ?').run(targetKeepId, targetRemoveId);
+    db.prepare('UPDATE knowledge SET object_entity_id = ? WHERE object_entity_id = ?').run(targetKeepId, targetRemoveId);
+    db.prepare('DELETE FROM entities WHERE id = ?').run(targetRemoveId);
+    return { merged: true, kept: targetKeepId, removed: targetRemoveId, aliases: mergedAliases };
+  }, keepId, removeId);
 }
 
 function listEntities({ entity_type, limit } = {}) {
@@ -2247,14 +2303,16 @@ function searchChatMessages({ query, limit } = {}) {
 
 function deleteChatMessages({ session_id, user_content, assistant_content }) {
   const db = getDb();
-  if (user_content) {
-    db.prepare('DELETE FROM chat_messages WHERE session_id = ? AND role = ? AND content = ?')
-      .run(session_id || 'default', 'user', user_content);
-  }
-  if (assistant_content) {
-    db.prepare('DELETE FROM chat_messages WHERE session_id = ? AND role = ? AND content = ?')
-      .run(session_id || 'default', 'assistant', assistant_content);
-  }
+  return runImmediateTransaction(db, (targetSessionId, userContent, assistantContent) => {
+    if (userContent) {
+      db.prepare('DELETE FROM chat_messages WHERE session_id = ? AND role = ? AND content = ?')
+        .run(targetSessionId || 'default', 'user', userContent);
+    }
+    if (assistantContent) {
+      db.prepare('DELETE FROM chat_messages WHERE session_id = ? AND role = ? AND content = ?')
+        .run(targetSessionId || 'default', 'assistant', assistantContent);
+    }
+  }, session_id, user_content, assistant_content);
 }
 
 function clearChatSession(session_id) {
@@ -2379,11 +2437,14 @@ function updateSkillStats(skillId, success) {
 }
 
 function deleteSkill(id) {
-  const skill = getDb().prepare('SELECT * FROM skills WHERE id = ?').get(id);
-  if (!skill) throw new Error(`Skill not found: ${id}`);
-  getDb().prepare('DELETE FROM skill_executions WHERE skill_id = ?').run(id);
-  getDb().prepare('DELETE FROM skills WHERE id = ?').run(id);
-  return skill;
+  const db = getDb();
+  return runImmediateTransaction(db, (skillId) => {
+    const skill = db.prepare('SELECT * FROM skills WHERE id = ?').get(skillId);
+    if (!skill) throw new Error(`Skill not found: ${skillId}`);
+    db.prepare('DELETE FROM skill_executions WHERE skill_id = ?').run(skillId);
+    db.prepare('DELETE FROM skills WHERE id = ?').run(skillId);
+    return skill;
+  }, id);
 }
 
 // ── Skill dispatch decision log (kv-backed) ──
@@ -2496,21 +2557,25 @@ function deleteAuthProfile(id) {
 // ── Tasks ──
 
 function insertTask({ title, description, priority, type, execution, script, schedule, due_at, next_run_at, skill, skill_config, source, source_ref }) {
-  // Dedup: skip if a task with the same title+source+source_ref already exists and isn't archived
-  if (title && source) {
-    const existing = source_ref
-      ? getDb().prepare(`SELECT id, status FROM tasks WHERE title = ? AND source = ? AND source_ref = ? AND status != 'archived' LIMIT 1`).get(title, source, source_ref)
-      : getDb().prepare(`SELECT id, status FROM tasks WHERE title = ? AND source = ? AND status NOT IN ('archived','completed') LIMIT 1`).get(title, source);
-    if (existing) return { id: existing.id, deduplicated: true };
-  }
-  const id = uuidv4();
-  getDb().prepare(`
-    INSERT INTO tasks (id, title, description, priority, type, execution, script, schedule, due_at, next_run_at, skill, skill_config, source, source_ref)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-  `).run(id, title, description || null, priority || 'normal', type || 'once',
-    execution || 'chat', script || null, schedule || null, due_at || null, next_run_at || due_at || null,
-    skill || null, skill_config || null, source || 'system', source_ref || null);
-  return { id };
+  const db = getDb();
+  return runImmediateTransaction(db, (task) => {
+    const taskSource = task.source || 'system';
+    // Dedup: skip if a task with the same title+source+source_ref already exists and isn't archived.
+    if (task.title && taskSource) {
+      const existing = task.source_ref
+        ? db.prepare(`SELECT id, status FROM tasks WHERE title = ? AND source = ? AND source_ref = ? AND status != 'archived' LIMIT 1`).get(task.title, taskSource, task.source_ref)
+        : db.prepare(`SELECT id, status FROM tasks WHERE title = ? AND source = ? AND status NOT IN ('archived','completed') LIMIT 1`).get(task.title, taskSource);
+      if (existing) return { id: existing.id, deduplicated: true };
+    }
+    const id = uuidv4();
+    db.prepare(`
+      INSERT INTO tasks (id, title, description, priority, type, execution, script, schedule, due_at, next_run_at, skill, skill_config, source, source_ref)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(id, task.title, task.description || null, task.priority || 'normal', task.type || 'once',
+      task.execution || 'chat', task.script || null, task.schedule || null, task.due_at || null, task.next_run_at || task.due_at || null,
+      task.skill || null, task.skill_config || null, taskSource, task.source_ref || null);
+    return { id };
+  }, { title, description, priority, type, execution, script, schedule, due_at, next_run_at, skill, skill_config, source, source_ref });
 }
 
 function getTask(id) {
@@ -2710,6 +2775,48 @@ function deleteExpiredSlackThreads(watchDurationMs) {
   return getDb().prepare('DELETE FROM slack_threads WHERE last_activity < ?').run(cutoff).changes;
 }
 
+// -- Slack Inbound Event Ledger --
+
+function slackInboundEventId(channelId, messageTs) {
+  return `${channelId}:${messageTs}`;
+}
+
+function claimSlackInboundEvent({ channelId, messageTs, threadTs, source, payloadHash } = {}) {
+  if (!channelId || !messageTs) throw new Error('Slack inbound event requires channelId and messageTs');
+  const id = slackInboundEventId(channelId, messageTs);
+  const now = new Date().toISOString();
+  const result = getDb().prepare(`
+    INSERT OR IGNORE INTO slack_inbound_events
+      (id, channel_id, message_ts, thread_ts, source, status, payload_hash, updated_at)
+    VALUES (?, ?, ?, ?, ?, 'claimed', ?, ?)
+  `).run(id, channelId, messageTs, threadTs || null, source || null, payloadHash || null, now);
+
+  if (result.changes > 0) return { id, claimed: true };
+  const event = getDb().prepare('SELECT * FROM slack_inbound_events WHERE id = ?').get(id);
+  return { id, claimed: false, event };
+}
+
+function completeSlackInboundEvent(id, { status = 'completed', error = null } = {}) {
+  if (!id) return 0;
+  const now = new Date().toISOString();
+  const result = getDb().prepare(`
+    UPDATE slack_inbound_events
+    SET status = ?, error = ?, updated_at = ?
+    WHERE id = ?
+  `).run(status, error || null, now, id);
+  return result.changes;
+}
+
+function releaseSlackInboundEvent(id) {
+  if (!id) return 0;
+  return getDb().prepare('DELETE FROM slack_inbound_events WHERE id = ?').run(id).changes;
+}
+
+function pruneSlackInboundEvents(ttlMs = 48 * 60 * 60 * 1000) {
+  const cutoff = new Date(Date.now() - ttlMs).toISOString();
+  return getDb().prepare('DELETE FROM slack_inbound_events WHERE updated_at < ?').run(cutoff).changes;
+}
+
 // -- Model Provider CRUD --
 
 function upsertModelProvider({ id, name, type, baseUrl, apiKeyEncrypted, customHeaders, enabled }) {
@@ -2776,6 +2883,121 @@ function getProviderAuthMethod(type) {
   return row?.auth_method || null;
 }
 
+function saveSetupProvider({
+  type,
+  name,
+  id,
+  baseUrl,
+  apiKeyEncrypted,
+  customHeaders,
+  enabled = true,
+  model,
+  setDefault = false,
+  authMethod,
+  preserveExistingKey = true,
+  registerModel = true,
+}) {
+  if (!type) throw new Error('Provider type required');
+  if (authMethod && !VALID_AUTH_METHODS.has(authMethod)) {
+    throw new Error(`Invalid auth_method: ${authMethod}. Must be one of: ${[...VALID_AUTH_METHODS].join(', ')}`);
+  }
+  const providerId = id || `${type}-default`;
+  const displayName = name || type;
+  const db = getDb();
+  return runImmediateTransaction(db, () => {
+    const currentDefaultProvider = getKv('walle_provider') || process.env.WALLE_PROVIDER || 'anthropic';
+    const syncDefaultModel = !!(model && currentDefaultProvider === type && !setDefault);
+    let storedKey = apiKeyEncrypted || null;
+    if (!storedKey && preserveExistingKey) {
+      const existingProvider = getModelProviderWithKey(providerId);
+      storedKey = existingProvider?.api_key_encrypted || null;
+      if (!storedKey) {
+        const row = db.prepare(
+          'SELECT api_key_encrypted FROM model_providers WHERE type = ? AND enabled = 1 AND api_key_encrypted IS NOT NULL ORDER BY updated_at DESC LIMIT 1'
+        ).get(type);
+        storedKey = row?.api_key_encrypted || null;
+      }
+    }
+
+    upsertModelProvider({
+      id: providerId,
+      name: displayName,
+      type,
+      baseUrl: baseUrl || null,
+      apiKeyEncrypted: storedKey,
+      customHeaders: customHeaders || null,
+      enabled,
+    });
+    if (authMethod) setProviderAuthMethod(type, authMethod);
+    if (model) {
+      setKv(`walle_model_${type}`, model);
+      if (syncDefaultModel) setKv('walle_model', model);
+      if (registerModel) {
+        upsertModelRegistryEntry({
+          id: `${providerId}:${model}`,
+          providerId,
+          modelId: model,
+          displayName: model,
+          enabled: true,
+        });
+      }
+    }
+    if (setDefault) {
+      setKv('walle_provider', type);
+      if (model) setKv('walle_model', model);
+    }
+
+    return {
+      provider_id: providerId,
+      type,
+      default_provider: getKv('walle_provider') || null,
+      default_model: getKv('walle_model') || null,
+      synced_default_model: syncDefaultModel,
+      registry_id: model && registerModel ? `${providerId}:${model}` : null,
+    };
+  });
+}
+
+function setDefaultProviderSelection({ type, requestedModel, targetModel }) {
+  if (!type) throw new Error('Provider type required');
+  const db = getDb();
+  return runImmediateTransaction(db, () => {
+    setKv('walle_provider', type);
+    if (requestedModel) setKv(`walle_model_${type}`, requestedModel);
+    setKv('walle_model', targetModel || '');
+    return {
+      type,
+      default_provider: getKv('walle_provider') || null,
+      default_model: getKv('walle_model') || '',
+    };
+  });
+}
+
+function disableModelProviderByType(type) {
+  if (!type) throw new Error('Provider type required');
+  const db = getDb();
+  return runImmediateTransaction(db, () => {
+    const changes = db.prepare('UPDATE model_providers SET enabled = 0, updated_at = datetime(\'now\') WHERE type = ?').run(type).changes;
+    let fallback = null;
+    if (getKv('walle_provider') === type) {
+      const enabled = listEnabledProviders();
+      if (enabled.length > 0) {
+        fallback = enabled[0];
+        setKv('walle_provider', fallback.type);
+        const fallbackModel = getKv(`walle_model_${fallback.type}`);
+        if (fallbackModel) setKv('walle_model', fallbackModel);
+      }
+    }
+    return {
+      type,
+      disabled: changes,
+      fallback_provider: fallback?.type || null,
+      default_provider: getKv('walle_provider') || null,
+      default_model: getKv('walle_model') || null,
+    };
+  });
+}
+
 // -- Model Registry CRUD --
 
 function upsertModelRegistryEntry({ id, providerId, modelId, displayName, capabilities, costPer1mInput, costPer1mOutput, maxContextTokens, maxOutputTokens, speedTier, enabled, isFineTuned }) {
@@ -2955,6 +3177,24 @@ function setModelDefault(taskType, modelRegistryId, strategy, quorumSize) {
   `).run(taskType, modelRegistryId || null, strategy || 'single', quorumSize || 1);
 }
 
+function setTaskModelDefault({ taskType, modelRegistryId, strategy, quorumSize }) {
+  if (!taskType) throw new Error('Default requires taskType');
+  const db = getDb();
+  return runImmediateTransaction(db, () => {
+    if (modelRegistryId) {
+      const exists = db.prepare('SELECT 1 FROM model_registry WHERE id = ?').get(modelRegistryId);
+      if (!exists) throw new Error('model_registry_id does not match any registered model');
+    }
+    setModelDefault(taskType, modelRegistryId, strategy, quorumSize);
+    return {
+      task_type: taskType,
+      model_registry_id: modelRegistryId || null,
+      strategy: strategy || 'single',
+      quorum_size: quorumSize || 1,
+    };
+  });
+}
+
 function getModelDefault(taskType) {
   return getDb().prepare('SELECT * FROM model_defaults WHERE task_type = ?').get(taskType) || null;
 }
@@ -3810,6 +4050,7 @@ module.exports = {
   saveChatBranches,
   loadChatBranches,
   // Backup
+  checkpointWalOrThrow,
   createBackup,
   listBackups,
   deleteBackup,
@@ -3861,6 +4102,10 @@ module.exports = {
   listActiveSlackThreads,
   updateSlackThread,
   deleteExpiredSlackThreads,
+  claimSlackInboundEvent,
+  completeSlackInboundEvent,
+  releaseSlackInboundEvent,
+  pruneSlackInboundEvents,
   // Model Providers
   upsertModelProvider,
   getModelProvider,
@@ -3870,6 +4115,9 @@ module.exports = {
   getProviderByType,
   setProviderAuthMethod,
   getProviderAuthMethod,
+  saveSetupProvider,
+  setDefaultProviderSelection,
+  disableModelProviderByType,
   VALID_AUTH_METHODS,
   SCHEMA_VERSION,
   deleteModelProvider,
@@ -3888,6 +4136,7 @@ module.exports = {
   getAgentRunnerScorecard,
   // Model Defaults
   setModelDefault,
+  setTaskModelDefault,
   getModelDefault,
   listModelDefaults,
   // Benchmark Evaluations