create-walle 0.1.0

package/template/wall-e/scripts/pull-slack-via-claude.js

@@ -0,0 +1,98 @@
+#!/usr/bin/env node
+/**
+ * Pull Slack history into WALL-E brain by parsing search results.
+ * This script is meant to be run manually — it reads pre-fetched search results
+ * from stdin or a file and ingests them into the brain.
+ *
+ * Usage:
+ *   node scripts/pull-slack-via-claude.js < /tmp/slack-results.txt
+ *   node scripts/pull-slack-via-claude.js /tmp/slack-results.txt
+ */
+'use strict';
+const fs = require('fs');
+const brain = require('../brain');
+
+brain.initDb();
+
+const OWNER_ID = process.env.SLACK_OWNER_USER_ID;
+if (!OWNER_ID) throw new Error('SLACK_OWNER_USER_ID env var required');
+const OWNER_NAME = process.env.WALLE_OWNER_NAME || 'Owner';
+
+function parseSearchResults(text) {
+  const messages = [];
+  const blocks = text.split('---').filter(b => b.trim());
+
+  for (const block of blocks) {
+    const lines = block.trim().split('\n');
+    let channel = '', channelId = '', sender = '', senderId = '', ts = '', msgTs = '', msgText = '';
+    let participants = [];
+
+    for (const line of lines) {
+      const l = line.trim();
+      if (l.startsWith('Channel:')) {
+        const m = l.match(/Channel:\s*(.*?)\s*\(ID:\s*(.*?)\)/);
+        if (m) { channel = m[1].trim(); channelId = m[2]; }
+      }
+      if (l.startsWith('From:')) {
+        const m = l.match(/From:\s*(.*?)\s*\(ID:\s*(.*?)\)/);
+        if (m) { sender = m[1].trim(); senderId = m[2]; }
+      }
+      if (l.startsWith('Time:')) {
+        ts = l.replace('Time:', '').trim();
+      }
+      if (l.startsWith('Message_ts:')) {
+        msgTs = l.replace('Message_ts:', '').trim();
+      }
+      if (l.startsWith('Text:')) {
+        msgText = lines.slice(lines.indexOf(line) + 1).join('\n').split('Context')[0].trim();
+        break;
+      }
+      if (l.startsWith('Participants:')) {
+        const pMatch = l.matchAll(/(\w[\w\s]+?)\s*\(ID:/g);
+        for (const pm of pMatch) participants.push(pm[1].trim());
+      }
+    }
+
+    if (!msgText || !msgTs) continue;
+
+    const isOwner = senderId === OWNER_ID;
+    const timestamp = ts ? new Date(ts + ' UTC').toISOString() : new Date(parseFloat(msgTs) * 1000).toISOString();
+
+    messages.push({
+      source_id: `slack-${channelId}-${msgTs}`,
+      channel: channel || channelId,
+      sender: sender || (isOwner ? OWNER_NAME : 'unknown'),
+      is_owner: isOwner,
+      text: msgText,
+      timestamp,
+      participants: participants.length > 0 ? participants : [sender || 'unknown'],
+    });
+  }
+
+  return messages;
+}
+
+// Read input
+const inputFile = process.argv[2] || '/dev/stdin';
+const text = fs.readFileSync(inputFile, 'utf8');
+const messages = parseSearchResults(text);
+
+let ingested = 0;
+for (const m of messages) {
+  const result = brain.insertMemory({
+    source: 'slack',
+    source_id: m.source_id,
+    source_channel: m.channel,
+    memory_type: m.is_owner ? 'message_sent' : 'message_received',
+    direction: m.is_owner ? 'outbound' : 'inbound',
+    participants: JSON.stringify(m.participants),
+    content: m.sender + ': ' + m.text,
+    timestamp: m.timestamp,
+    metadata: JSON.stringify({ source: 'slack_import' }),
+  });
+  if (result) ingested++;
+}
+
+console.log(`Parsed: ${messages.length} messages, Ingested: ${ingested} new`);
+console.log(`Total memories: ${brain.getBrainStats().memory_count}`);
+brain.closeDb();

package/template/wall-e/scripts/slack-backfill.js

@@ -0,0 +1,295 @@
+#!/usr/bin/env node
+'use strict';
+/**
+ * Standalone Slack backfill script — NO Claude API tokens used.
+ * Directly calls Slack MCP + writes to brain DB.
+ * Paginates through all results (up to 20 pages per query).
+ *
+ * Usage:
+ *   node scripts/slack-backfill.js              # full backfill (2022-now)
+ *   node scripts/slack-backfill.js incremental  # only new messages
+ *   node scripts/slack-backfill.js 2024-01      # single month
+ */
+
+const brain = require('../brain');
+const slackMcp = require('../tools/slack-mcp');
+
+const PAUSE_MS = 1500;
+const MAX_PAGES = 20;
+const OWNER_ID = process.env.SLACK_OWNER_USER_ID;
+if (!OWNER_ID) throw new Error('SLACK_OWNER_USER_ID env var required');
+const SLACK_OWNER_HANDLE = process.env.SLACK_OWNER_HANDLE || 'owner';
+
+async function main() {
+  const mode = process.argv[2] || 'full';
+
+  // Init
+  brain.initDb();
+  const db = brain.getDb();
+
+  if (!slackMcp.isAuthenticated()) {
+    console.error('Not authenticated with Slack. Run: node tools/slack-mcp.js auth');
+    process.exit(1);
+  }
+
+  const totalBefore = db.prepare("SELECT count(*) as c FROM memories WHERE source = 'slack'").get().c;
+  console.log(`Starting. Mode: ${mode}. Currently have ${totalBefore} Slack messages.\n`);
+
+  let stats = { new_messages: 0, skipped: 0 };
+
+  if (mode === 'incremental') {
+    const latest = db.prepare("SELECT max(timestamp) as ts FROM memories WHERE source = 'slack'").get();
+    const latestDate = (latest?.ts || '2020-01-01').slice(0, 10);
+    console.log(`Incremental from ${latestDate}\n`);
+    await pullMonth(db, stats, `from:${SLACK_OWNER_HANDLE} after:${latestDate}`, 'incremental');
+  } else if (mode.match(/^\d{4}-\d{2}$/)) {
+    // Single month
+    const [y, m] = mode.split('-').map(Number);
+    const after = `${y}-${String(m).padStart(2, '0')}-01`;
+    const before = m === 12 ? `${y + 1}-01-01` : `${y}-${String(m + 1).padStart(2, '0')}-01`;
+    console.log(`Single month: ${mode}\n`);
+    await pullMonth(db, stats, `from:${SLACK_OWNER_HANDLE} after:${after} before:${before}`, mode);
+  } else {
+    // Full backfill with checkpoint support
+    const months = generateMonths(2022, 6);
+    const checkpoint = process.env.WALL_E_CHECKPOINT || '';
+    let startIdx = 0;
+    if (checkpoint) {
+      const idx = months.indexOf(checkpoint);
+      if (idx >= 0) {
+        startIdx = idx;
+        console.log(`Resuming from checkpoint: ${checkpoint}\n`);
+      }
+    }
+    console.log(`Full backfill: ${months.length} months (starting from ${months[startIdx]})\n`);
+
+    for (let mi = startIdx; mi < months.length; mi++) {
+      const month = months[mi];
+      const existing = db.prepare(
+        "SELECT count(*) as c FROM memories WHERE source='slack' AND substr(timestamp,1,7)=?"
+      ).get(month).c;
+
+      const [y, m] = month.split('-').map(Number);
+      const after = `${y}-${String(m).padStart(2, '0')}-01`;
+      const before = m === 12 ? `${y + 1}-01-01` : `${y}-${String(m + 1).padStart(2, '0')}-01`;
+
+      process.stdout.write(`[${month}] ${existing} existing... `);
+      const beforeNew = stats.new_messages;
+      await pullMonth(db, stats, `from:${SLACK_OWNER_HANDLE} after:${after} before:${before}`, month);
+      const added = stats.new_messages - beforeNew;
+
+      const nowCount = db.prepare(
+        "SELECT count(*) as c FROM memories WHERE source='slack' AND substr(timestamp,1,7)=?"
+      ).get(month).c;
+      console.log(`+${added} new → ${nowCount} total`);
+
+      // Emit checkpoint so task runner can save progress
+      console.log(`CHECKPOINT:${month}`);
+    }
+  }
+
+  const totalAfter = db.prepare("SELECT count(*) as c FROM memories WHERE source = 'slack'").get().c;
+  console.log(`\nDone! New: ${stats.new_messages}, Skipped dupes: ${stats.skipped}`);
+  console.log(`Total: ${totalBefore} → ${totalAfter} (+${totalAfter - totalBefore})`);
+}
+
+/**
+ * Pull all pages for a single query, paginating with cursors.
+ */
+async function pullMonth(db, stats, query, label) {
+  let cursor = undefined;
+
+  for (let page = 1; page <= MAX_PAGES; page++) {
+    try {
+      const args = {
+        query,
+        sort: 'timestamp',
+        sort_dir: 'asc',
+        limit: 20,
+        include_context: false,
+        response_format: 'detailed',
+      };
+      if (cursor) args.cursor = cursor;
+
+      const result = await slackMcp.callSlackMcp('slack_search_public_and_private', args);
+      const texts = (result?.content || []).filter(c => c.type === 'text').map(c => c.text);
+      let combined = texts.join('\n');
+      if (!combined || combined.length < 10) break;
+
+      // The Slack MCP wraps the markdown in a JSON envelope: {"results":"...","pagination_info":"..."}
+      let paginationInfo = '';
+      try {
+        const parsed = JSON.parse(combined);
+        combined = parsed.results || combined;
+        paginationInfo = parsed.pagination_info || '';
+      } catch {}
+
+      // Parse the markdown response
+      const messages = parseSlackResults(combined);
+      if (messages.length === 0) break;
+
+      for (const msg of messages) {
+        const sourceId = `slack-${msg.message_ts}`;
+        const exists = db.prepare("SELECT 1 FROM memories WHERE source = 'slack' AND source_id = ?").get(sourceId);
+        if (exists) { stats.skipped++; continue; }
+
+        brain.insertMemory({
+          source: 'slack',
+          source_id: sourceId,
+          memory_type: msg.isOwner ? 'message_sent' : 'message_received',
+          direction: msg.isOwner ? 'outbound' : 'inbound',
+          content: msg.text,
+          source_channel: msg.channel,
+          participants: msg.participants || msg.sender,
+          timestamp: msg.timestamp,
+          metadata: JSON.stringify({ channel: msg.channel, sender: msg.sender, channel_id: msg.channelId }),
+        });
+        stats.new_messages++;
+      }
+
+      // Check for next page cursor (in pagination_info or in the text body)
+      const cursorMatch = (paginationInfo + '\n' + combined).match(/use cursor `([^`]+)`/);
+      if (!cursorMatch) break;
+      cursor = cursorMatch[1];
+
+      await sleep(PAUSE_MS);
+    } catch (err) {
+      if (err.message.includes('401') || err.message.includes('expired')) {
+        console.error('\nSlack token expired! Run: node tools/slack-mcp.js auth');
+        process.exit(1);
+      }
+      // Other errors: stop this month, continue
+      process.stderr.write(`[${label} p${page}] Error: ${err.message.slice(0, 100)}\n`);
+      break;
+    }
+  }
+}
+
+/**
+ * Parse the Slack MCP detailed response format.
+ * Each result looks like:
+ *   ### Result N of M
+ *   Channel: #channel-name (ID: CXXX) or DM (ID: DXXX)
+ *   Participants: Name1 (ID: UXXX), Name2 (ID: UXXX)
+ *   From: Name (ID: UXXX)
+ *   Time: 2024-09-04 17:12:28 BST
+ *   Message_ts: 1725466348.869379
+ *   Text:
+ *   The actual message content
+ */
+function parseSlackResults(text) {
+  const messages = [];
+  // Split by result headers
+  const blocks = text.split(/### Result \d+ of \d+/).slice(1);
+
+  for (const block of blocks) {
+    const lines = block.trim().split('\n');
+    let channel = 'unknown', channelId = '', sender = '', participants = '', timestamp = '', message_ts = '', content = '';
+    let inText = false;
+
+    for (const line of lines) {
+      if (inText) {
+        if (line.startsWith('---')) break;
+        content += (content ? '\n' : '') + line;
+        continue;
+      }
+
+      const channelMatch = line.match(/^Channel:\s*(?:#?(\S+)|DM|Group DM)\s*\(ID:\s*(\w+)\)/);
+      if (channelMatch) {
+        channel = channelMatch[1] || (line.includes('DM') ? 'DM' : 'unknown');
+        channelId = channelMatch[2] || '';
+        continue;
+      }
+
+      const participantsMatch = line.match(/^Participants:\s*(.+)/);
+      if (participantsMatch) {
+        participants = participantsMatch[1].replace(/\s*\(ID:\s*\w+\)/g, '').trim();
+        continue;
+      }
+
+      const fromMatch = line.match(/^From:\s*(.+?)\s*\(ID:\s*(\w+)\)/);
+      if (fromMatch) {
+        sender = fromMatch[1].trim() || fromMatch[2];
+        continue;
+      }
+
+      const timeMatch = line.match(/^Time:\s*(.+)/);
+      if (timeMatch) {
+        try {
+          const d = new Date(timeMatch[1].trim());
+          if (!isNaN(d.getTime())) timestamp = d.toISOString();
+        } catch {}
+        continue;
+      }
+
+      const tsMatch = line.match(/^Message_ts:\s*(\S+)/);
+      if (tsMatch) {
+        message_ts = tsMatch[1];
+        // Derive timestamp from message_ts if not already set
+        if (!timestamp) {
+          try {
+            const epoch = parseFloat(message_ts);
+            if (epoch > 0) timestamp = new Date(epoch * 1000).toISOString();
+          } catch {}
+        }
+        continue;
+      }
+
+      if (line.match(/^Text:\s*$/)) {
+        inText = true;
+        continue;
+      }
+      if (line.startsWith('Text:')) {
+        inText = true;
+        content = line.replace(/^Text:\s*/, '');
+        continue;
+      }
+    }
+
+    content = content.trim();
+    if (!content || !message_ts) continue;
+
+    const isOwner = sender.toLowerCase().includes(SLACK_OWNER_HANDLE.split('.')[0]) || sender === OWNER_ID;
+
+    // For DMs/Group DMs, use participant names as channel name instead of raw ID
+    if ((channel === 'DM' || channel === 'unknown' || channel === 'Group') && participants) {
+      const ownerFirst = (process.env.WALLE_OWNER_NAME || '').split(' ')[0].toLowerCase();
+      const others = participants.split(',')
+        .map(p => p.trim())
+        .filter(p => p && (!ownerFirst || !p.toLowerCase().startsWith(ownerFirst)))
+        .map(p => p.split(' ')[0]) // first names only
+        .slice(0, 3);
+      if (others.length > 0) {
+        channel = `DM with ${others.join(', ')}`;
+      }
+    }
+
+    messages.push({
+      channel,
+      channelId,
+      sender,
+      participants,
+      timestamp,
+      message_ts,
+      text: content,
+      isOwner,
+    });
+  }
+
+  return messages;
+}
+
+function generateMonths(startYear, startMonth) {
+  const months = [];
+  const now = new Date();
+  let y = startYear, m = startMonth;
+  while (y < now.getFullYear() || (y === now.getFullYear() && m <= now.getMonth() + 1)) {
+    months.push(`${y}-${String(m).padStart(2, '0')}`);
+    m++; if (m > 12) { m = 1; y++; }
+  }
+  return months;
+}
+
+function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
+
+main().catch(err => { console.error('Fatal:', err.message); process.exit(1); });