create-walle 0.1.0

package/template/wall-e/skills/_bundled/slack-backfill/SKILL.md

@@ -0,0 +1,60 @@
+---
+name: slack-backfill
+description: >
+  Full Slack history backfill from 2022 to present. Searches month by month,
+  paginates through all results, and stores messages in WALL-E's brain.
+  Use for initial setup or catching up on missed months.
+version: 1.0.0
+author: juncao
+execution: script
+entry: ../../../scripts/slack-backfill.js
+args: []
+trigger:
+  type: manual
+config:
+  mode:
+    type: string
+    enum: [full, incremental]
+    default: full
+    description: "full = month-by-month from 2022, incremental = only new messages"
+  month:
+    type: string
+    default: ""
+    description: "Single month to backfill (e.g. 2024-09). Empty = all months."
+tags: [slack, messaging, backfill, ingestion, history]
+permissions:
+  - slack:read
+  - brain:write
+---
+# Slack History Backfill
+
+## What This Skill Does
+
+Performs a comprehensive backfill of Slack message history from June 2022
+to the present month. Uses Slack search API with pagination (up to 20 pages
+per query) to capture all messages.
+
+## How It Works
+
+1. Generate month ranges from 2022-06 to current month
+2. For each month, search for messages from the owner (via SLACK_OWNER_HANDLE env var)
+3. Paginate through all results (up to 20 pages x 20 results)
+4. Parse detailed result format (channel, sender, timestamp, message_ts)
+5. Deduplicate using source_id = `slack-{message_ts}`
+6. Emit CHECKPOINT after each month for task runner resume support
+
+## Modes
+
+- `full` (default) -- Scans every month from 2022-06 to now. Skips already-ingested messages.
+- `incremental` -- Only searches for messages after the latest timestamp in the brain.
+- Single month -- Pass a month string like `2024-09` to backfill just that month.
+
+## Checkpoint Support
+
+The script reads `WALL_E_CHECKPOINT` env var to resume from a specific month.
+Each completed month emits `CHECKPOINT:YYYY-MM` for the task runner to save.
+
+## Output
+
+Prints progress per month and final summary:
+`Total: {before} -> {after} (+{delta})`

package/template/wall-e/skills/_bundled/slack-sync/SKILL.md

@@ -0,0 +1,55 @@
+---
+name: slack-sync
+description: >
+  Pull latest Slack messages from active conversations into WALL-E's brain.
+  Discovers recently active channels, fetches new messages incrementally.
+  Use for keeping Slack context fresh.
+version: 1.0.0
+author: juncao
+execution: script
+entry: ../../../scripts/slack-channel-history.js
+args: ["--sync"]
+trigger:
+  type: interval
+  schedule: "every 15m"
+config:
+  mode:
+    type: string
+    enum: [incremental, full]
+    default: incremental
+tags: [slack, messaging, sync, ingestion]
+permissions:
+  - slack:read
+  - brain:write
+---
+# Slack Conversation Sync
+
+## What This Skill Does
+
+Incrementally syncs Slack messages from your most active conversations
+into WALL-E's brain. Only checks recently active channels (last 7 days)
+to minimize API calls and run time.
+
+## How It Works
+
+1. Search Slack for YOUR recent messages to discover active channels
+2. Query brain DB for channels with recent activity (last 3 days)
+3. For each active channel, fetch messages since last sync using pagination
+4. Deduplicate and store new messages as memories with source_id
+5. Emit CHECKPOINT after each channel for resume support
+
+## Configuration
+
+- `mode: incremental` (default) -- Only pulls channels with activity in the last 7 days (~35 channels, ~1 min)
+- `mode: full` -- Pulls ALL known channels (~800 channels, ~15 min)
+
+## Output
+
+Prints progress per channel and emits CHECKPOINT lines for task runner resume support.
+Returns summary: `{ new_messages, skipped, channels_done, total_after }`
+
+## Error Handling
+
+- Slack token expiry triggers exit code 1 with clear error message
+- Channel not found errors are logged and skipped
+- Other errors are logged per-channel and processing continues

package/template/wall-e/skills/claude-code-reader.js

@@ -0,0 +1,144 @@
+'use strict';
+const fs = require('fs');
+const path = require('path');
+
+const CLAUDE_DIR = path.join(process.env.HOME, '.claude');
+const SKILLS_DIR = path.join(CLAUDE_DIR, 'skills');
+const PLUGINS_DIR = path.join(CLAUDE_DIR, 'plugins');
+const SETTINGS_PATH = path.join(CLAUDE_DIR, 'settings.json');
+const SETTINGS_LOCAL_PATH = path.join(CLAUDE_DIR, 'settings.local.json');
+
+/**
+ * Read Claude Code's MCP server configurations.
+ */
+function readMcpConfig() {
+  const configs = [];
+  for (const settingsPath of [SETTINGS_PATH, SETTINGS_LOCAL_PATH]) {
+    try {
+      if (!fs.existsSync(settingsPath)) continue;
+      const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
+      const mcpServers = settings.mcpServers || {};
+      for (const [name, config] of Object.entries(mcpServers)) {
+        configs.push({
+          name,
+          command: config.command,
+          args: config.args || [],
+          env: Object.keys(config.env || {}), // keys only, not values (security)
+          source: settingsPath,
+        });
+      }
+    } catch {}
+  }
+  return configs;
+}
+
+/**
+ * Read Claude Code's installed skills.
+ * Returns skill metadata (name, description, triggers) but not full content.
+ */
+function readClaudeSkills() {
+  const skills = [];
+
+  // Read from ~/.claude/skills/
+  if (fs.existsSync(SKILLS_DIR)) {
+    for (const file of fs.readdirSync(SKILLS_DIR)) {
+      if (!file.endsWith('.md')) continue;
+      try {
+        const content = fs.readFileSync(path.join(SKILLS_DIR, file), 'utf8');
+        const name = file.replace('.md', '');
+        const description = extractDescription(content);
+        skills.push({ name, description, source: 'user-skills', path: path.join(SKILLS_DIR, file) });
+      } catch {}
+    }
+  }
+
+  // Read from ~/.claude/plugins/ (installed plugins with skills)
+  if (fs.existsSync(PLUGINS_DIR)) {
+    try {
+      scanPluginSkills(PLUGINS_DIR, skills);
+    } catch {}
+  }
+
+  return skills;
+}
+
+function scanPluginSkills(dir, skills, depth) {
+  if ((depth || 0) > 4) return;
+  try {
+    for (const entry of fs.readdirSync(dir)) {
+      const fullPath = path.join(dir, entry);
+      const stat = fs.statSync(fullPath);
+      if (stat.isDirectory()) {
+        // Look for skill definition files
+        const skillFiles = ['skill.md', 'README.md', 'index.md'];
+        for (const sf of skillFiles) {
+          const sfPath = path.join(fullPath, sf);
+          if (fs.existsSync(sfPath)) {
+            try {
+              const content = fs.readFileSync(sfPath, 'utf8');
+              const description = extractDescription(content);
+              skills.push({ name: entry, description, source: 'plugin', path: sfPath });
+            } catch {}
+          }
+        }
+        scanPluginSkills(fullPath, skills, (depth || 0) + 1);
+      }
+    }
+  } catch {}
+}
+
+function extractDescription(content) {
+  // Extract first paragraph or heading as description
+  const lines = content.split('\n').filter(l => l.trim());
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (trimmed.startsWith('#')) {
+      return trimmed.replace(/^#+\s*/, '');
+    }
+    if (trimmed.length > 20 && !trimmed.startsWith('```') && !trimmed.startsWith('-')) {
+      return trimmed.slice(0, 200);
+    }
+  }
+  return '';
+}
+
+/**
+ * Generate WALL-E skill suggestions based on Claude Code's capabilities.
+ */
+function suggestSkillsFromClaudeCode() {
+  const mcpServers = readMcpConfig();
+  const claudeSkills = readClaudeSkills();
+  const suggestions = [];
+
+  // Suggest skills based on MCP servers
+  for (const mcp of mcpServers) {
+    if (mcp.name.includes('slack') || mcp.name.includes('Slack')) {
+      suggestions.push({
+        name: 'fetch-slack-messages',
+        description: `Fetch recent Slack messages using ${mcp.name} MCP server`,
+        source: 'mcp',
+        mcp_server: mcp.name,
+      });
+    }
+    if (mcp.name.includes('glean') || mcp.name.includes('Glean')) {
+      suggestions.push({
+        name: 'search-company-docs',
+        description: `Search company documents and knowledge via ${mcp.name}`,
+        source: 'mcp',
+        mcp_server: mcp.name,
+      });
+    }
+    if (mcp.name.includes('github') || mcp.name.includes('GitHub')) {
+      suggestions.push({
+        name: 'fetch-github-activity',
+        description: `Fetch GitHub PRs, commits, and reviews via ${mcp.name}`,
+        source: 'mcp',
+        mcp_server: mcp.name,
+      });
+    }
+  }
+
+  return { mcpServers, claudeSkills, suggestions };
+}
+
+module.exports = { readMcpConfig, readClaudeSkills, suggestSkillsFromClaudeCode };

package/template/wall-e/skills/mcp-client.js

@@ -0,0 +1,407 @@
+'use strict';
+const { spawn } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+const { EventEmitter } = require('events');
+
+const CLAUDE_DIR = path.join(process.env.HOME, '.claude');
+
+// Cache of active MCP connections
+const connections = new Map();
+
+/**
+ * Read all MCP server configs from Claude Code.
+ */
+function loadMcpConfigs() {
+  const configs = {};
+
+  // From settings.json (HTTP servers)
+  try {
+    const settings = JSON.parse(fs.readFileSync(path.join(CLAUDE_DIR, 'settings.json'), 'utf8'));
+    for (const [name, cfg] of Object.entries(settings.mcpServers || {})) {
+      configs[name] = { ...cfg, name };
+    }
+  } catch {}
+
+  // From mcp.json (stdio servers)
+  try {
+    const mcp = JSON.parse(fs.readFileSync(path.join(CLAUDE_DIR, 'mcp.json'), 'utf8'));
+    for (const [name, cfg] of Object.entries(mcp.mcpServers || mcp)) {
+      if (name === 'mcpServers') continue;
+      configs[name] = { ...cfg, name };
+    }
+  } catch {}
+
+  // From .credentials.json — OAuth-authenticated MCP servers (Glean, Slack, Mezmo, etc.)
+  // Re-read every time since Claude Code may refresh tokens
+  try {
+    const creds = JSON.parse(fs.readFileSync(path.join(CLAUDE_DIR, '.credentials.json'), 'utf8'));
+    const mcpOAuth = creds.mcpOAuth || {};
+    for (const [key, entry] of Object.entries(mcpOAuth)) {
+      const serverName = entry.serverName || key.split('|')[0];
+      // Skip if already configured from settings.json (but allow credential override)
+      if (configs[serverName] && !configs[serverName]._fromCredentials) {
+        // Update token on existing config if we have a fresher one
+        if (entry.accessToken && entry.serverUrl) {
+          configs[serverName].oauth = {
+            accessToken: entry.accessToken,
+            refreshToken: entry.refreshToken,
+            expiresAt: entry.expiresAt,
+          };
+        }
+        continue;
+      }
+      // Only add servers that have a URL and a valid (non-expired) access token
+      const isExpired = entry.expiresAt && entry.expiresAt < Date.now();
+      if (entry.serverUrl && entry.accessToken && !isExpired) {
+        configs[serverName] = {
+          name: serverName,
+          type: 'http',
+          url: entry.serverUrl,
+          oauth: {
+            accessToken: entry.accessToken,
+            refreshToken: entry.refreshToken,
+            expiresAt: entry.expiresAt,
+          },
+          _fromCredentials: true,
+        };
+      }
+    }
+  } catch {}
+
+  return configs;
+}
+
+/**
+ * MCP stdio transport — spawns a process and communicates via JSON-RPC over stdin/stdout
+ */
+class StdioTransport extends EventEmitter {
+  constructor(config) {
+    super();
+    this.config = config;
+    this.process = null;
+    this.buffer = '';
+    this.nextId = 1;
+    this.pending = new Map();
+    this.initialized = false;
+  }
+
+  async connect() {
+    const env = { ...process.env, ...(this.config.env || {}) };
+    this.process = spawn(this.config.command, this.config.args || [], {
+      env,
+      stdio: ['pipe', 'pipe', 'pipe'],
+      cwd: process.env.HOME,
+    });
+
+    this.process.stdout.on('data', (data) => {
+      this.buffer += data.toString();
+      this._processBuffer();
+    });
+
+    this.process.stderr.on('data', (data) => {
+      // MCP servers may log to stderr — ignore
+    });
+
+    this.process.on('error', (err) => {
+      console.error(`[mcp-client] Process error for ${this.config.name}:`, err.message);
+    });
+
+    this.process.on('exit', (code) => {
+      this.initialized = false;
+      // Reject all pending requests
+      for (const [id, { reject }] of this.pending) {
+        reject(new Error(`MCP server ${this.config.name} exited with code ${code}`));
+      }
+      this.pending.clear();
+    });
+
+    // Initialize
+    await this._send('initialize', {
+      protocolVersion: '2024-11-05',
+      capabilities: {},
+      clientInfo: { name: 'wall-e', version: '0.1.0' },
+    });
+
+    // Send initialized notification (no id, no response expected)
+    this._write({ jsonrpc: '2.0', method: 'notifications/initialized' });
+    this.initialized = true;
+  }
+
+  async _send(method, params) {
+    return new Promise((resolve, reject) => {
+      const id = this.nextId++;
+      const timeout = setTimeout(() => {
+        this.pending.delete(id);
+        reject(new Error(`MCP request ${method} timed out after 30s`));
+      }, 30000);
+
+      this.pending.set(id, {
+        resolve: (result) => { clearTimeout(timeout); resolve(result); },
+        reject: (err) => { clearTimeout(timeout); reject(err); },
+      });
+
+      this._write({ jsonrpc: '2.0', id, method, params });
+    });
+  }
+
+  _write(msg) {
+    if (!this.process || !this.process.stdin.writable) {
+      throw new Error(`MCP server ${this.config.name} not connected`);
+    }
+    this.process.stdin.write(JSON.stringify(msg) + '\n');
+  }
+
+  _processBuffer() {
+    const lines = this.buffer.split('\n');
+    this.buffer = lines.pop() || '';
+    for (const line of lines) {
+      if (!line.trim()) continue;
+      try {
+        const msg = JSON.parse(line);
+        if (msg.id && this.pending.has(msg.id)) {
+          const { resolve, reject } = this.pending.get(msg.id);
+          this.pending.delete(msg.id);
+          if (msg.error) {
+            reject(new Error(msg.error.message || JSON.stringify(msg.error)));
+          } else {
+            resolve(msg.result);
+          }
+        }
+      } catch {}
+    }
+  }
+
+  async listTools() {
+    const result = await this._send('tools/list', {});
+    return result.tools || [];
+  }
+
+  async callTool(name, args) {
+    const result = await this._send('tools/call', { name, arguments: args });
+    return result;
+  }
+
+  disconnect() {
+    if (this.process) {
+      this.process.kill();
+      this.process = null;
+    }
+    this.initialized = false;
+  }
+}
+
+/**
+ * MCP HTTP transport — sends JSON-RPC over HTTP POST
+ */
+class HttpTransport {
+  constructor(config) {
+    this.config = config;
+    this.url = config.url;
+    this.nextId = 1;
+    this.initialized = false;
+    this.sessionUrl = null;
+    // OAuth tokens would be loaded from stored auth
+    this.authHeaders = {};
+  }
+
+  async connect() {
+    // Try to load OAuth token if configured
+    if (this.config.oauth) {
+      this._loadOAuthToken();
+    }
+
+    // Initialize
+    const result = await this._send('initialize', {
+      protocolVersion: '2024-11-05',
+      capabilities: {},
+      clientInfo: { name: 'wall-e', version: '0.1.0' },
+    });
+
+    // Send initialized notification
+    await this._sendNotification('notifications/initialized');
+    this.initialized = true;
+    return result;
+  }
+
+  _loadOAuthToken() {
+    // First check if the config already has an OAuth token (from .credentials.json)
+    if (this.config.oauth && this.config.oauth.accessToken) {
+      this.authHeaders['Authorization'] = `Bearer ${this.config.oauth.accessToken}`;
+      return;
+    }
+    // Fallback: try file-based token storage
+    const tokenPaths = [
+      path.join(CLAUDE_DIR, '.mcp-auth', this.config.name, 'tokens.json'),
+      path.join(CLAUDE_DIR, 'mcp-tokens', this.config.name + '.json'),
+    ];
+    for (const p of tokenPaths) {
+      try {
+        if (fs.existsSync(p)) {
+          const tokens = JSON.parse(fs.readFileSync(p, 'utf8'));
+          if (tokens.access_token) {
+            this.authHeaders['Authorization'] = `Bearer ${tokens.access_token}`;
+            return;
+          }
+        }
+      } catch {}
+    }
+  }
+
+  async _send(method, params) {
+    const id = this.nextId++;
+    const body = { jsonrpc: '2.0', id, method };
+    if (params) body.params = params;
+
+    const res = await fetch(this.sessionUrl || this.url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Accept': 'application/json, text/event-stream',
+        ...this.authHeaders,
+      },
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(30000),
+    });
+
+    // Check for session URL in Mcp-Session header
+    const sessionHeader = res.headers.get('mcp-session');
+    if (sessionHeader) this.sessionUrl = this.url;
+
+    if (!res.ok) {
+      const text = await res.text();
+      throw new Error(`MCP HTTP error ${res.status}: ${text.slice(0, 200)}`);
+    }
+
+    const contentType = res.headers.get('content-type') || '';
+
+    // Handle SSE responses
+    if (contentType.includes('text/event-stream')) {
+      const text = await res.text();
+      // Parse SSE for the result
+      for (const line of text.split('\n')) {
+        if (line.startsWith('data: ')) {
+          try {
+            const data = JSON.parse(line.slice(6));
+            if (data.id === id) {
+              if (data.error) throw new Error(data.error.message || JSON.stringify(data.error));
+              return data.result;
+            }
+          } catch (e) {
+            if (e.message.includes('MCP')) throw e;
+          }
+        }
+      }
+      return null;
+    }
+
+    // Handle JSON response
+    const data = await res.json();
+    if (data.error) throw new Error(data.error.message || JSON.stringify(data.error));
+    return data.result;
+  }
+
+  async _sendNotification(method) {
+    try {
+      await fetch(this.sessionUrl || this.url, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', ...this.authHeaders },
+        body: JSON.stringify({ jsonrpc: '2.0', method }),
+        signal: AbortSignal.timeout(5000),
+      });
+    } catch {}
+  }
+
+  async listTools() {
+    const result = await this._send('tools/list', {});
+    return result?.tools || [];
+  }
+
+  async callTool(name, args) {
+    return this._send('tools/call', { name, arguments: args });
+  }
+
+  disconnect() {
+    this.initialized = false;
+    this.sessionUrl = null;
+  }
+}
+
+/**
+ * Get or create a connection to an MCP server.
+ */
+async function getConnection(serverName) {
+  if (connections.has(serverName)) {
+    const conn = connections.get(serverName);
+    if (conn.initialized) return conn;
+    // Reconnect if disconnected
+    connections.delete(serverName);
+  }
+
+  const configs = loadMcpConfigs();
+  const config = configs[serverName];
+  if (!config) throw new Error(`MCP server "${serverName}" not found in config`);
+
+  let transport;
+  if (config.type === 'http' && config.url) {
+    transport = new HttpTransport(config);
+  } else if (config.command) {
+    transport = new StdioTransport(config);
+  } else {
+    throw new Error(`Unsupported MCP server config for "${serverName}"`);
+  }
+
+  await transport.connect();
+  connections.set(serverName, transport);
+  return transport;
+}
+
+/**
+ * List all available MCP tools across all configured servers.
+ */
+async function listAllTools() {
+  const configs = loadMcpConfigs();
+  const allTools = [];
+
+  for (const [name, config] of Object.entries(configs)) {
+    try {
+      const conn = await getConnection(name);
+      const tools = await conn.listTools();
+      for (const tool of tools) {
+        allTools.push({ ...tool, server: name });
+      }
+    } catch (err) {
+      console.error(`[mcp-client] Failed to list tools from ${name}:`, err.message);
+    }
+  }
+
+  return allTools;
+}
+
+/**
+ * Call an MCP tool on a specific server.
+ */
+async function callMcpTool(serverName, toolName, args) {
+  const conn = await getConnection(serverName);
+  return conn.callTool(toolName, args);
+}
+
+/**
+ * Disconnect all MCP servers.
+ */
+function disconnectAll() {
+  for (const [name, conn] of connections) {
+    try { conn.disconnect(); } catch {}
+  }
+  connections.clear();
+}
+
+module.exports = {
+  loadMcpConfigs,
+  getConnection,
+  listAllTools,
+  callMcpTool,
+  disconnectAll,
+  StdioTransport,
+  HttpTransport,
+};