create-walle 0.9.11 → 0.9.13

package/template/wall-e/mcp-server.js

@@ -1,6 +1,13 @@
 'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
 const { v4: uuidv4 } = require('uuid');
 const brain = require('./brain');
+const { createSessionIngestService } = require('./memory/session-ingest-service');
+const { collectIngestRecords } = require('./sources/base');
+const { ensureBuiltinSourceAdapters } = require('./sources/builtin');
+const sourceRegistry = require('./sources/registry');
+const { RECORD_TYPES } = require('./sources/record-types');
 let _embeddings;
 try { _embeddings = require('./embeddings'); } catch { _embeddings = null; }
 
@@ -153,6 +160,129 @@ const MCP_TOOLS = [
       properties: {},
     },
   },
+  {
+    name: 'walle_memory_status',
+    description: 'Get coding-session memory status: brain stats, source adapters, diary count, and operational tool availability.',
+    inputSchema: {
+      type: 'object',
+      properties: {},
+    },
+  },
+  {
+    name: 'walle_list_sources',
+    description: 'List source adapters Wall-E can ingest, including Claude Code, Codex, Gemini, and Wall-E JSONL sessions.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        privacy_class: { type: 'string', description: 'Optional adapter default privacy class filter' },
+        incremental: { type: 'boolean', description: 'Optional incremental-support filter' },
+      },
+    },
+  },
+  {
+    name: 'walle_source_ingest',
+    description: 'Ingest a source file through a Wall-E source adapter. Use dry_run first when inspecting an unfamiliar source.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        adapter_id: { type: 'string', description: 'Source adapter id, e.g. codex-jsonl or claude-code-jsonl' },
+        uri: { type: 'string', description: 'Path to the JSONL source file' },
+        source_id: { type: 'string', description: 'Optional stable source/session id' },
+        cwd: { type: 'string', description: 'Optional project working directory' },
+        privacy_class: { type: 'string', description: 'Optional source privacy class' },
+        privacy_floor: { type: 'string', description: 'Optional maximum privacy class allowed to write' },
+        dry_run: { type: 'boolean', description: 'Parse and validate without writing memories' },
+        metadata: { type: 'object', description: 'Optional adapter metadata' },
+      },
+      required: ['adapter_id', 'uri'],
+    },
+  },
+  {
+    name: 'walle_search_sessions',
+    description: 'Search ingested coding sessions and Wall-E diary entries. Returns attributed snippets plus session pointers.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        query: { type: 'string', description: 'Search query' },
+        limit: { type: 'number', description: 'Max session snippets (default 10, max 50)' },
+      },
+      required: ['query'],
+    },
+  },
+  {
+    name: 'walle_get_session',
+    description: 'Retrieve session snippets from ingested memory, or parse a JSONL session directly when adapter_id and uri are provided.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        source_id: { type: 'string', description: 'Stable source/session id to retrieve from memory' },
+        adapter_id: { type: 'string', description: 'Optional adapter id for direct JSONL parse' },
+        uri: { type: 'string', description: 'Optional JSONL path for direct parse' },
+        limit: { type: 'number', description: 'Max records (default 50, max 200)' },
+        include_raw: { type: 'boolean', description: 'Include raw content when available' },
+      },
+    },
+  },
+  {
+    name: 'walle_diary_write',
+    description: 'Write an idempotent agent diary entry for a coding session stop, handoff, or compaction boundary.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        agent: { type: 'string', description: 'Agent name or harness' },
+        session_id: { type: 'string', description: 'Stable session id' },
+        event: { type: 'string', description: 'Diary event, e.g. stop, precompact, handoff' },
+        cwd: { type: 'string', description: 'Project working directory' },
+        branch: { type: 'string', description: 'Git branch' },
+        changed_files: { type: 'array', items: { type: 'string' } },
+        commands: { type: 'array', items: { type: 'string' } },
+        decisions: { type: 'array', items: { type: 'string' } },
+        blockers: { type: 'array', items: { type: 'string' } },
+        next_steps: { type: 'array', items: { type: 'string' } },
+        summary: { type: 'string', description: 'Short human-readable summary' },
+      },
+      required: ['session_id'],
+    },
+  },
+  {
+    name: 'walle_diary_read',
+    description: 'Read recent agent diary entries, optionally filtered by agent or session id.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        agent: { type: 'string', description: 'Optional agent filter' },
+        session_id: { type: 'string', description: 'Optional session filter' },
+        limit: { type: 'number', description: 'Max entries (default 20, max 100)' },
+      },
+    },
+  },
+  {
+    name: 'walle_reconnect',
+    description: 'Refresh Wall-E MCP process state after external DB/source updates. Runs a passive checkpoint and reports current health.',
+    inputSchema: {
+      type: 'object',
+      properties: {},
+    },
+  },
+  {
+    name: 'walle_repair_status',
+    description: 'Run a dry integrity scan for brain memory, knowledge, entity, and index consistency.',
+    inputSchema: {
+      type: 'object',
+      properties: {},
+    },
+  },
+  {
+    name: 'walle_rebuild_source_index',
+    description: 'Rebuild the memory/source index from stored memories. Use dry_run to inspect expected work without writing.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        source_id: { type: 'string', description: 'Optional stable source/session id to rebuild' },
+        dry_run: { type: 'boolean', description: 'Return current index status without rebuilding' },
+      },
+    },
+  },
 ];
 
 async function executeTool(name, args) {
@@ -276,11 +406,322 @@ async function executeTool(name, args) {
       try { embeddingCount = brain.getDb().prepare("SELECT COUNT(*) as c FROM embedding_map").get().c; } catch {}
       return { ...stats, entity_count: entityCount, embedding_count: embeddingCount };
     }
+    case 'walle_memory_status': {
+      ensureBuiltinSourceAdapters();
+      const stats = brain.getBrainStats();
+      return {
+        ok: true,
+        stats,
+        source_adapters: sourceRegistry.list().map(formatSourceAdapter),
+        diary_count: countDiaryEntries(),
+        tools: MCP_TOOLS.filter((tool) => tool.name.startsWith('walle_')).map((tool) => tool.name),
+      };
+    }
+    case 'walle_list_sources': {
+      ensureBuiltinSourceAdapters();
+      return {
+        sources: sourceRegistry.list({
+          privacyClass: args.privacy_class,
+          incremental: args.incremental,
+        }).map(formatSourceAdapter),
+      };
+    }
+    case 'walle_source_ingest': {
+      ensureBuiltinSourceAdapters();
+      assertJsonlPath(args.uri);
+      const sourceRef = {
+        adapterId: args.adapter_id,
+        uri: path.resolve(args.uri),
+        sourceFile: path.resolve(args.uri),
+        sourceId: args.source_id,
+        cwd: args.cwd || '',
+        privacyClass: args.privacy_class,
+        metadata: args.metadata || {},
+      };
+      const service = createSessionIngestService({ brain });
+      return service.ingestSource(sourceRef, {
+        privacyFloor: args.privacy_floor || null,
+        dryRun: Boolean(args.dry_run),
+      });
+    }
+    case 'walle_search_sessions': {
+      const limit = clampLimit(args.limit, 10, 50);
+      const results = brain.searchMemories({ query: args.query, limit: limit * 4 })
+        .filter(isSessionMemory)
+        .slice(0, limit)
+        .map(formatSessionMemory);
+      return { results };
+    }
+    case 'walle_get_session': {
+      const limit = clampLimit(args.limit, 50, 200);
+      if (args.adapter_id && args.uri) {
+        ensureBuiltinSourceAdapters();
+        assertJsonlPath(args.uri);
+        const adapter = sourceRegistry.instantiate(args.adapter_id);
+        if (!adapter) throw new Error(`Unknown source adapter: ${args.adapter_id}`);
+        const sourceRef = {
+          adapterId: args.adapter_id,
+          uri: path.resolve(args.uri),
+          sourceFile: path.resolve(args.uri),
+          sourceId: args.source_id,
+        };
+        const { records, diagnostics, source } = await collectIngestRecords(adapter, sourceRef);
+        return {
+          source: { adapterId: source.adapterId, sourceId: source.sourceId, sourceFile: source.sourceFile },
+          diagnostics,
+          records: records
+            .filter((record) => record.type === RECORD_TYPES.MEMORY_RECORD)
+            .slice(0, limit)
+            .map((record) => formatSourceRecord(record, { includeRaw: Boolean(args.include_raw) })),
+        };
+      }
+      if (!args.source_id) throw new Error('source_id is required when adapter_id and uri are not provided');
+      return {
+        source_id: args.source_id,
+        records: getSessionMemories(args.source_id, { limit }).map(formatSessionMemory),
+      };
+    }
+    case 'walle_diary_write': {
+      return writeDiaryEntry(args);
+    }
+    case 'walle_diary_read': {
+      return readDiaryEntries(args);
+    }
+    case 'walle_reconnect': {
+      const status = reconnectBrain();
+      return { ok: true, ...status };
+    }
+    case 'walle_repair_status': {
+      const repair = require('./utils/repair');
+      return repair.scanIntegrity();
+    }
+    case 'walle_rebuild_source_index': {
+      const { rebuildSourceIndex } = require('./memory/source-indexer');
+      return rebuildSourceIndex({
+        brain,
+        sourceId: args.source_id || '',
+        dryRun: Boolean(args.dry_run),
+      });
+    }
     default:
       return null;
   }
 }
 
+function clampLimit(value, fallback, max) {
+  const n = Number(value || fallback);
+  if (!Number.isFinite(n)) return fallback;
+  return Math.min(Math.max(Math.trunc(n), 1), max);
+}
+
+function assertJsonlPath(filePath) {
+  if (!filePath || typeof filePath !== 'string') throw new Error('uri must be a JSONL file path');
+  if (filePath.split(/[\\/]+/).includes('..')) throw new Error('uri cannot contain traversal segments');
+  if (path.extname(filePath) !== '.jsonl') throw new Error('uri must end in .jsonl');
+  const resolved = path.resolve(filePath);
+  const stat = fs.statSync(resolved);
+  if (!stat.isFile()) throw new Error('uri must point to a file');
+  return resolved;
+}
+
+function formatSourceAdapter(entry) {
+  const schema = entry.schema || {};
+  return {
+    id: entry.id || schema.adapterId,
+    adapterId: schema.adapterId,
+    version: schema.version,
+    supportsIncremental: Boolean(schema.supportsIncremental),
+    defaultPrivacyClass: schema.defaultPrivacyClass,
+    declaredTransformations: schema.declaredTransformations || [],
+  };
+}
+
+function isSessionMemory(memory) {
+  const source = memory?.source || '';
+  const type = memory?.memory_type || '';
+  return source === 'walle-diary'
+    || source.endsWith('-jsonl')
+    || type.startsWith('coding_session_')
+    || type === 'agent_diary';
+}
+
+function parseMetadata(memory) {
+  try {
+    const parsed = JSON.parse(memory?.metadata || '{}');
+    return parsed && typeof parsed === 'object' ? parsed : {};
+  } catch {
+    return {};
+  }
+}
+
+function formatSessionMemory(memory) {
+  const metadata = parseMetadata(memory);
+  return {
+    id: memory.id,
+    source: memory.source,
+    source_id: memory.source_id,
+    session_id: metadata.sessionId || metadata.sourceId || inferSourceId(memory.source_id),
+    cwd: metadata.cwd || memory.source_channel || '',
+    branch: metadata.gitBranch || metadata.branch || '',
+    memory_type: memory.memory_type,
+    timestamp: memory.timestamp,
+    snippet: String(memory.content || '').slice(0, 800),
+    metadata: {
+      itemId: metadata.itemId,
+      role: metadata.role,
+      toolCalls: metadata.toolCalls,
+      filesEdited: metadata.filesEdited || metadata.changed_files,
+    },
+  };
+}
+
+function formatSourceRecord(record, { includeRaw = false } = {}) {
+  return {
+    source_id: record.sourceId,
+    item_id: record.itemId,
+    role: record.role,
+    memory_type: record.memoryType,
+    timestamp: record.timestamp,
+    snippet: String(record.content || '').slice(0, 800),
+    raw: includeRaw ? record.contentRaw : undefined,
+    metadata: record.metadata || {},
+  };
+}
+
+function inferSourceId(sourceId = '') {
+  const parts = String(sourceId).split(':');
+  if (parts.length <= 2) return sourceId || '';
+  return parts.slice(0, 2).join(':');
+}
+
+function getSessionMemories(sourceId, { limit }) {
+  const db = brain.getDb();
+  return db.prepare(`
+    SELECT * FROM memories
+    WHERE archived_at IS NULL
+      AND (source_id = ? OR source_id LIKE ?)
+    ORDER BY timestamp ASC
+    LIMIT ?
+  `).all(sourceId, `${sourceId}:%`, limit);
+}
+
+function countDiaryEntries() {
+  try {
+    return brain.getDb().prepare(`
+      SELECT COUNT(*) as c FROM memories
+      WHERE archived_at IS NULL AND source = 'walle-diary' AND memory_type = 'agent_diary'
+    `).get().c;
+  } catch {
+    return 0;
+  }
+}
+
+function writeDiaryEntry(args = {}) {
+  if (!args.session_id) throw new Error('session_id is required');
+  const agent = sanitizeToken(args.agent || 'walle');
+  const event = sanitizeToken(args.event || 'manual');
+  const sourceId = `diary:${agent}:${args.session_id}:${event}`;
+  const timestamp = new Date().toISOString();
+  const metadata = {
+    agent,
+    sessionId: args.session_id,
+    event,
+    cwd: args.cwd || '',
+    branch: args.branch || '',
+    changed_files: arrayOfStrings(args.changed_files),
+    commands: arrayOfStrings(args.commands),
+    decisions: arrayOfStrings(args.decisions),
+    blockers: arrayOfStrings(args.blockers),
+    next_steps: arrayOfStrings(args.next_steps),
+  };
+  const content = formatDiaryContent({ ...metadata, summary: args.summary || '' });
+  const inserted = brain.insertMemory({
+    source: 'walle-diary',
+    source_id: sourceId,
+    source_channel: metadata.cwd,
+    memory_type: 'agent_diary',
+    direction: 'outbound',
+    participants: agent,
+    subject: args.session_id,
+    content,
+    content_raw: content,
+    metadata: JSON.stringify(metadata),
+    importance: 0.75,
+    timestamp,
+  });
+  return {
+    stored: Boolean(inserted),
+    duplicate: !inserted,
+    id: inserted?.id || null,
+    source_id: sourceId,
+  };
+}
+
+function readDiaryEntries(args = {}) {
+  const limit = clampLimit(args.limit, 20, 100);
+  const entries = brain.listMemories({ source: 'walle-diary', limit: limit * 3 })
+    .filter((memory) => memory.memory_type === 'agent_diary')
+    .map((memory) => ({ memory, metadata: parseMetadata(memory) }))
+    .filter(({ metadata }) => !args.agent || metadata.agent === args.agent)
+    .filter(({ metadata }) => !args.session_id || metadata.sessionId === args.session_id)
+    .slice(0, limit)
+    .map(({ memory, metadata }) => ({
+      id: memory.id,
+      source_id: memory.source_id,
+      session_id: metadata.sessionId || '',
+      agent: metadata.agent || '',
+      event: metadata.event || '',
+      cwd: metadata.cwd || '',
+      branch: metadata.branch || '',
+      timestamp: memory.timestamp,
+      content: memory.content,
+      changed_files: metadata.changed_files || [],
+      commands: metadata.commands || [],
+      decisions: metadata.decisions || [],
+      blockers: metadata.blockers || [],
+      next_steps: metadata.next_steps || [],
+    }));
+  return { entries };
+}
+
+function formatDiaryContent(entry) {
+  const lines = [];
+  lines.push(`Agent diary for ${entry.agent} session ${entry.sessionId} (${entry.event})`);
+  if (entry.summary) lines.push(`Summary: ${entry.summary}`);
+  if (entry.cwd) lines.push(`CWD: ${entry.cwd}`);
+  if (entry.branch) lines.push(`Branch: ${entry.branch}`);
+  for (const [label, values] of [
+    ['Changed files', entry.changed_files],
+    ['Commands', entry.commands],
+    ['Decisions', entry.decisions],
+    ['Blockers', entry.blockers],
+    ['Next steps', entry.next_steps],
+  ]) {
+    if (values.length) lines.push(`${label}: ${values.join('; ')}`);
+  }
+  return lines.join('\n');
+}
+
+function arrayOfStrings(value) {
+  return Array.isArray(value)
+    ? value.filter((item) => typeof item === 'string' && item.trim()).map((item) => item.trim())
+    : [];
+}
+
+function sanitizeToken(value) {
+  return String(value || '').trim().replace(/[^A-Za-z0-9_.:-]+/g, '-').slice(0, 80) || 'walle';
+}
+
+function reconnectBrain() {
+  const db = brain.getDb();
+  let checkpoint = null;
+  try { checkpoint = db.pragma('wal_checkpoint(PASSIVE)'); } catch {}
+  return {
+    stats: brain.getBrainStats(),
+    checkpoint,
+  };
+}
+
 function readBody(req) {
   return new Promise((resolve, reject) => {
     const chunks = [];
@@ -378,4 +819,4 @@ async function handleMcp(req, res) {
   }
 }
 
-module.exports = { handleMcp, MCP_TOOLS };
+module.exports = { handleMcp, MCP_TOOLS, executeTool };

package/template/wall-e/memory/session-ingest-service.js

@@ -0,0 +1,159 @@
+'use strict';
+
+const { SourceIngestContext, collectIngestRecords } = require('../sources/base');
+const { PRIVACY_CLASS_ORDER, RECORD_TYPES, privacyClassAllows } = require('../sources/record-types');
+const sourceRegistry = require('../sources/registry');
+const { ensureBuiltinSourceAdapters } = require('../sources/builtin');
+const { indexMemory } = require('./source-indexer');
+
+function createSessionIngestService({ brain = null, registry = sourceRegistry } = {}) {
+  return new SessionIngestService({ brain, registry });
+}
+
+class SessionIngestService {
+  constructor({ brain = null, registry = sourceRegistry } = {}) {
+    this.brain = brain || require('../brain');
+    this.registry = registry;
+    ensureBuiltinSourceAdapters();
+  }
+
+  async ingestSource(sourceRef, { privacyFloor = null, dryRun = false } = {}) {
+    const adapter = this.registry.instantiate(sourceRef.adapterId);
+    if (!adapter) throw new Error(`Unknown source adapter: ${sourceRef.adapterId}`);
+
+    const context = new SourceIngestContext();
+    const { records, diagnostics, source } = await collectIngestRecords(adapter, sourceRef, { context });
+    const result = {
+      adapterId: adapter.schema.adapterId,
+      sourceId: source.sourceId,
+      records: records.length,
+      memoryRecords: 0,
+      inserted: 0,
+      indexed: 0,
+      skipped: 0,
+      rejected: 0,
+      diagnostics,
+      dryRun,
+    };
+
+    for (const record of records) {
+      if (record.type !== RECORD_TYPES.MEMORY_RECORD) continue;
+      result.memoryRecords += 1;
+      if (!privacyClassAllows({ recordClass: record.privacyClass, floor: privacyFloor })) {
+        result.rejected += 1;
+        result.diagnostics.push({
+          level: 'warn',
+          message: `Rejected ${record.itemId} above privacy floor ${privacyFloor}`,
+          sourceId: record.sourceId,
+          itemId: record.itemId,
+        });
+        continue;
+      }
+      if (dryRun) continue;
+      const memory = toBrainMemory(record, adapter.schema.adapterId);
+      const downgrade = this._privacyDowngrade(memory, record);
+      if (downgrade) {
+        result.rejected += 1;
+        result.diagnostics.push({
+          level: 'warn',
+          message: downgrade,
+          sourceId: record.sourceId,
+          itemId: record.itemId,
+        });
+        continue;
+      }
+      const inserted = this.brain.insertMemory(memory);
+      if (inserted) {
+        result.inserted += 1;
+        try {
+          indexMemory({ ...memory, id: inserted.id }, { brain: this.brain });
+          result.indexed += 1;
+        } catch (err) {
+          result.diagnostics.push({
+            level: 'warn',
+            message: `Indexing failed for ${record.itemId}: ${err.message}`,
+            sourceId: record.sourceId,
+            itemId: record.itemId,
+          });
+        }
+      } else {
+        result.skipped += 1;
+      }
+    }
+
+    this._auditSourceIngest(result);
+    return result;
+  }
+
+  _privacyDowngrade(memory, record) {
+    if (!this.brain || typeof this.brain.getDb !== 'function') return null;
+    try {
+      const existing = this.brain.getDb().prepare(
+        'SELECT metadata FROM memories WHERE source = ? AND source_id = ?'
+      ).get(memory.source, memory.source_id);
+      if (!existing) return null;
+      const metadata = JSON.parse(existing.metadata || '{}');
+      const previous = metadata.privacyClass || '';
+      const next = record.privacyClass || '';
+      if (!previous || !next) return null;
+      const previousRank = PRIVACY_CLASS_ORDER.indexOf(previous);
+      const nextRank = PRIVACY_CLASS_ORDER.indexOf(next);
+      if (previousRank >= 0 && nextRank >= 0 && nextRank < previousRank) {
+        return `Rejected privacy downgrade for ${memory.source_id}: ${previous} -> ${next}`;
+      }
+    } catch {}
+    return null;
+  }
+
+  _auditSourceIngest(result) {
+    if (!this.brain || typeof this.brain.logWrite !== 'function' || result.dryRun) return;
+    try {
+      this.brain.logWrite('source_ingest', 'memories', {
+        id: result.sourceId,
+        source: result.adapterId,
+        subject: `inserted:${result.inserted};indexed:${result.indexed};rejected:${result.rejected}`,
+      });
+    } catch {}
+  }
+}
+
+function toBrainMemory(record, adapterId) {
+  const metadata = {
+    ...(record.metadata || {}),
+    adapterId,
+    sourceId: record.sourceId,
+    sourceFile: record.sourceFile,
+    itemId: record.itemId,
+    role: record.role || '',
+    transforms: record.transforms || [],
+    ingestMode: record.ingestMode || '',
+    privacyClass: record.privacyClass || '',
+  };
+  return {
+    source: adapterId,
+    source_id: `${record.sourceId}:${record.itemId}`,
+    source_channel: metadata.cwd || metadata.sessionId || '',
+    memory_type: record.memoryType,
+    direction: directionForRole(record.role),
+    participants: metadata.participants || '',
+    subject: metadata.gitBranch || metadata.cwd || record.sourceId,
+    content: record.content,
+    content_raw: record.contentRaw || record.content,
+    metadata: JSON.stringify(metadata),
+    importance: 0.55,
+    timestamp: record.timestamp,
+  };
+}
+
+function directionForRole(role) {
+  if (role === 'user') return 'outbound';
+  if (role === 'assistant' || role === 'tool') return 'observed';
+  if (role === 'exchange') return 'exchange';
+  return 'observed';
+}
+
+module.exports = {
+  SessionIngestService,
+  createSessionIngestService,
+  toBrainMemory,
+};