create-walle 0.9.11 → 0.9.13

package/template/wall-e/sources/coding-session-utils.js

@@ -0,0 +1,117 @@
+'use strict';
+
+const { INGEST_MODES, PRIVACY_CLASSES, RECORD_TYPES } = require('./record-types');
+const { cleanedContent, toIso } = require('./jsonl-utils');
+
+const DEFAULT_TEXT_TRANSFORMS = Object.freeze([
+  'newline_normalize',
+  'strip_tool_chrome',
+  'base64_payload_omit',
+  'tool_result_summarize',
+  'whitespace_trim',
+]);
+
+function sourceItem({ source, itemId, version, timestamp, ingestMode, privacyClass, metadata }) {
+  return {
+    type: RECORD_TYPES.SOURCE_ITEM,
+    sourceId: source.sourceId,
+    sourceFile: source.sourceFile,
+    itemId,
+    version,
+    timestamp: toIso(timestamp),
+    ingestMode: ingestMode || INGEST_MODES.WHOLE_RECORD,
+    privacyClass: privacyClass || source.privacyClass || PRIVACY_CLASSES.PII_POTENTIAL,
+    metadata: metadata || {},
+  };
+}
+
+function memoryRecord({
+  source,
+  itemId,
+  memoryType,
+  role,
+  contentRaw,
+  transforms = DEFAULT_TEXT_TRANSFORMS,
+  timestamp,
+  privacyClass,
+  metadata,
+}) {
+  return {
+    type: RECORD_TYPES.MEMORY_RECORD,
+    sourceId: source.sourceId,
+    sourceFile: source.sourceFile,
+    itemId,
+    role,
+    memoryType,
+    contentRaw,
+    content: cleanedContent(contentRaw, transforms),
+    transforms,
+    timestamp: toIso(timestamp),
+    ingestMode: INGEST_MODES.WHOLE_RECORD,
+    privacyClass: privacyClass || source.privacyClass || PRIVACY_CLASSES.PII_POTENTIAL,
+    metadata: metadata || {},
+  };
+}
+
+function messageToRecords({ source, message, version }) {
+  const metadata = {
+    role: message.role,
+    cwd: message.cwd || '',
+    gitBranch: message.gitBranch || '',
+    toolCalls: message.toolCalls || [],
+    rawType: message.rawType || '',
+    harness: message.harness || '',
+  };
+  const item = sourceItem({
+    source,
+    itemId: message.itemId,
+    version,
+    timestamp: message.timestamp,
+    metadata,
+  });
+  const memory = memoryRecord({
+    source,
+    itemId: `${message.itemId}:message`,
+    memoryType: message.memoryType || `coding_session_${message.role}_message`,
+    role: message.role,
+    contentRaw: message.content,
+    timestamp: message.timestamp,
+    metadata,
+  });
+  return [item, memory];
+}
+
+function buildExchangeRecords(source, messages) {
+  const records = [];
+  for (let i = 0; i < messages.length - 1; i++) {
+    const user = messages[i];
+    const assistant = messages[i + 1];
+    if (user.role !== 'user' || assistant.role !== 'assistant') continue;
+    const contentRaw = `Q: ${user.content}\nA: ${assistant.content}`;
+    records.push(memoryRecord({
+      source,
+      itemId: `exchange:${user.itemId}:${assistant.itemId}`,
+      memoryType: 'coding_session_exchange',
+      role: 'exchange',
+      contentRaw,
+      transforms: [...DEFAULT_TEXT_TRANSFORMS, 'exchange_pair_chunk'],
+      timestamp: user.timestamp || assistant.timestamp,
+      metadata: {
+        userItemId: user.itemId,
+        assistantItemId: assistant.itemId,
+        cwd: user.cwd || assistant.cwd || '',
+        gitBranch: user.gitBranch || assistant.gitBranch || '',
+        toolCalls: assistant.toolCalls || [],
+      },
+    }));
+  }
+  return records;
+}
+
+module.exports = {
+  DEFAULT_TEXT_TRANSFORMS,
+  buildExchangeRecords,
+  memoryRecord,
+  messageToRecords,
+  sourceItem,
+};

package/template/wall-e/sources/contract-suite.js

@@ -0,0 +1,59 @@
+'use strict';
+
+const assert = require('node:assert/strict');
+const { collectIngestRecords } = require('./base');
+const {
+  RECORD_TYPES,
+  SourceValidationError,
+  validateAdapterSchema,
+  validateIngestRecord,
+  validateSourceRef,
+} = require('./record-types');
+
+async function assertSourceAdapterContract(adapter, { source, minRecords = 1, requireMemoryRecord = true } = {}) {
+  const schemaValidation = validateAdapterSchema(adapter.describeSchema ? adapter.describeSchema() : adapter.schema);
+  assert.equal(schemaValidation.ok, true, schemaValidation.errors.join('; '));
+
+  const sourceValidation = validateSourceRef(source);
+  assert.equal(sourceValidation.ok, true, sourceValidation.errors.join('; '));
+
+  const { records } = await collectIngestRecords(adapter, sourceValidation.value, { validate: false });
+  assert.ok(records.length >= minRecords, `expected at least ${minRecords} source records`);
+
+  let memoryRecords = 0;
+  for (const record of records) {
+    const validation = validateIngestRecord(record, schemaValidation.value);
+    assert.equal(validation.ok, true, validation.errors.join('; '));
+    if (record.type === RECORD_TYPES.MEMORY_RECORD) memoryRecords += 1;
+  }
+
+  if (requireMemoryRecord) {
+    assert.ok(memoryRecords > 0, 'expected at least one memory_record');
+  }
+
+  return { schema: schemaValidation.value, records };
+}
+
+async function assertSourceAdapterContractFails(adapter, { source, pattern } = {}) {
+  let error = null;
+  try {
+    await assertSourceAdapterContract(adapter, { source });
+  } catch (err) {
+    error = err;
+  }
+  assert.ok(error, 'expected adapter contract to fail');
+  if (pattern) assert.match(error.message, pattern);
+  return error;
+}
+
+async function assertAdapterRejectsInvalidRecords(adapter, { source } = {}) {
+  await assert.rejects(async () => {
+    await collectIngestRecords(adapter, source);
+  }, SourceValidationError);
+}
+
+module.exports = {
+  assertAdapterRejectsInvalidRecords,
+  assertSourceAdapterContract,
+  assertSourceAdapterContractFails,
+};

package/template/wall-e/sources/gemini-jsonl.js

@@ -0,0 +1,85 @@
+'use strict';
+
+const { SourceAdapterBase } = require('./base');
+const {
+  cleanUserText,
+  extractMessageText,
+  fileVersion,
+  looksLikeToolResult,
+  readJsonlFile,
+  sourceIdFromFile,
+  toIso,
+} = require('./jsonl-utils');
+const { buildExchangeRecords, messageToRecords } = require('./coding-session-utils');
+
+class GeminiJsonlAdapter extends SourceAdapterBase {
+  static schema = {
+    adapterId: 'gemini-jsonl',
+    version: '1.0.0',
+    supportsIncremental: true,
+    defaultPrivacyClass: 'pii_potential',
+    declaredTransformations: [
+      'newline_normalize',
+      'strip_tool_chrome',
+      'base64_payload_omit',
+      'tool_result_summarize',
+      'whitespace_trim',
+      'exchange_pair_chunk',
+    ],
+    fields: {
+      sourceId: { type: 'string', required: true },
+      sourceFile: { type: 'string', required: true },
+      cwd: { type: 'string', required: false },
+    },
+  };
+
+  validateSource(source) {
+    const normalized = super.validateSource(source);
+    if (!source.sourceId) normalized.sourceId = sourceIdFromFile('gemini', normalized.sourceFile || normalized.uri);
+    return normalized;
+  }
+
+  async *ingest({ source, context }) {
+    const filePath = source.sourceFile || source.uri;
+    const version = fileVersion(filePath);
+    const { events, diagnostics } = readJsonlFile(filePath);
+    for (const diagnostic of diagnostics) context?.warn?.(diagnostic.message, diagnostic);
+
+    let cwd = source.cwd || '';
+    const messages = [];
+
+    for (const { line, event } of events) {
+      if (event.type === 'session_metadata' || event.session_metadata) {
+        const meta = event.session_metadata || event;
+        cwd = meta.cwd || meta.workspace || cwd;
+        continue;
+      }
+      const rawRole = event.role || event.type || event.author || event.sender;
+      const role = rawRole === 'model' || rawRole === 'gemini' || rawRole === 'assistant'
+        ? 'assistant'
+        : rawRole === 'user' ? 'user' : null;
+      if (!role) continue;
+      const content = role === 'user'
+        ? cleanUserText(extractMessageText(event))
+        : extractMessageText(event);
+      if (!content || looksLikeToolResult(content)) continue;
+
+      const message = {
+        itemId: event.id || event.uuid || `line-${line}`,
+        role,
+        content,
+        timestamp: toIso(event.timestamp || event.created_at),
+        cwd,
+        rawType: event.type || '',
+        harness: 'gemini',
+        toolCalls: [],
+      };
+      messages.push(message);
+      for (const record of messageToRecords({ source, message, version })) yield record;
+    }
+
+    for (const record of buildExchangeRecords(source, messages)) yield record;
+  }
+}
+
+module.exports = GeminiJsonlAdapter;

package/template/wall-e/sources/index.js

@@ -0,0 +1,9 @@
+'use strict';
+
+module.exports = {
+  ...require('./base'),
+  ...require('./record-types'),
+  ...require('./transforms'),
+  ...require('./builtin'),
+  registry: require('./registry'),
+};

package/template/wall-e/sources/jsonl-utils.js

@@ -0,0 +1,181 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+
+const { applyTransforms } = require('./transforms');
+
+function readJsonlFile(filePath) {
+  const events = [];
+  const diagnostics = [];
+  let raw = '';
+  try {
+    raw = fs.readFileSync(filePath, 'utf8');
+  } catch (err) {
+    return { events, diagnostics: [{ level: 'error', message: err.message, filePath }] };
+  }
+  const lines = raw.split('\n');
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (!line.trim()) continue;
+    try {
+      events.push({ line: i + 1, event: JSON.parse(line) });
+    } catch (err) {
+      diagnostics.push({ level: 'warn', message: `Malformed JSONL line ${i + 1}: ${err.message}`, filePath, line: i + 1 });
+    }
+  }
+  return { events, diagnostics };
+}
+
+function fileVersion(filePath) {
+  try {
+    const stat = fs.statSync(filePath);
+    return `mtime:${Math.round(stat.mtimeMs)}:size:${stat.size}`;
+  } catch {
+    return 'missing';
+  }
+}
+
+function sourceIdFromFile(prefix, filePath) {
+  const base = path.basename(filePath || 'unknown', '.jsonl');
+  const hash = crypto.createHash('sha1').update(path.resolve(filePath || base)).digest('hex').slice(0, 12);
+  return `${prefix}:${base}:${hash}`;
+}
+
+function extractMessageText(value) {
+  if (value == null) return '';
+  if (typeof value === 'string') return value;
+  if (Array.isArray(value)) return extractBlocksText(value);
+  if (typeof value !== 'object') return String(value);
+  if (value.message?.content != null) return extractMessageText(value.message.content);
+  if (value.content != null) return extractMessageText(value.content);
+  if (value.text != null) return String(value.text);
+  if (value.parts != null) return extractMessageText(value.parts);
+  return '';
+}
+
+function extractBlocksText(blocks) {
+  const parts = [];
+  for (const block of blocks || []) {
+    if (!block) continue;
+    const type = block.type || '';
+    if (type === 'text' || type === 'input_text' || type === 'output_text') {
+      if (typeof block.text === 'string') parts.push(block.text);
+      continue;
+    }
+    if (type === 'tool_use' || type === 'function_call') {
+      const name = block.name || block.function?.name || 'tool';
+      parts.push(`[Tool: ${name}] ${summarizeToolInput(name, block.input ?? block.arguments)}`);
+      continue;
+    }
+    if (type === 'tool_result' || type === 'function_call_output') {
+      const content = block.content ?? block.output ?? '';
+      const text = extractMessageText(content);
+      if (text) parts.push(`[Tool result] ${text}`);
+      continue;
+    }
+    if (block.text) parts.push(String(block.text));
+  }
+  return parts.join('\n').trim();
+}
+
+function extractToolCalls(value) {
+  const calls = [];
+  const blocks = Array.isArray(value?.content) ? value.content
+    : Array.isArray(value) ? value
+      : [];
+  for (const block of blocks) {
+    if (!block || typeof block !== 'object') continue;
+    if (block.type === 'tool_use' || block.type === 'function_call') {
+      calls.push({
+        name: block.name || block.function?.name || 'tool',
+        input: parseToolInput(block.input ?? block.arguments ?? {}),
+      });
+    }
+  }
+  return calls;
+}
+
+function parseToolInput(value) {
+  if (value == null) return {};
+  if (typeof value === 'object') return value;
+  if (typeof value !== 'string') return { value: String(value) };
+  try {
+    const parsed = JSON.parse(value);
+    return parsed && typeof parsed === 'object' ? parsed : { value: parsed };
+  } catch {
+    return { raw: value };
+  }
+}
+
+function summarizeToolInput(toolName, input) {
+  const parsed = parseToolInput(input);
+  if (!parsed || typeof parsed !== 'object') return '';
+  const direct = parsed.file_path || parsed.filePath || parsed.path || parsed.command || parsed.cmd || parsed.pattern || parsed.query || parsed.raw;
+  if (direct) return truncate(String(direct), 160);
+  const keys = Object.keys(parsed);
+  if (keys.length === 0) return '';
+  return truncate(JSON.stringify(parsed), 160);
+}
+
+function cleanUserText(text) {
+  if (!text) return '';
+  return String(text)
+    .replace(/\r\n/g, '\n')
+    .replace(/^\s*<environment_context>[\s\S]*?<\/environment_context>\s*/gi, '')
+    .replace(/^\s*<permissions instructions>[\s\S]*?<\/permissions instructions>\s*/gi, '')
+    .replace(/^\s*<skills_instructions>[\s\S]*?<\/skills_instructions>\s*/gi, '')
+    .replace(/^\s*<plugins_instructions>[\s\S]*?<\/plugins_instructions>\s*/gi, '')
+    .replace(/^\s*<skill>[\s\S]*?<\/skill>\s*/gi, '')
+    .trim();
+}
+
+function looksLikeToolResult(text) {
+  const trimmed = String(text || '').trim();
+  if (!trimmed) return true;
+  const head = trimmed.slice(0, 600);
+  return [
+    /^\[Request interrupted/i,
+    /^<tool_use_error>/i,
+    /^<system-reminder>/i,
+    /^<local-command-/i,
+    /^<command-/i,
+    /^<task-notification>/i,
+    /^<environment_context>/i,
+    /^<permissions instructions>/i,
+    /^<skills_instructions>/i,
+    /^<plugins_instructions>/i,
+    /^Base directory for this skill:/i,
+  ].some((re) => re.test(head));
+}
+
+function cleanedContent(raw, transforms, context = {}) {
+  return applyTransforms(raw, transforms, context);
+}
+
+function truncate(value, max) {
+  const text = String(value || '');
+  return text.length > max ? `${text.slice(0, max - 3)}...` : text;
+}
+
+function toIso(value) {
+  if (!value) return new Date().toISOString();
+  const date = new Date(value);
+  return Number.isNaN(date.getTime()) ? new Date().toISOString() : date.toISOString();
+}
+
+module.exports = {
+  cleanUserText,
+  cleanedContent,
+  extractMessageText,
+  extractToolCalls,
+  fileVersion,
+  looksLikeToolResult,
+  parseToolInput,
+  readJsonlFile,
+  sourceIdFromFile,
+  summarizeToolInput,
+  toIso,
+  truncate,
+};