create-walle 0.9.11 → 0.9.13

package/template/wall-e/sources/record-types.js

@@ -0,0 +1,252 @@
+'use strict';
+
+const RECORD_TYPES = Object.freeze({
+  SOURCE_ITEM: 'source_item',
+  MEMORY_RECORD: 'memory_record',
+  SOURCE_SUMMARY: 'source_summary',
+});
+
+const INGEST_MODES = Object.freeze({
+  CHUNKED_CONTENT: 'chunked_content',
+  WHOLE_RECORD: 'whole_record',
+  METADATA_ONLY: 'metadata_only',
+});
+
+const PRIVACY_CLASSES = Object.freeze({
+  PUBLIC: 'public',
+  INTERNAL: 'internal',
+  PII_POTENTIAL: 'pii_potential',
+  SENSITIVE: 'sensitive',
+  SECRETS_POSSIBLE: 'secrets_possible',
+});
+
+const PRIVACY_CLASS_ORDER = Object.freeze([
+  PRIVACY_CLASSES.PUBLIC,
+  PRIVACY_CLASSES.INTERNAL,
+  PRIVACY_CLASSES.PII_POTENTIAL,
+  PRIVACY_CLASSES.SENSITIVE,
+  PRIVACY_CLASSES.SECRETS_POSSIBLE,
+]);
+
+class SourceValidationError extends Error {
+  constructor(message, { errors = [], record = null } = {}) {
+    super(message);
+    this.name = 'SourceValidationError';
+    this.errors = errors;
+    this.record = record;
+  }
+}
+
+function validateSourceRef(ref = {}) {
+  const errors = [];
+  if (!ref || typeof ref !== 'object') errors.push('source ref must be an object');
+  if (!nonEmpty(ref.adapterId)) errors.push('adapterId is required');
+  if (!nonEmpty(ref.uri) && !nonEmpty(ref.sourceFile)) errors.push('uri or sourceFile is required');
+  if (ref.sourceId != null && !nonEmpty(ref.sourceId)) errors.push('sourceId must be a non-empty string when provided');
+  if (ref.privacyClass != null && !isPrivacyClass(ref.privacyClass)) {
+    errors.push(`privacyClass must be one of: ${PRIVACY_CLASS_ORDER.join(', ')}`);
+  }
+  if (ref.metadata != null && !plainObject(ref.metadata)) errors.push('metadata must be an object when provided');
+  return result(errors, normalizeSourceRef(ref));
+}
+
+function normalizeSourceRef(ref = {}) {
+  const uri = ref.uri || ref.sourceFile || '';
+  return {
+    adapterId: String(ref.adapterId || ''),
+    uri: String(uri || ''),
+    sourceFile: ref.sourceFile || uri || '',
+    sourceId: ref.sourceId || `${ref.adapterId || 'source'}:${uri || 'unknown'}`,
+    cwd: ref.cwd || '',
+    privacyClass: ref.privacyClass || PRIVACY_CLASSES.PII_POTENTIAL,
+    metadata: plainObject(ref.metadata) ? { ...ref.metadata } : {},
+  };
+}
+
+function validateAdapterSchema(schema = {}) {
+  const errors = [];
+  if (!schema || typeof schema !== 'object') errors.push('adapter schema must be an object');
+  if (!nonEmpty(schema.adapterId)) errors.push('adapterId is required');
+  if (!nonEmpty(schema.version)) errors.push('version is required');
+  if (schema.defaultPrivacyClass != null && !isPrivacyClass(schema.defaultPrivacyClass)) {
+    errors.push(`defaultPrivacyClass must be one of: ${PRIVACY_CLASS_ORDER.join(', ')}`);
+  }
+  if (schema.declaredTransformations != null && !Array.isArray(schema.declaredTransformations)) {
+    errors.push('declaredTransformations must be an array');
+  }
+  if (schema.fields != null && !plainObject(schema.fields)) errors.push('fields must be an object when provided');
+  return result(errors, {
+    adapterId: schema.adapterId || '',
+    version: schema.version || '',
+    supportsIncremental: Boolean(schema.supportsIncremental),
+    defaultPrivacyClass: schema.defaultPrivacyClass || PRIVACY_CLASSES.PII_POTENTIAL,
+    declaredTransformations: normalizeTransformList(schema.declaredTransformations || []),
+    fields: plainObject(schema.fields) ? { ...schema.fields } : {},
+    capabilities: plainObject(schema.capabilities) ? { ...schema.capabilities } : {},
+  });
+}
+
+function validateSourceItemMetadata(record = {}) {
+  const errors = [];
+  if (!plainObject(record)) errors.push('source item must be an object');
+  if (record.type !== RECORD_TYPES.SOURCE_ITEM) errors.push(`type must be ${RECORD_TYPES.SOURCE_ITEM}`);
+  validateCommonSourceRecord(record, errors);
+  if (!nonEmpty(record.version)) errors.push('version is required');
+  if (record.ingestMode != null && !isIngestMode(record.ingestMode)) {
+    errors.push(`ingestMode must be one of: ${Object.values(INGEST_MODES).join(', ')}`);
+  }
+  return result(errors, {
+    ...record,
+    ingestMode: record.ingestMode || INGEST_MODES.WHOLE_RECORD,
+    privacyClass: record.privacyClass || PRIVACY_CLASSES.PII_POTENTIAL,
+    metadata: normalizeMetadata(record.metadata),
+  });
+}
+
+function validateMemoryRecord(record = {}, schema = null) {
+  const errors = [];
+  if (!plainObject(record)) errors.push('memory record must be an object');
+  if (record.type !== RECORD_TYPES.MEMORY_RECORD) errors.push(`type must be ${RECORD_TYPES.MEMORY_RECORD}`);
+  validateCommonSourceRecord(record, errors);
+  if (!nonEmpty(record.memoryType)) errors.push('memoryType is required');
+  if (!nonEmpty(record.content) && !nonEmpty(record.contentRaw)) errors.push('content or contentRaw is required');
+  if (record.content != null && typeof record.content !== 'string') errors.push('content must be a string');
+  if (record.contentRaw != null && typeof record.contentRaw !== 'string') errors.push('contentRaw must be a string');
+  if (record.transforms != null && !Array.isArray(record.transforms)) errors.push('transforms must be an array');
+  if (record.ingestMode != null && !isIngestMode(record.ingestMode)) {
+    errors.push(`ingestMode must be one of: ${Object.values(INGEST_MODES).join(', ')}`);
+  }
+  if (schema) errors.push(...undeclaredTransformErrors(record, schema));
+  return result(errors, {
+    ...record,
+    content: record.content == null ? record.contentRaw : record.content,
+    contentRaw: record.contentRaw == null ? record.content : record.contentRaw,
+    transforms: normalizeTransformList(record.transforms || []),
+    ingestMode: record.ingestMode || INGEST_MODES.WHOLE_RECORD,
+    privacyClass: record.privacyClass || PRIVACY_CLASSES.PII_POTENTIAL,
+    metadata: normalizeMetadata(record.metadata),
+  });
+}
+
+function validateSourceSummary(record = {}) {
+  const errors = [];
+  if (!plainObject(record)) errors.push('source summary must be an object');
+  if (record.type !== RECORD_TYPES.SOURCE_SUMMARY) errors.push(`type must be ${RECORD_TYPES.SOURCE_SUMMARY}`);
+  validateCommonSourceRecord(record, errors, { itemIdRequired: false });
+  if (!nonEmpty(record.summary)) errors.push('summary is required');
+  return result(errors, {
+    ...record,
+    privacyClass: record.privacyClass || PRIVACY_CLASSES.PII_POTENTIAL,
+    metadata: normalizeMetadata(record.metadata),
+  });
+}
+
+function validateIngestRecord(record = {}, schema = null) {
+  if (record?.type === RECORD_TYPES.SOURCE_ITEM) return validateSourceItemMetadata(record);
+  if (record?.type === RECORD_TYPES.MEMORY_RECORD) return validateMemoryRecord(record, schema);
+  if (record?.type === RECORD_TYPES.SOURCE_SUMMARY) return validateSourceSummary(record);
+  return result([`unknown source record type: ${record?.type || '(missing)'}`], record);
+}
+
+function assertValid(validation, message = 'invalid source record') {
+  if (!validation.ok) {
+    throw new SourceValidationError(`${message}: ${validation.errors.join('; ')}`, {
+      errors: validation.errors,
+      record: validation.value,
+    });
+  }
+  return validation.value;
+}
+
+function assertIngestRecord(record, schema = null) {
+  return assertValid(validateIngestRecord(record, schema), 'invalid ingest record');
+}
+
+function assertAdapterSchema(schema) {
+  return assertValid(validateAdapterSchema(schema), 'invalid adapter schema');
+}
+
+function assertSourceRef(ref) {
+  return assertValid(validateSourceRef(ref), 'invalid source ref');
+}
+
+function undeclaredTransformErrors(record = {}, schema = {}) {
+  const declared = new Set(normalizeTransformList(schema.declaredTransformations || []));
+  const transforms = normalizeTransformList(record.transforms || []);
+  return transforms
+    .filter((name) => !declared.has(name))
+    .map((name) => `transform is not declared by adapter ${schema.adapterId || '(unknown)'}: ${name}`);
+}
+
+function validateCommonSourceRecord(record, errors, { itemIdRequired = true } = {}) {
+  if (!nonEmpty(record.sourceId)) errors.push('sourceId is required');
+  if (!nonEmpty(record.sourceFile)) errors.push('sourceFile is required');
+  if (itemIdRequired && !nonEmpty(record.itemId)) errors.push('itemId is required');
+  if (record.timestamp != null && !validDate(record.timestamp)) errors.push('timestamp must be parseable when provided');
+  if (record.privacyClass != null && !isPrivacyClass(record.privacyClass)) {
+    errors.push(`privacyClass must be one of: ${PRIVACY_CLASS_ORDER.join(', ')}`);
+  }
+  if (record.metadata != null && !plainObject(record.metadata)) errors.push('metadata must be an object when provided');
+}
+
+function normalizeTransformList(list) {
+  return [...new Set((list || [])
+    .filter((name) => typeof name === 'string')
+    .map((name) => name.trim())
+    .filter(Boolean))];
+}
+
+function normalizeMetadata(metadata) {
+  return plainObject(metadata) ? { ...metadata } : {};
+}
+
+function privacyClassAllows({ recordClass, floor }) {
+  if (!floor) return true;
+  if (!isPrivacyClass(recordClass) || !isPrivacyClass(floor)) return false;
+  return PRIVACY_CLASS_ORDER.indexOf(recordClass) <= PRIVACY_CLASS_ORDER.indexOf(floor);
+}
+
+function result(errors, value) {
+  return { ok: errors.length === 0, errors, value };
+}
+
+function isIngestMode(value) {
+  return Object.values(INGEST_MODES).includes(value);
+}
+
+function isPrivacyClass(value) {
+  return PRIVACY_CLASS_ORDER.includes(value);
+}
+
+function nonEmpty(value) {
+  return typeof value === 'string' && value.trim().length > 0;
+}
+
+function plainObject(value) {
+  return !!value && typeof value === 'object' && !Array.isArray(value);
+}
+
+function validDate(value) {
+  return typeof value === 'string' && !Number.isNaN(new Date(value).getTime());
+}
+
+module.exports = {
+  INGEST_MODES,
+  PRIVACY_CLASSES,
+  PRIVACY_CLASS_ORDER,
+  RECORD_TYPES,
+  SourceValidationError,
+  assertAdapterSchema,
+  assertIngestRecord,
+  assertSourceRef,
+  assertValid,
+  normalizeSourceRef,
+  normalizeTransformList,
+  privacyClassAllows,
+  validateAdapterSchema,
+  validateIngestRecord,
+  validateMemoryRecord,
+  validateSourceItemMetadata,
+  validateSourceRef,
+  validateSourceSummary,
+};

package/template/wall-e/sources/registry.js

@@ -0,0 +1,92 @@
+'use strict';
+
+const { Registry } = require('../plugins/registry-base');
+const { SourceAdapterBase } = require('./base');
+const { assertAdapterSchema } = require('./record-types');
+
+const _registry = new Registry('source-adapter');
+
+function register(entry, opts = {}) {
+  const normalized = normalizeEntry(entry);
+  return _registry.register(normalized, opts);
+}
+
+function normalizeEntry(entry) {
+  if (typeof entry === 'function') {
+    const instance = new entry();
+    return {
+      id: instance.schema.adapterId,
+      adapterClass: entry,
+      schema: instance.schema,
+      source: 'class',
+    };
+  }
+  if (entry instanceof SourceAdapterBase) {
+    return {
+      id: entry.schema.adapterId,
+      adapter: entry,
+      schema: entry.schema,
+      source: 'instance',
+    };
+  }
+  if (entry && typeof entry === 'object') {
+    const schema = assertAdapterSchema(entry.schema || entry.manifest || entry);
+    return {
+      ...entry,
+      id: entry.id || schema.adapterId,
+      schema,
+    };
+  }
+  throw new Error('source adapter registry entry must be a class, instance, or object');
+}
+
+function instantiate(id, opts = {}) {
+  const entry = _registry.get(id);
+  if (!entry) return null;
+  if (entry.adapter) return entry.adapter;
+  if (entry.adapterClass) return new entry.adapterClass(opts);
+  if (typeof entry.factory === 'function') return entry.factory(opts);
+  return null;
+}
+
+function get(id) {
+  return _registry.get(id) || null;
+}
+
+function has(id) {
+  return _registry.has(id);
+}
+
+function list(filter = {}) {
+  let entries = _registry.list();
+  if (filter.privacyClass) {
+    entries = entries.filter((entry) => entry.schema.defaultPrivacyClass === filter.privacyClass);
+  }
+  if (filter.incremental != null) {
+    entries = entries.filter((entry) => Boolean(entry.schema.supportsIncremental) === Boolean(filter.incremental));
+  }
+  return entries;
+}
+
+function ids(filter = {}) {
+  return list(filter).map((entry) => entry.id);
+}
+
+function size() {
+  return _registry.size();
+}
+
+function clear() {
+  _registry.clear();
+}
+
+module.exports = {
+  clear,
+  get,
+  has,
+  ids,
+  instantiate,
+  list,
+  register,
+  size,
+};

package/template/wall-e/sources/transforms.js

@@ -0,0 +1,100 @@
+'use strict';
+
+const crypto = require('node:crypto');
+
+let stripNoise;
+try {
+  stripNoise = require('../extraction/noise-stripper').stripNoise;
+} catch {
+  stripNoise = (value) => value;
+}
+
+const BASE64_BLOCK_RE = /\b[A-Za-z0-9+/]{512,}={0,2}\b/g;
+
+const RESERVED_TRANSFORMATIONS = Object.freeze({
+  utf8_replace_invalid: {
+    name: 'utf8_replace_invalid',
+    description: 'Decode bytes as UTF-8 and replace invalid sequences.',
+    apply(value) {
+      if (Buffer.isBuffer(value)) return value.toString('utf8');
+      return String(value ?? '');
+    },
+  },
+  newline_normalize: {
+    name: 'newline_normalize',
+    description: 'Normalize CRLF/CR newlines to LF.',
+    apply(value) {
+      return String(value ?? '').replace(/\r\n?/g, '\n');
+    },
+  },
+  whitespace_trim: {
+    name: 'whitespace_trim',
+    description: 'Trim leading and trailing whitespace.',
+    apply(value) {
+      return String(value ?? '').trim();
+    },
+  },
+  strip_tool_chrome: {
+    name: 'strip_tool_chrome',
+    description: 'Remove Claude/Codex/Wall-E tool wrapper chrome from searchable text.',
+    apply(value) {
+      return stripNoise(String(value ?? ''));
+    },
+  },
+  tool_result_summarize: {
+    name: 'tool_result_summarize',
+    description: 'Bound very large tool results while keeping their existence visible.',
+    apply(value, { maxToolResultChars = 4000 } = {}) {
+      const text = String(value ?? '');
+      if (text.length <= maxToolResultChars) return text;
+      return `${text.slice(0, maxToolResultChars)}\n[tool result truncated: ${text.length - maxToolResultChars} chars omitted]`;
+    },
+  },
+  base64_payload_omit: {
+    name: 'base64_payload_omit',
+    description: 'Replace large inline base64 payloads with deterministic hashes.',
+    apply(value) {
+      return String(value ?? '').replace(BASE64_BLOCK_RE, (match) => {
+        const hash = crypto.createHash('sha256').update(match).digest('hex').slice(0, 16);
+        return `[base64 omitted sha256:${hash} bytes:${Buffer.byteLength(match, 'utf8')}]`;
+      });
+    },
+  },
+  exchange_pair_chunk: {
+    name: 'exchange_pair_chunk',
+    description: 'Content was formed by pairing adjacent user and assistant turns.',
+    apply(value) {
+      return String(value ?? '');
+    },
+  },
+});
+
+function getTransformation(name) {
+  return RESERVED_TRANSFORMATIONS[name] || null;
+}
+
+function hasTransformation(name) {
+  return !!getTransformation(name);
+}
+
+function listReservedTransformations() {
+  return Object.keys(RESERVED_TRANSFORMATIONS);
+}
+
+function applyTransforms(value, transforms = [], context = {}) {
+  let current = value;
+  for (const name of transforms || []) {
+    const transform = getTransformation(name);
+    if (!transform) throw new Error(`Unknown reserved transformation: ${name}`);
+    current = transform.apply(current, context);
+  }
+  return current;
+}
+
+module.exports = {
+  RESERVED_TRANSFORMATIONS,
+  applyTransforms,
+  getTransformation,
+  hasTransformation,
+  listReservedTransformations,
+};

package/template/wall-e/sources/walle-jsonl.js

@@ -0,0 +1,108 @@
+'use strict';
+
+const { SourceAdapterBase } = require('./base');
+const {
+  cleanUserText,
+  extractMessageText,
+  fileVersion,
+  looksLikeToolResult,
+  readJsonlFile,
+  sourceIdFromFile,
+  summarizeToolInput,
+  toIso,
+} = require('./jsonl-utils');
+const { buildExchangeRecords, memoryRecord, messageToRecords, sourceItem } = require('./coding-session-utils');
+
+class WalleJsonlAdapter extends SourceAdapterBase {
+  static schema = {
+    adapterId: 'walle-jsonl',
+    version: '1.0.0',
+    supportsIncremental: true,
+    defaultPrivacyClass: 'pii_potential',
+    declaredTransformations: [
+      'newline_normalize',
+      'strip_tool_chrome',
+      'base64_payload_omit',
+      'tool_result_summarize',
+      'whitespace_trim',
+      'exchange_pair_chunk',
+    ],
+    fields: {
+      sourceId: { type: 'string', required: true },
+      sourceFile: { type: 'string', required: true },
+      cwd: { type: 'string', required: false },
+      gitBranch: { type: 'string', required: false },
+    },
+  };
+
+  validateSource(source) {
+    const normalized = super.validateSource(source);
+    if (!source.sourceId) normalized.sourceId = sourceIdFromFile('walle', normalized.sourceFile || normalized.uri);
+    return normalized;
+  }
+
+  async *ingest({ source, context }) {
+    const filePath = source.sourceFile || source.uri;
+    const version = fileVersion(filePath);
+    const { events, diagnostics } = readJsonlFile(filePath);
+    for (const diagnostic of diagnostics) context?.warn?.(diagnostic.message, diagnostic);
+
+    let cwd = source.cwd || '';
+    let gitBranch = source.metadata?.gitBranch || '';
+    const messages = [];
+
+    for (const { line, event } of events) {
+      if (event.type === 'session_meta' || event.type === 'session_start') {
+        cwd = event.cwd || event.data?.cwd || cwd;
+        gitBranch = event.gitBranch || gitBranch;
+        continue;
+      }
+      if (event.type === 'user' || event.type === 'assistant') {
+        const role = event.type;
+        const content = role === 'user'
+          ? cleanUserText(extractMessageText(event.message || event))
+          : extractMessageText(event.message || event);
+        if (!content || looksLikeToolResult(content)) continue;
+        const message = {
+          itemId: event.uuid || `line-${line}`,
+          role,
+          content,
+          timestamp: toIso(event.timestamp),
+          cwd: event.cwd || cwd,
+          gitBranch: event.gitBranch || gitBranch,
+          rawType: event.type,
+          harness: 'walle',
+          toolCalls: [],
+        };
+        messages.push(message);
+        for (const record of messageToRecords({ source, message, version })) yield record;
+        continue;
+      }
+      if (event.type === 'walle_part' && (event.partType === 'tool_call' || event.partType === 'tool')) {
+        const data = event.data || {};
+        const name = data.name || data.tool || 'tool';
+        const itemId = event.uuid || `tool-${line}`;
+        yield sourceItem({
+          source,
+          itemId,
+          version,
+          timestamp: event.timestamp,
+          metadata: { role: 'tool', cwd, gitBranch, name, input: data.input || {} },
+        });
+        yield memoryRecord({
+          source,
+          itemId: `${itemId}:tool`,
+          memoryType: 'coding_session_tool_call',
+          role: 'tool',
+          contentRaw: `[Tool: ${name}] ${summarizeToolInput(name, data.input || {})}`,
+          timestamp: event.timestamp,
+          metadata: { cwd, gitBranch, name, input: data.input || {} },
+        });
+      }
+    }
+
+    for (const record of buildExchangeRecords(source, messages)) yield record;
+  }
+}
+
+module.exports = WalleJsonlAdapter;

package/template/wall-e/tools/coding-middleware.js

@@ -25,9 +25,10 @@ const HOOK_TYPES = new Set([
 ]);
 
 class CodingMiddleware {
-  constructor() {
+  constructor({ hookRuntime = null } = {}) {
     this._hooks = {};
     for (const type of HOOK_TYPES) this._hooks[type] = [];
+    this.hookRuntime = hookRuntime;
 
     // Config notification
     // Ported from OpenCode plugin/index.ts — hook.config?.() pattern (lines 237-245)
@@ -43,6 +44,10 @@ class CodingMiddleware {
     this._hooks[hookName].push(fn);
   }
 
+  attachHookRuntime(hookRuntime) {
+    this.hookRuntime = hookRuntime;
+  }
+
   /**
    * Run all hooks for a given type.
    * - For context-mutating hooks (llm.before, llm.after): hooks mutate ctx in place
@@ -75,6 +80,31 @@ class CodingMiddleware {
         console.error(`[middleware] Error in ${hookName} hook:`, err.message);
       }
     }
+    return this._runExternalHook(hookName, args, result);
+  }
+
+  async _runExternalHook(hookName, args, result) {
+    if (!this.hookRuntime?.trigger) return result;
+    if (hookName === 'llm.before') {
+      const payload = await this.hookRuntime.trigger('llm_input', { context: args[0] || {} }, { hookName });
+      return payload?.context || result;
+    }
+    if (hookName === 'llm.after') {
+      const payload = await this.hookRuntime.trigger('llm_output', {
+        context: args[0] || {},
+        response: result,
+      }, { hookName });
+      return payload?.response || result;
+    }
+    if (hookName === 'tool.after') {
+      const payload = await this.hookRuntime.trigger('tool_result', {
+        context: args[0] || {},
+        toolName: args[1],
+        args: args[2],
+        result,
+      }, { hookName });
+      return payload?.result ?? result;
+    }
     return result;
   }
 

package/template/wall-e/tools/file-tracker.js

@@ -17,6 +17,18 @@ const sessions = new Map();
  * @param {string} sessionId - Coding session ID
  */
 function recordRead(filePath, sessionId) {
+  recordCurrent(filePath, sessionId);
+}
+
+/**
+ * Record the current mtime and size of a file after this session writes it.
+ * This keeps stale-edit protection focused on external races, not on the
+ * agent's own prior successful edits.
+ * @param {string} filePath - Absolute path
+ * @param {string} sessionId - Coding session ID
+ */
+function recordCurrent(filePath, sessionId) {
+  if (!sessionId) return;
   const resolved = path.resolve(filePath);
   const stat = fs.statSync(resolved);
   if (!sessions.has(sessionId)) sessions.set(sessionId, new Map());
@@ -26,6 +38,18 @@ function recordRead(filePath, sessionId) {
   });
 }
 
+/**
+ * Forget a tracked file after this session deletes it.
+ * @param {string} filePath - Absolute path
+ * @param {string} sessionId - Coding session ID
+ */
+function forgetFile(filePath, sessionId) {
+  if (!sessionId) return;
+  const resolved = path.resolve(filePath);
+  const sessionMap = sessions.get(sessionId);
+  if (sessionMap) sessionMap.delete(resolved);
+}
+
 /**
  * Assert a file has not changed since the last recordRead().
  * Called from editFile()/writeFile() in local-tools.js before writing.
@@ -69,4 +93,4 @@ function clearSession(sessionId) {
   sessions.delete(sessionId);
 }
 
-module.exports = { recordRead, assertUnchanged, clearSession };
+module.exports = { recordRead, recordCurrent, forgetFile, assertUnchanged, clearSession };