create-walle 0.9.11 → 0.9.13

package/template/wall-e/brain.js

@@ -2,6 +2,7 @@
 const Database = require('better-sqlite3');
 const path = require('path');
 const fs = require('fs');
+const crypto = require('crypto');
 const { v4: uuidv4 } = require('uuid');
 const { EventEmitter } = require('events');
 const { createWriteAuditLog } = require('./db/write-audit');
@@ -33,7 +34,7 @@ let currentDbPath = null;
 let _daemonOwned = false; // When true, closeDb() is a no-op (daemon manages lifecycle)
 
 // --- Schema versioning via PRAGMA user_version ---
-const SCHEMA_VERSION = 9; // Bump on every migration addition
+const SCHEMA_VERSION = 10; // Bump on every migration addition
 
 const MIGRATIONS = {
   1: (d) => {
@@ -297,6 +298,31 @@ const MIGRATIONS = {
     d.prepare("CREATE INDEX IF NOT EXISTS idx_agent_runner_eval_runner ON agent_runner_evaluations(runner_id, task_type)").run();
     d.prepare("CREATE INDEX IF NOT EXISTS idx_agent_runner_eval_created ON agent_runner_evaluations(created_at)").run();
   },
+  10: (d) => {
+    const addCol = (table, col, type) => {
+      try { d.prepare(`SELECT ${col} FROM ${table} LIMIT 0`).run(); } catch (_) {
+        d.prepare(`ALTER TABLE ${table} ADD COLUMN ${col} ${type}`).run();
+      }
+    };
+    addCol('eval_benchmark_runs', 'sample_id', 'TEXT');
+    addCol('eval_benchmark_runs', 'dataset_version', 'TEXT');
+    addCol('eval_benchmark_runs', 'dataset_hash', 'TEXT');
+    addCol('eval_benchmark_runs', 'prompt_hash', 'TEXT');
+    addCol('eval_benchmark_runs', 'system_prompt_hash', 'TEXT');
+    addCol('eval_benchmark_runs', 'repo_sha', 'TEXT');
+    addCol('eval_benchmark_runs', 'scorer_version', 'TEXT');
+    addCol('eval_benchmark_runs', 'evaluator_version', 'TEXT');
+    addCol('eval_benchmark_runs', 'scoring_method', 'TEXT');
+    addCol('eval_benchmark_runs', 'run_config_json', 'TEXT');
+    addCol('eval_benchmark_runs', 'eval_manifest_json', 'TEXT');
+    addCol('eval_benchmark_runs', 'artifact_path', 'TEXT');
+    addCol('eval_benchmark_runs', 'model_snapshot', 'TEXT');
+    addCol('eval_benchmark_runs', 'temperature', 'REAL');
+    addCol('eval_benchmark_runs', 'seed', 'TEXT');
+    addCol('eval_benchmark_runs', 'trusted', 'INTEGER DEFAULT 0');
+    d.prepare("CREATE INDEX IF NOT EXISTS idx_eval_bench_trusted ON eval_benchmark_runs(trusted, provider, model)").run();
+    d.prepare("CREATE INDEX IF NOT EXISTS idx_eval_bench_dataset ON eval_benchmark_runs(suite, dataset_version)").run();
+  },
 };
 
 // Schema invariants — columns/tables that MUST exist after the named migration.
@@ -312,6 +338,7 @@ const SCHEMA_INVARIANTS = [
   { migration: 7, table: 'chat_messages', column: 'attachments' },
   { migration: 8, table: 'model_providers', column: 'auth_method' },
   { migration: 9, table: 'agent_runner_evaluations', column: 'runner_id' },
+  { migration: 10, table: 'eval_benchmark_runs', column: 'dataset_version' },
 ];
 
 function _columnExists(d, table, column) {
@@ -393,9 +420,22 @@ function _detectUpgrade(previousSchemaVersion) {
 
     const storedVersion = getKv('installed_version');
     if (!storedVersion) {
+      const existingInstall = previousSchemaVersion > 0 || _hasExistingInstallFootprint();
       setKv('installed_version', pkgVersion);
-      setKv('first_installed_at', new Date().toISOString());
-      telemetry.track('install', { version: pkgVersion, fresh: true, schema: SCHEMA_VERSION });
+      if (existingInstall) {
+        telemetry.track('upgrade', {
+          from: 'unknown',
+          to: pkgVersion,
+          schema_from: previousSchemaVersion,
+          schema_to: SCHEMA_VERSION,
+          migrations_applied: Math.max(0, SCHEMA_VERSION - previousSchemaVersion),
+          first_tracked: true,
+        });
+        setKv('last_upgrade_at', new Date().toISOString());
+      } else {
+        setKv('first_installed_at', new Date().toISOString());
+        telemetry.track('install', { version: pkgVersion, fresh: true, schema: SCHEMA_VERSION });
+      }
     } else if (storedVersion !== pkgVersion) {
       telemetry.track('upgrade', {
         from: storedVersion,
@@ -420,6 +460,32 @@ function _readVersion() {
   } catch { return 'unknown'; }
 }
 
+function _hasExistingInstallFootprint() {
+  try {
+    const d = getDb();
+    const hasTable = d.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name = ? LIMIT 1");
+    const footprintTables = [
+      ['memories', 'SELECT count(*) AS c FROM memories'],
+      ['knowledge', 'SELECT count(*) AS c FROM knowledge'],
+      ['tasks', 'SELECT count(*) AS c FROM tasks'],
+      ['skills', 'SELECT count(*) AS c FROM skills'],
+      ['chat_messages', 'SELECT count(*) AS c FROM chat_messages'],
+      ['sessions', 'SELECT count(*) AS c FROM sessions'],
+    ];
+    for (const [table, countSql] of footprintTables) {
+      if (!hasTable.get(table)) continue;
+      if ((d.prepare(countSql).get()?.c || 0) > 0) return true;
+    }
+    if (hasTable.get('kv_store')) {
+      const row = d.prepare(
+        "SELECT 1 FROM kv_store WHERE key NOT IN ('installed_version', 'first_installed_at', 'last_upgrade_at') LIMIT 1"
+      ).get();
+      if (row) return true;
+    }
+  } catch {}
+  return false;
+}
+
 function getDb() {
   if (db) return db;
   throw new Error('Database not initialized. Call initDb() first.');
@@ -971,6 +1037,22 @@ function createTables() {
       input_tokens INTEGER,
       output_tokens INTEGER,
       gen_tok_per_sec REAL,
+      sample_id TEXT,
+      dataset_version TEXT,
+      dataset_hash TEXT,
+      prompt_hash TEXT,
+      system_prompt_hash TEXT,
+      repo_sha TEXT,
+      scorer_version TEXT,
+      evaluator_version TEXT,
+      scoring_method TEXT,
+      run_config_json TEXT,
+      eval_manifest_json TEXT,
+      artifact_path TEXT,
+      model_snapshot TEXT,
+      temperature REAL,
+      seed TEXT,
+      trusted INTEGER DEFAULT 0,
       created_at TEXT DEFAULT (datetime('now'))
     );
     CREATE INDEX IF NOT EXISTS idx_eval_bench_run ON eval_benchmark_runs(run_id);
@@ -2889,39 +2971,131 @@ function listModelDefaults() {
 
 // -- Benchmark Evaluations --
 
+function _benchmarkHash(value) {
+  return crypto.createHash('sha256').update(String(value ?? '')).digest('hex');
+}
+
+function _safeJsonString(value) {
+  if (value == null) return null;
+  if (typeof value === 'string') return value;
+  try { return JSON.stringify(value); } catch { return null; }
+}
+
+function _inferBenchmarkTrust(entry = {}) {
+  if (entry.trusted !== undefined) return entry.trusted ? 1 : 0;
+  if (entry.error) return 0;
+  if (entry.compositeScore == null && entry.composite_score == null) return 0;
+  const method = String(entry.scoringMethod || entry.scoring_method || '').toLowerCase();
+  if (
+    method.includes('judge') ||
+    method.includes('executable') ||
+    method.includes('official') ||
+    method.includes('agent-rubric') ||
+    method.includes('tests')
+  ) return 1;
+  return 0;
+}
+
+function _normalizeBenchmarkResult(entry = {}) {
+  const runId = entry.runId || entry.run_id || 'manual';
+  const suite = entry.suite || 'unknown';
+  const benchmarkId = entry.promptId || entry.benchmark_id || entry.sampleId || entry.sample_id || 'unknown';
+  const prompt = entry.prompt || '';
+  const sampleId = entry.sampleId || entry.sample_id || benchmarkId;
+  const datasetVersion = entry.datasetVersion || entry.dataset_version || `${suite}:local-v1`;
+  const datasetHash = entry.datasetHash || entry.dataset_hash || _benchmarkHash(JSON.stringify({
+    suite,
+    datasetVersion,
+    sampleId,
+    prompt,
+  }));
+  const scoringMethod = entry.scoringMethod || entry.scoring_method || null;
+  const scorerVersion = entry.scorerVersion || entry.scorer_version || 'wall-e-eval-v1-legacy';
+  const evaluatorVersion = entry.evaluatorVersion || entry.evaluator_version || 'wall-e-evaluator-v1';
+  const runConfigJson = entry.runConfigJson || entry.run_config_json || _safeJsonString(entry.runConfig);
+  const evalManifestJson = entry.evalManifestJson || entry.eval_manifest_json || _safeJsonString({
+    runId,
+    suite,
+    datasetVersion,
+    datasetHash,
+    sampleId,
+    promptHash: entry.promptHash || entry.prompt_hash || _benchmarkHash(prompt),
+    provider: entry.provider || null,
+    model: entry.model || null,
+    modelSnapshot: entry.modelSnapshot || entry.model_snapshot || entry.model || null,
+    scorerVersion,
+    evaluatorVersion,
+    scoringMethod,
+    repoSha: entry.repoSha || entry.repo_sha || null,
+    runConfig: runConfigJson ? (() => { try { return JSON.parse(runConfigJson); } catch { return null; } })() : null,
+    artifactPath: entry.artifactPath || entry.artifact_path || null,
+    trusted: !!_inferBenchmarkTrust(entry),
+  });
+
+  return {
+    ...entry,
+    runId,
+    suite,
+    promptId: benchmarkId,
+    prompt,
+    sampleId,
+    datasetVersion,
+    datasetHash,
+    promptHash: entry.promptHash || entry.prompt_hash || _benchmarkHash(prompt),
+    systemPromptHash: entry.systemPromptHash || entry.system_prompt_hash || null,
+    repoSha: entry.repoSha || entry.repo_sha || null,
+    scorerVersion,
+    evaluatorVersion,
+    scoringMethod,
+    runConfigJson,
+    evalManifestJson,
+    artifactPath: entry.artifactPath || entry.artifact_path || null,
+    modelSnapshot: entry.modelSnapshot || entry.model_snapshot || entry.model || null,
+    temperature: entry.temperature ?? null,
+    seed: entry.seed == null ? null : String(entry.seed),
+    trusted: _inferBenchmarkTrust(entry),
+  };
+}
+
 function isInvalidBenchmarkResult(entry = {}) {
-  const error = typeof entry.error === 'string' ? entry.error.trim() : entry.error;
   const provider = typeof entry.provider === 'string' ? entry.provider.trim() : entry.provider;
   const model = typeof entry.model === 'string' ? entry.model.trim() : entry.model;
-  return !!error || !provider || !model || provider === 'unknown' || model === 'unknown';
+  return !provider || !model || provider === 'unknown' || model === 'unknown';
 }
 
 function insertBenchmarkResult(entry) {
-  if (isInvalidBenchmarkResult(entry)) return null;
+  const normalized = _normalizeBenchmarkResult(entry);
+  if (isInvalidBenchmarkResult(normalized)) return null;
 
-  const id = entry.id || require('crypto').randomUUID();
+  const id = normalized.id || crypto.randomUUID();
   getDb().prepare(`
     INSERT OR REPLACE INTO eval_benchmark_runs
     (id, run_id, suite, benchmark_id, provider, model, prompt, response, trait_score, judge_score, composite_score, latency_ms, cost_estimate, error,
-     cost_dollars, tests_before, tests_after, total_tests, dimensions_json, model_metadata_json, input_tokens, output_tokens, gen_tok_per_sec)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+     cost_dollars, tests_before, tests_after, total_tests, dimensions_json, model_metadata_json, input_tokens, output_tokens, gen_tok_per_sec,
+     sample_id, dataset_version, dataset_hash, prompt_hash, system_prompt_hash, repo_sha, scorer_version, evaluator_version, scoring_method,
+     run_config_json, eval_manifest_json, artifact_path, model_snapshot, temperature, seed, trusted)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   `).run(
-    id, entry.runId, entry.suite, entry.promptId || entry.benchmark_id,
-    entry.provider, entry.model, entry.prompt,
-    entry.response || null, entry.traitScore ?? null, entry.judgeScore ?? null,
-    entry.compositeScore ?? null, entry.latencyMs || null, entry.costEstimate || null,
-    entry.error || null,
-    entry.costDollars ?? null, entry.testsBefore ?? null, entry.testsAfter ?? null,
-    entry.totalTests ?? null, entry.dimensionsJson || null, entry.modelMetadataJson || null,
-    entry.inputTokens ?? null, entry.outputTokens ?? null, entry.genTokPerSec ?? null
+    id, normalized.runId, normalized.suite, normalized.promptId,
+    normalized.provider, normalized.model, normalized.prompt,
+    normalized.response || null, normalized.traitScore ?? null, normalized.judgeScore ?? null,
+    normalized.compositeScore ?? null, normalized.latencyMs || null, normalized.costEstimate || null,
+    normalized.error || null,
+    normalized.costDollars ?? null, normalized.testsBefore ?? null, normalized.testsAfter ?? null,
+    normalized.totalTests ?? null, normalized.dimensionsJson || null, normalized.modelMetadataJson || null,
+    normalized.inputTokens ?? null, normalized.outputTokens ?? null, normalized.genTokPerSec ?? null,
+    normalized.sampleId || null, normalized.datasetVersion || null, normalized.datasetHash || null,
+    normalized.promptHash || null, normalized.systemPromptHash || null, normalized.repoSha || null,
+    normalized.scorerVersion || null, normalized.evaluatorVersion || null, normalized.scoringMethod || null,
+    normalized.runConfigJson || null, normalized.evalManifestJson || null, normalized.artifactPath || null,
+    normalized.modelSnapshot || null, normalized.temperature ?? null, normalized.seed ?? null, normalized.trusted ? 1 : 0
   );
   return id;
 }
 
 function purgeInvalidBenchmarkResults({ dryRun = false } = {}) {
   const where = `
-    (error IS NOT NULL AND trim(error) != '')
-    OR model IS NULL OR trim(model) = '' OR model = 'unknown'
+    model IS NULL OR trim(model) = '' OR model = 'unknown'
     OR provider IS NULL OR trim(provider) = '' OR provider = 'unknown'
   `;
   const matched = getDb().prepare(`SELECT COUNT(*) AS count FROM eval_benchmark_runs WHERE ${where}`).get();
@@ -2940,7 +3114,7 @@ function getBenchmarkResults({ suite, days } = {}) {
 }
 
 function getBenchmarkLeaderboard({ suite, days } = {}) {
-  let where = "model != 'unknown'";
+  let where = "model != 'unknown' AND provider != 'unknown'";
   const params = [];
   if (suite) { where += ' AND suite = ?'; params.push(suite); }
   if (days) { where += " AND created_at >= datetime('now', ?)"; params.push(`-${days} days`); }
@@ -2948,15 +3122,21 @@ function getBenchmarkLeaderboard({ suite, days } = {}) {
     SELECT provider, model,
       AVG(CASE WHEN error IS NULL THEN composite_score END) AS avg_score,
       AVG(composite_score) AS avg_score_all,
+      AVG(CASE WHEN error IS NULL AND trusted = 1 THEN composite_score END) AS trusted_avg_score,
       COUNT(*) AS total_evals,
       SUM(CASE WHEN error IS NULL THEN 1 ELSE 0 END) AS successful_evals,
       SUM(CASE WHEN error IS NOT NULL THEN 1 ELSE 0 END) AS errors,
+      SUM(CASE WHEN error IS NULL AND trusted = 1 THEN 1 ELSE 0 END) AS trusted_evals,
+      COUNT(DISTINCT COALESCE(sample_id, benchmark_id)) AS sample_count,
       AVG(CASE WHEN error IS NULL THEN latency_ms END) AS avg_latency_ms,
-      SUM(COALESCE(cost_dollars, cost_estimate, 0)) AS total_cost
+      SUM(COALESCE(cost_dollars, cost_estimate, 0)) AS total_cost,
+      GROUP_CONCAT(DISTINCT dataset_version) AS dataset_versions,
+      GROUP_CONCAT(DISTINCT scorer_version) AS scorer_versions,
+      MAX(created_at) AS last_eval_at
     FROM eval_benchmark_runs
     WHERE ${where}
     GROUP BY provider, model
-    ORDER BY COALESCE(avg_score, 0) DESC, successful_evals DESC
+    ORDER BY COALESCE(trusted_avg_score, avg_score, 0) DESC, trusted_evals DESC, successful_evals DESC
   `).all(...params);
 }
 
@@ -2969,7 +3149,8 @@ function getBenchmarkLeaderboard({ suite, days } = {}) {
  *   efficiency = costEfficiency (single dimension)
  */
 function getBenchmarkLeaderboardWithDimensions({ suite, days } = {}) {
-  let where = "model != 'unknown'";
+  const MIN_TRUSTED_EVALS = 10;
+  let where = "model != 'unknown' AND provider != 'unknown'";
   const params = [];
   if (suite) { where += ' AND suite = ?'; params.push(suite); }
   if (days) { where += " AND created_at >= datetime('now', ?)"; params.push(`-${days} days`); }
@@ -2977,7 +3158,9 @@ function getBenchmarkLeaderboardWithDimensions({ suite, days } = {}) {
   const rows = getDb().prepare(`
     SELECT provider, model, composite_score, dimensions_json,
       latency_ms, cost_estimate, cost_dollars, error, created_at,
-      input_tokens, output_tokens, gen_tok_per_sec
+      input_tokens, output_tokens, gen_tok_per_sec,
+      benchmark_id, sample_id, dataset_version, scorer_version, evaluator_version,
+      scoring_method, repo_sha, model_snapshot, trusted, artifact_path
     FROM eval_benchmark_runs
     WHERE ${where}
   `).all(...params);
@@ -2999,14 +3182,40 @@ function getBenchmarkLeaderboardWithDimensions({ suite, days } = {}) {
     'costEfficiency',
   ];
 
+  function finiteScores(rowsForStats) {
+    return rowsForStats
+      .map(r => Number(r.composite_score))
+      .filter(n => Number.isFinite(n));
+  }
+
+  function meanAndCi(scores) {
+    if (!scores.length) return { mean: 0, low: null, high: null, stdev: null };
+    const mean = scores.reduce((s, n) => s + n, 0) / scores.length;
+    if (scores.length < 2) return { mean, low: mean, high: mean, stdev: 0 };
+    const variance = scores.reduce((s, n) => s + Math.pow(n - mean, 2), 0) / (scores.length - 1);
+    const stdev = Math.sqrt(variance);
+    const margin = 1.96 * (stdev / Math.sqrt(scores.length));
+    return {
+      mean,
+      low: Math.max(0, mean - margin),
+      high: Math.min(1, mean + margin),
+      stdev,
+    };
+  }
+
+  function uniq(values) {
+    return [...new Set(values.filter(Boolean))];
+  }
+
   const results = [];
   for (const g of Object.values(groups)) {
     const n = g.rows.length;
     const successRows = g.rows.filter(r => !r.error);
+    const trustedRows = successRows.filter(r => Number(r.trusted) === 1);
     const qualityRows = successRows;
-    const avgComposite = qualityRows.length > 0
-      ? qualityRows.reduce((s, r) => s + (r.composite_score || 0), 0) / qualityRows.length
-      : 0;
+    const scoreStats = meanAndCi(finiteScores(qualityRows));
+    const trustedStats = meanAndCi(finiteScores(trustedRows));
+    const avgComposite = scoreStats.mean;
     const avgCompositeAll = g.rows.reduce((s, r) => s + (r.composite_score || 0), 0) / n;
     const errors = g.rows.filter(r => r.error).length;
     const avgLatency = qualityRows.length > 0
@@ -3067,15 +3276,35 @@ function getBenchmarkLeaderboardWithDimensions({ suite, days } = {}) {
       model: g.model,
       avg_score: avgComposite,
       avg_score_all: avgCompositeAll,
+      score_confidence_low: scoreStats.low,
+      score_confidence_high: scoreStats.high,
+      score_stdev: scoreStats.stdev,
+      trusted_avg_score: trustedRows.length ? trustedStats.mean : null,
+      trusted_score_confidence_low: trustedRows.length ? trustedStats.low : null,
+      trusted_score_confidence_high: trustedRows.length ? trustedStats.high : null,
       total_evals: n,
       successful_evals: successRows.length,
+      trusted_evals: trustedRows.length,
+      min_trusted_evals: MIN_TRUSTED_EVALS,
+      min_n_met: trustedRows.length >= MIN_TRUSTED_EVALS,
+      trust_status: trustedRows.length >= MIN_TRUSTED_EVALS ? 'trusted' : trustedRows.length > 0 ? 'provisional' : 'legacy',
+      sample_count: uniq(g.rows.map(r => r.sample_id || r.benchmark_id)).length,
       dim_evals: dimCount,
       errors,
       error_rate: n > 0 ? errors / n : 0,
+      success_rate: n > 0 ? successRows.length / n : 0,
       avg_latency_ms: avgLatency,
       total_cost: totalCost,
       dimensions,
       categories,
+      dataset_versions: uniq(g.rows.map(r => r.dataset_version)),
+      scorer_versions: uniq(g.rows.map(r => r.scorer_version)),
+      evaluator_versions: uniq(g.rows.map(r => r.evaluator_version)),
+      scoring_methods: uniq(g.rows.map(r => r.scoring_method)),
+      repo_shas: uniq(g.rows.map(r => r.repo_sha)),
+      model_snapshots: uniq(g.rows.map(r => r.model_snapshot)),
+      artifact_count: g.rows.filter(r => !!r.artifact_path).length,
+      last_eval_at: g.rows.reduce((max, r) => !max || r.created_at > max ? r.created_at : max, null),
       avg_input_tokens: avgInputTokens,
       avg_output_tokens: avgOutputTokens,
       avg_tok_per_sec: avgTokPerSec,
@@ -3084,7 +3313,9 @@ function getBenchmarkLeaderboardWithDimensions({ suite, days } = {}) {
     });
   }
 
-  results.sort((a, b) => b.avg_score - a.avg_score);
+  results.sort((a, b) => (
+    (b.trusted_avg_score ?? b.avg_score ?? 0) - (a.trusted_avg_score ?? a.avg_score ?? 0)
+  ));
   return results;
 }
 
@@ -3142,6 +3373,11 @@ function getBenchmarkRuns({ limit: lim = 20 } = {}) {
     SELECT run_id, suite, COUNT(*) AS total_prompts,
       COUNT(DISTINCT provider || '/' || model) AS providers,
       AVG(composite_score) AS avg_score,
+      SUM(CASE WHEN error IS NULL THEN 1 ELSE 0 END) AS successful_prompts,
+      SUM(CASE WHEN error IS NOT NULL THEN 1 ELSE 0 END) AS errors,
+      SUM(CASE WHEN error IS NULL AND trusted = 1 THEN 1 ELSE 0 END) AS trusted_prompts,
+      GROUP_CONCAT(DISTINCT dataset_version) AS dataset_versions,
+      GROUP_CONCAT(DISTINCT scorer_version) AS scorer_versions,
       MIN(created_at) AS started_at,
       MAX(created_at) AS ended_at
     FROM eval_benchmark_runs
@@ -3635,6 +3871,7 @@ module.exports = {
   setProviderAuthMethod,
   getProviderAuthMethod,
   VALID_AUTH_METHODS,
+  SCHEMA_VERSION,
   deleteModelProvider,
   deleteModelRegistryEntry,
   deleteModelRegistryByProvider,
@@ -3702,4 +3939,5 @@ module.exports = {
   insertMemoryIndex, searchMemoryIndex, getMemoryIndex,
   // Memory Lifecycle Hooks
   on, off, once,
+  logWrite,
 };

package/template/wall-e/channels/policy.js

@@ -0,0 +1,88 @@
+'use strict';
+
+const crypto = require('node:crypto');
+
+function normalizePrincipal(input = {}) {
+  return {
+    channel: input.channel || input.channelId || 'unknown',
+    userId: String(input.userId || input.senderId || input.from || ''),
+    threadId: String(input.threadId || input.conversationId || input.roomId || ''),
+    groupId: String(input.groupId || input.roomId || ''),
+    isGroup: Boolean(input.isGroup || input.groupId || input.roomId),
+    command: input.command || '',
+  };
+}
+
+function makePrincipalKey(principal = {}) {
+  const p = normalizePrincipal(principal);
+  return [p.channel, p.isGroup ? 'group' : 'dm', p.groupId || p.threadId || p.userId].join(':');
+}
+
+function evaluateDirectMessageAccess(input = {}) {
+  const principal = normalizePrincipal(input);
+  const policy = input.policy || {};
+  if (policy.enabled === false) return decision('block', 'dm-policy-disabled', principal);
+  const allowlist = normalizeList(policy.allowUsers || policy.allowlist || []);
+  const pairedUsers = new Set(normalizeList(input.pairedUsers || []));
+  if (allowlist.length === 0 && pairedUsers.size === 0) return decision('allow', 'dm-open', principal);
+  if (allowlist.includes(principal.userId)) return decision('allow', 'dm-user-allowlisted', principal);
+  if (pairedUsers.has(principal.userId)) return decision('allow', 'dm-user-paired', principal);
+  if (policy.pairingRequired) return decision('pairing_required', 'dm-pairing-required', principal);
+  return decision('block', 'dm-user-not-allowlisted', principal);
+}
+
+function evaluateGroupAccess(input = {}) {
+  const principal = normalizePrincipal({ ...input, isGroup: true });
+  const policy = input.policy || {};
+  if (policy.enabled === false) return decision('block', 'group-policy-disabled', principal);
+  const allowGroups = normalizeList(policy.allowGroups || policy.groupAllowlist || []);
+  const allowUsers = normalizeList(policy.allowUsers || policy.senderAllowlist || []);
+  if (allowGroups.length === 0 && allowUsers.length === 0) {
+    return decision(policy.defaultOpen === false ? 'block' : 'allow', policy.defaultOpen === false ? 'group-default-closed' : 'group-open', principal);
+  }
+  if (allowGroups.includes(principal.groupId || principal.threadId)) return decision('allow', 'group-allowlisted', principal);
+  if (allowUsers.includes(principal.userId)) return decision('allow', 'group-sender-allowlisted', principal);
+  return decision('block', 'group-not-allowlisted', principal);
+}
+
+function createPairingChallenge({ channel, userId, ttlMs = 5 * 60 * 1000, now = Date.now } = {}) {
+  const nonce = crypto.randomBytes(8).toString('hex');
+  return {
+    channel: channel || 'unknown',
+    userId: String(userId || ''),
+    nonce,
+    code: nonce.slice(0, 6).toUpperCase(),
+    expiresAt: now() + ttlMs,
+  };
+}
+
+function verifyPairingChallenge(challenge, code, { now = Date.now } = {}) {
+  if (!challenge) return { ok: false, reason: 'missing-challenge' };
+  if (now() > challenge.expiresAt) return { ok: false, reason: 'challenge-expired' };
+  if (String(code || '').trim().toUpperCase() !== challenge.code) return { ok: false, reason: 'code-mismatch' };
+  return { ok: true, userId: challenge.userId, channel: challenge.channel };
+}
+
+function decision(status, reason, principal) {
+  return {
+    status,
+    allow: status === 'allow',
+    reason,
+    principal,
+  };
+}
+
+function normalizeList(values) {
+  return (Array.isArray(values) ? values : [values])
+    .map(value => String(value || '').trim())
+    .filter(Boolean);
+}
+
+module.exports = {
+  createPairingChallenge,
+  evaluateDirectMessageAccess,
+  evaluateGroupAccess,
+  makePrincipalKey,
+  normalizePrincipal,
+  verifyPairingChallenge,
+};

package/template/wall-e/channels/registry.js

@@ -4,6 +4,7 @@ const path = require('path');
 const { Registry } = require('../plugins/registry-base');
 const { loadSiblingManifestsFromDir } = require('../plugins/manifest-loader');
 const { addDiagnostics, addDiagnostic } = require('../plugins/diagnostics');
+const { evaluateDirectMessageAccess, evaluateGroupAccess } = require('./policy');
 
 /**
  * Channel registry — replaces hardcoded require list in agent.js:10-12.
@@ -79,7 +80,18 @@ function instantiate(id, ctorOpts) {
   const entry = _registry.get(id);
   if (!entry || typeof entry.ChannelClass !== 'function') return null;
   try {
-    return new entry.ChannelClass(ctorOpts || {});
+    const opts = { ...(ctorOpts || {}) };
+    if (!opts.channelPolicy) {
+      Object.defineProperty(opts, 'channelPolicy', {
+        enumerable: false,
+        configurable: true,
+        value: {
+          evaluateDirectMessageAccess,
+          evaluateGroupAccess,
+        },
+      });
+    }
+    return new entry.ChannelClass(opts);
   } catch (err) {
     addDiagnostic({
       level: 'error',
@@ -229,5 +241,7 @@ module.exports = {
   ensureBootstrapped,
   isEnabledByConfig,
   configRequirementsSatisfied,
+  evaluateDirectMessageAccess,
+  evaluateGroupAccess,
   _resetForTest,
 };

package/template/wall-e/channels/reply-dispatcher.js

@@ -0,0 +1,70 @@
+'use strict';
+
+class ReplyDispatcher {
+  constructor({ send, now = () => new Date().toISOString() } = {}) {
+    if (typeof send !== 'function') throw new Error('ReplyDispatcher requires send function');
+    this.send = send;
+    this.now = now;
+    this.buffer = [];
+    this.seenDeliveryIds = new Set();
+    this.statuses = [];
+  }
+
+  async dispatch(reply = {}) {
+    const deliveryId = reply.deliveryId || `${reply.channel || 'channel'}:${reply.target || ''}:${hash(reply.message || '')}`;
+    if (this.seenDeliveryIds.has(deliveryId)) {
+      return this._status(deliveryId, 'skipped', 'duplicate');
+    }
+    this.seenDeliveryIds.add(deliveryId);
+    if (reply.mode === 'buffered') {
+      this.buffer.push({ ...reply, deliveryId });
+      return this._status(deliveryId, 'buffered', '');
+    }
+    return this._sendNow({ ...reply, deliveryId });
+  }
+
+  async flush() {
+    const pending = this.buffer.splice(0);
+    const results = [];
+    for (const reply of pending) {
+      results.push(await this._sendNow(reply));
+    }
+    return results;
+  }
+
+  listStatuses() {
+    return [...this.statuses];
+  }
+
+  async _sendNow(reply) {
+    try {
+      const result = await this.send(reply);
+      return this._status(reply.deliveryId, 'sent', '', result);
+    } catch (err) {
+      return this._status(reply.deliveryId, 'failed', err.message);
+    }
+  }
+
+  _status(deliveryId, status, reason = '', result = null) {
+    const record = {
+      deliveryId,
+      status,
+      reason,
+      result,
+      timestamp: this.now(),
+    };
+    this.statuses.push(record);
+    return record;
+  }
+}
+
+function hash(text) {
+  let h = 0;
+  const input = String(text || '');
+  for (let i = 0; i < input.length; i += 1) h = ((h << 5) - h + input.charCodeAt(i)) | 0;
+  return Math.abs(h).toString(36);
+}
+
+module.exports = {
+  ReplyDispatcher,
+};