create-walle 0.9.25 → 0.9.26

package/template/wall-e/training/replay-eval-analysis.js

@@ -0,0 +1,151 @@
+'use strict';
+
+function summarizeActionMemoryReplayEval(rows = [], {
+  maxBadCopies = 0,
+  maxRegressions = 0,
+  minNetPassDelta = 1,
+  allowSameRowCopies = false,
+} = {}) {
+  const normalizedRows = Array.isArray(rows) ? rows : [];
+  const counters = {
+    total: normalizedRows.length,
+    baselinePass: 0,
+    finalPass: 0,
+    rescuedCount: 0,
+    regressionCount: 0,
+    acceptedSelectionCount: 0,
+    selectedExactCount: 0,
+    selectedTargetCount: 0,
+    badCopyCount: 0,
+    sameRowCopyCount: 0,
+    byLane: {},
+    byTool: {},
+  };
+
+  for (const row of normalizedRows) {
+    const baselinePass = Boolean(row.baselineScore?.pass);
+    const finalPass = Boolean(row.score?.pass);
+    const selection = row.actionMemorySelection || null;
+    const oracle = row.actionMemoryOracle || null;
+    const accepted = Boolean(selection?.accepted);
+    const selectedExact = Boolean(oracle?.selectedExact);
+    const selectedTarget = Boolean(oracle?.selectedTarget);
+    const sameRowCopy = Boolean(accepted && selection?.sourceRowId && row.id && selection.sourceRowId === row.id);
+    const badCopy = Boolean(accepted && oracle && !selectedTarget);
+    const rescued = Boolean(!baselinePass && finalPass);
+    const regression = Boolean(baselinePass && !finalPass);
+
+    if (baselinePass) counters.baselinePass += 1;
+    if (finalPass) counters.finalPass += 1;
+    if (rescued) counters.rescuedCount += 1;
+    if (regression) counters.regressionCount += 1;
+    if (accepted) counters.acceptedSelectionCount += 1;
+    if (selectedExact) counters.selectedExactCount += 1;
+    if (selectedTarget) counters.selectedTargetCount += 1;
+    if (badCopy) counters.badCopyCount += 1;
+    if (sameRowCopy) counters.sameRowCopyCount += 1;
+
+    incrementGroup(counters.byLane, row.lane || 'unknown', {
+      baselinePass,
+      finalPass,
+      rescued,
+      regression,
+      accepted,
+      badCopy,
+      sameRowCopy,
+    });
+    incrementGroup(counters.byTool, row.expected?.tool || 'unknown', {
+      baselinePass,
+      finalPass,
+      rescued,
+      regression,
+      accepted,
+      badCopy,
+      sameRowCopy,
+    });
+  }
+
+  counters.baselinePassRate = rate(counters.baselinePass, counters.total);
+  counters.finalPassRate = rate(counters.finalPass, counters.total);
+  counters.acceptedSelectionRate = rate(counters.acceptedSelectionCount, counters.total);
+  counters.selectedExactRate = rate(counters.selectedExactCount, counters.acceptedSelectionCount);
+  counters.selectedTargetRate = rate(counters.selectedTargetCount, counters.acceptedSelectionCount);
+  counters.badCopyRate = rate(counters.badCopyCount, counters.acceptedSelectionCount);
+  counters.sameRowCopyRate = rate(counters.sameRowCopyCount, counters.acceptedSelectionCount);
+  counters.netPassDelta = counters.finalPass - counters.baselinePass;
+  counters.promotionGate = buildPromotionGate(counters, {
+    maxBadCopies,
+    maxRegressions,
+    minNetPassDelta,
+    allowSameRowCopies,
+  });
+  return counters;
+}
+
+function incrementGroup(groups, key, values) {
+  if (!groups[key]) {
+    groups[key] = {
+      total: 0,
+      baselinePass: 0,
+      finalPass: 0,
+      rescued: 0,
+      regressions: 0,
+      acceptedSelections: 0,
+      badCopies: 0,
+      sameRowCopies: 0,
+    };
+  }
+  const group = groups[key];
+  group.total += 1;
+  if (values.baselinePass) group.baselinePass += 1;
+  if (values.finalPass) group.finalPass += 1;
+  if (values.rescued) group.rescued += 1;
+  if (values.regression) group.regressions += 1;
+  if (values.accepted) group.acceptedSelections += 1;
+  if (values.badCopy) group.badCopies += 1;
+  if (values.sameRowCopy) group.sameRowCopies += 1;
+}
+
+function buildPromotionGate(report, {
+  maxBadCopies = 0,
+  maxRegressions = 0,
+  minNetPassDelta = 1,
+  allowSameRowCopies = false,
+} = {}) {
+  const checks = [
+    {
+      name: 'netPassDelta',
+      actual: report.netPassDelta,
+      minimum: minNetPassDelta,
+      ok: report.netPassDelta >= minNetPassDelta,
+    },
+    {
+      name: 'badCopyCount',
+      actual: report.badCopyCount,
+      maximum: maxBadCopies,
+      ok: report.badCopyCount <= maxBadCopies,
+    },
+    {
+      name: 'regressionCount',
+      actual: report.regressionCount,
+      maximum: maxRegressions,
+      ok: report.regressionCount <= maxRegressions,
+    },
+    {
+      name: 'sameRowCopyCount',
+      actual: report.sameRowCopyCount,
+      maximum: allowSameRowCopies ? null : 0,
+      ok: allowSameRowCopies || report.sameRowCopyCount === 0,
+    },
+  ];
+  const failed = checks.filter((check) => !check.ok);
+  return { ok: failed.length === 0, checks, failed };
+}
+
+function rate(numerator, denominator) {
+  return denominator ? Number((numerator / denominator).toFixed(4)) : 0;
+}
+
+module.exports = {
+  summarizeActionMemoryReplayEval,
+};

package/template/wall-e/training/run-shell-command-selector.js

@@ -0,0 +1,277 @@
+'use strict';
+
+const COMMAND_JSON_RE = /"command"\s*:\s*"((?:\\.|[^"\\])*)"/g;
+
+function buildRunShellCommandCandidates(example = {}, {
+  maxCandidates = 12,
+  actionMemoryStore = null,
+  memoryLimit = 50,
+} = {}) {
+  const candidates = [];
+  const add = (command, source, reason = null, extra = {}) => {
+    const value = normalizeCommand(command);
+    if (!value) return;
+    if (candidates.some((candidate) => candidate.command === value)) return;
+    candidates.push({
+      id: `cmd_${candidates.length + 1}`,
+      command: value,
+      source,
+      reason,
+      ...extra,
+    });
+  };
+
+  for (const command of example.labels?.verification_commands || []) add(command, 'task_memory', 'trajectory_verification');
+
+  const userContent = (example.messages || [])
+    .filter((message) => message.role === 'user')
+    .map((message) => message.content || '')
+    .join('\n');
+  for (const command of parseRememberedCommands(userContent, 'remembered task commands')) add(command, 'prompt_task_memory', 'prompt_memory');
+  for (const command of parseRememberedCommands(userContent, 'remembered workflow commands')) add(command, 'prompt_workflow_memory', 'prompt_memory');
+  for (const command of extractJsonCommandStrings(userContent)) add(command, 'prompt_state', 'json_command');
+
+  for (const message of example.messages || []) {
+    if (message.role !== 'assistant' || !Array.isArray(message.tool_calls)) continue;
+    for (const call of message.tool_calls) {
+      const name = call.function?.name || call.name;
+      if (name !== 'run_shell') continue;
+      const args = parseToolArguments(call.function?.arguments || call.arguments || call.input || {});
+      add(args.command, 'history', 'assistant_tool_call');
+    }
+  }
+
+  for (const candidate of loadRunShellActionMemoryCandidates(example, { actionMemoryStore, limit: memoryLimit })) {
+    add(
+      candidate.input?.command || candidate.input?.cmd,
+      'action_memory',
+      'action_memory_candidate',
+      {
+        memoryEntryId: candidate.id,
+        score: candidate.score,
+        memoryReasons: candidate.reasons || [],
+        sourceRowId: candidate.sourceRowId,
+        sourceSessionId: candidate.sourceSessionId,
+        sourceToolCallId: candidate.sourceToolCallId,
+      }
+    );
+  }
+
+  return candidates.slice(0, maxCandidates);
+}
+
+function selectRunShellCommandCandidate(example = {}, candidates = []) {
+  const values = candidates.map((candidate) => candidate.command).filter(Boolean);
+  if (!values.length) return null;
+
+  const recent = extractRecentShellCommands(example);
+  const successfulRecent = new Set(recent
+    .filter((item) => /tool_result|passed|success/i.test(item.status || ''))
+    .map((item) => normalizeCommand(item.command)));
+  const lastRecent = recent.at(-1)?.command || null;
+  const lastIndex = values.findIndex((command) => commandsEquivalent(command, lastRecent));
+
+  if (lastIndex >= 0 && lastIndex + 1 < values.length) {
+    const next = values.slice(lastIndex + 1).find((command) => !successfulRecent.has(normalizeCommand(command)));
+    if (next) return annotateSelection(candidates, next, 'after_last_recent_command');
+  }
+
+  if (hasOpaqueRunShellFailure(example) && values.length > 1) {
+    if (candidates[0]?.source === 'action_memory') {
+      return annotateSelection(candidates, values[0], 'top_action_memory_after_opaque_shell_failure');
+    }
+    return annotateSelection(candidates, values[1], 'after_opaque_shell_failure');
+  }
+
+  const firstUnrun = values.find((command) => !successfulRecent.has(normalizeCommand(command)));
+  if (firstUnrun) return annotateSelection(candidates, firstUnrun, 'first_unrun_memory_command');
+
+  return annotateSelection(candidates, values[0], 'first_memory_command');
+}
+
+function scoreRunShellCommandCandidates(expectedCommand = '', candidates = [], selection = null) {
+  const expected = normalizeCommand(expectedCommand);
+  const index = candidates.findIndex((candidate) => commandsEquivalent(candidate.command, expected));
+  const selectedIndex = selection?.command
+    ? candidates.findIndex((candidate) => commandsEquivalent(candidate.command, selection.command))
+    : -1;
+  return {
+    expectedPresent: index >= 0,
+    expectedRank: index >= 0 ? index + 1 : null,
+    expectedTop1: index === 0,
+    expectedTop3: index >= 0 && index < 3,
+    selectedRank: selectedIndex >= 0 ? selectedIndex + 1 : null,
+    selectedExpected: index >= 0 && selectedIndex === index,
+    candidateCount: candidates.length,
+  };
+}
+
+function applyRunShellCommandSelection(example = {}, prediction = null, {
+  maxCandidates = 12,
+  actionMemoryStore = null,
+  memoryLimit = 50,
+} = {}) {
+  if (!prediction || prediction.name !== 'run_shell') return { prediction, selection: null, candidates: [] };
+  const candidates = buildRunShellCommandCandidates(example, { maxCandidates, actionMemoryStore, memoryLimit });
+  const selection = selectRunShellCommandCandidate(example, candidates);
+  if (!selection?.command) return { prediction, selection, candidates };
+  return {
+    prediction: {
+      ...prediction,
+      input: {
+        ...(prediction.input || {}),
+        command: selection.command,
+      },
+    },
+    selection,
+    candidates,
+  };
+}
+
+function loadRunShellActionMemoryCandidates(example = {}, {
+  actionMemoryStore = null,
+  limit = 50,
+} = {}) {
+  if (!actionMemoryStore || typeof actionMemoryStore.searchActionMemoryCandidates !== 'function') return [];
+  try {
+    return actionMemoryStore.searchActionMemoryCandidates({
+      context: runShellActionContext(example),
+      toolName: 'run_shell',
+      prediction: { name: 'run_shell', input: {} },
+      limit,
+    });
+  } catch {
+    return [];
+  }
+}
+
+function runShellActionContext(example = {}) {
+  const userContent = (example.messages || [])
+    .filter((message) => message.role === 'user')
+    .map((message) => message.content || '')
+    .join('\n');
+  return {
+    familyId: example.labels?.workflow_family_id || null,
+    repo: example.labels?.workflow_repo || null,
+    intent: example.labels?.workflow_intent || null,
+    intendedTool: 'run_shell',
+    sequence: firstLineValue(userContent, 'tool sequence') || null,
+    lastTool: firstLineValue(userContent, 'last tool')?.split(/\s+/)[0] || null,
+    taskText: userContent.split(/\nCurrent Wall-E state:/)[0].replace(/^Task:\s*/i, '').trim(),
+  };
+}
+
+function extractRecentShellCommands(example = {}) {
+  const out = [];
+  const userContent = (example.messages || [])
+    .filter((message) => message.role === 'user')
+    .map((message) => message.content || '')
+    .join('\n');
+
+  const recentLine = userContent.split('\n').find((line) => /^\s*-\s*recent shell commands:/i.test(line));
+  if (recentLine) {
+    const raw = recentLine.replace(/^\s*-\s*recent shell commands:\s*/i, '');
+    for (const part of raw.split(/,\s+(?=(tool_result|tool_error|verification_failed|timeout|not_run|passed|failed):)/i)) {
+      const match = part.match(/^\s*([a-z_]+):\s*([\s\S]+)$/i);
+      if (match) out.push({ status: match[1], command: normalizeCommand(match[2]) });
+    }
+  }
+
+  const jsonCommands = extractJsonCommandStrings(userContent);
+  for (const command of jsonCommands) {
+    if (!out.some((item) => commandsEquivalent(item.command, command))) out.push({ status: 'unknown', command });
+  }
+  return out.filter((item) => item.command);
+}
+
+function hasOpaqueRunShellFailure(example = {}) {
+  const userContent = (example.messages || [])
+    .filter((message) => message.role === 'user')
+    .map((message) => message.content || '')
+    .join('\n');
+  if (!/known failures:\s*[^-\n]*run_shell tool error/i.test(userContent)) return false;
+  return extractRecentShellCommands(example).length === 0;
+}
+
+function parseRememberedCommands(text = '', label = '') {
+  const line = String(text || '').split('\n').find((item) => item.toLowerCase().includes(`${label.toLowerCase()}:`));
+  if (!line) return [];
+  const value = line.replace(new RegExp(`^\\s*-\\s*${escapeRegExp(label)}:\\s*`, 'i'), '');
+  return splitCommandList(value);
+}
+
+function firstLineValue(text = '', label = '') {
+  const re = new RegExp(`^\\s*-\\s*${escapeRegExp(label)}:\\s*(.+)$`, 'im');
+  const match = String(text || '').match(re);
+  return match ? match[1].trim() : null;
+}
+
+function splitCommandList(value = '') {
+  return String(value || '')
+    .split(/,\s+(?=(?:[a-zA-Z0-9_./~-]+|cd |git |npm |node |rg |sed |swiftc |codesign |find |python|pytest|pnpm|yarn))/)
+    .map(normalizeCommand)
+    .filter(Boolean);
+}
+
+function extractJsonCommandStrings(text = '') {
+  const out = [];
+  for (const match of String(text || '').matchAll(COMMAND_JSON_RE)) {
+    try {
+      out.push(JSON.parse(`"${match[1]}"`));
+    } catch {
+      out.push(match[1]);
+    }
+  }
+  return out.map(normalizeCommand).filter(Boolean);
+}
+
+function annotateSelection(candidates = [], command = '', reason = '') {
+  const candidate = candidates.find((item) => item.command === command) || null;
+  return {
+    id: candidate?.id || null,
+    command,
+    source: candidate?.source || null,
+    reason,
+    memoryEntryId: candidate?.memoryEntryId || null,
+    score: candidate?.score,
+    memoryReasons: candidate?.memoryReasons || [],
+    sourceRowId: candidate?.sourceRowId,
+    sourceSessionId: candidate?.sourceSessionId,
+    sourceToolCallId: candidate?.sourceToolCallId,
+  };
+}
+
+function parseToolArguments(value) {
+  if (!value) return {};
+  if (typeof value === 'object') return value;
+  try {
+    return JSON.parse(String(value));
+  } catch {
+    return {};
+  }
+}
+
+function commandsEquivalent(left, right) {
+  return normalizeCommand(left) === normalizeCommand(right);
+}
+
+function normalizeCommand(command) {
+  return String(command || '').replace(/\r\n/g, '\n').replace(/[ \t]+/g, ' ').trim();
+}
+
+function escapeRegExp(value = '') {
+  return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+module.exports = {
+  applyRunShellCommandSelection,
+  buildRunShellCommandCandidates,
+  commandsEquivalent,
+  extractRecentShellCommands,
+  hasOpaqueRunShellFailure,
+  loadRunShellActionMemoryCandidates,
+  normalizeCommand,
+  runShellActionContext,
+  scoreRunShellCommandCandidates,
+  selectRunShellCommandCandidate,
+};