clast-ai 0.1.2 → 0.2.1

package/bench/ab/onair-concept.json

@@ -0,0 +1,22 @@
+[
+  {
+    "id": "dc-spike",
+    "q": "Before the samples reach the demodulators, something takes out the strong spike in the middle of the spectrum that many radios produce. Which function in the engine does that, and in which file?",
+    "expect": ["cleanSamples", "engine.cpp"]
+  },
+  {
+    "id": "adc-overdrive",
+    "q": "Which function decides whether the analog-to-digital converter is being driven too hard or too weakly, from the measured signal levels? Name the function and the file.",
+    "expect": ["classifyAdc", "gain.cpp"]
+  },
+  {
+    "id": "mute-gate",
+    "q": "Where is the logic that keeps the speaker quiet until a signal is strong enough, with different levels for opening and closing? Name the struct or class and the source file with its update logic.",
+    "expect": ["SquelchCore", "audio_control"]
+  },
+  {
+    "id": "background-level",
+    "q": "Which function works out how strong the background hiss of the band is, so that the automatic gain can be set? Name the function and its file.",
+    "expect": ["estimateFloor", "autogain_measure.cpp"]
+  }
+]

package/bench/ab/onair-deadcode.json

@@ -0,0 +1,7 @@
+[
+  {
+    "id": "dead-code-area",
+    "q": "Find dead code under app/: name five functions defined there that nothing in the repository calls or references, each with its file and line. Only list functions you have verified are unused.",
+    "expect": []
+  }
+]

package/bench/ab/onair-hard.json

@@ -0,0 +1,32 @@
+[
+  {
+    "id": "iqsource-transitive",
+    "q": "List every non-test class that is a direct or indirect subclass of IqSource (follow the inheritance chain to the leaves).",
+    "expect": ["BeamIqSource", "DiversitySource", "SyntheticWideSource", "UsbHackrf", "NativeSource", "AirspySource", "AirspyHfSource", "BladeSource", "FobosSource", "HpsdrSource", "HydraSource", "IcomSource", "KiwiSource", "LimeSource", "NrspStSource", "PlutoSource", "RfnmSource", "RtlSource", "RtlTcpSource", "Rx888Source", "SdrplaySource", "SdrppSource", "SpyServerSource", "TciSource", "UsrpSource", "AudioSource", "HamlibSource", "SoapySource", "HackrfSource", "HackrfAuto", "PacedSource", "FileSource", "SyntheticSource"]
+  },
+  {
+    "id": "classify-adc-core",
+    "q": "Inside core/ (not the app UI), which classes' methods call classifyAdc? Name the class and method for each.",
+    "expect": ["AutoGain", "GainOptimiser", "GainSweep"]
+  },
+  {
+    "id": "vfo-chain",
+    "q": "Give the full call chain by which the per-frame UI draw code ends up calling VfoBank::setChannels: every function from the draw loop down to setChannels, with files.",
+    "expect": ["floatingWindows", "draw_ui.cpp", "vfoSync", "vfo_ui.cpp", "setVfoChannels", "engine.h"]
+  },
+  {
+    "id": "generic-gain-chain",
+    "q": "Which functions call genericGain outside tests, and for the one in the app status bar, which functions call it in turn? Give function names and files.",
+    "expect": ["gainControl", "statusbar.cpp", "GainSweep", "AutoGain", "jni_onair.cpp", "specx_ui.cpp", "draw_ui.cpp"]
+  },
+  {
+    "id": "set-channels-other",
+    "q": "Besides Engine::setVfoChannels, which other code in core calls setChannels on a VfoBank? Give the file and the enclosing functions.",
+    "expect": ["wide_session.cpp", "clearState", "bankPlace"]
+  },
+  {
+    "id": "paced-sources",
+    "q": "Which classes derive from PacedSource, what does PacedSource itself derive from, and in which file are they all defined?",
+    "expect": ["FileSource", "SyntheticSource", "IqSource", "core/src/source/source.cpp"]
+  }
+]

package/bench/ab/onair-impact.json

@@ -0,0 +1,17 @@
+[
+  {
+    "id": "tests-for-symbol",
+    "q": "I am about to change the behaviour of the function genericGain in core/src/source/gain.cpp. No test calls it by name. Which test files exercise it through the functions that call it? List the test files.",
+    "expect": ["test_gain.cpp", "test_agc.cpp"]
+  },
+  {
+    "id": "signature-change",
+    "q": "I want to add a parameter to the function genericGain (core/src/source/gain.cpp). List every file I have to edit so that the project still compiles, including the declaration.",
+    "expect": ["gain.h", "gain.cpp", "statusbar.cpp", "jni_onair.cpp"]
+  },
+  {
+    "id": "dead-code-file",
+    "q": "Which functions defined in core/src/uat/uat_decode.c are dead code, meaning nothing in the repository calls them, directly or only through other dead functions? Name the top-level functions that nothing calls at all.",
+    "expect": ["uat_display_adsb_mdb", "uat_display_uplink_mdb"]
+  }
+]

package/bench/ab/onair-refactors.json

@@ -0,0 +1,18 @@
+[
+  {
+    "id": "rename-generic-gain",
+    "task": true,
+    "q": "Rename the function genericGain to genericGainFor. Update its declaration, its definition and every caller in the whole repository (core, app, android, tests), so that nothing refers to the old name any more.",
+    "expect": ["genericGainFor", "core/include/dect2/source/gain.h", "core/src/source/gain.cpp", "android/native/src/jni_onair.cpp", "app/ui/statusbar.cpp"],
+    "mustNotRemain": "\\bgenericGain\\(",
+    "paths": ["core", "app", "android", "tests", "tools"]
+  },
+  {
+    "id": "adc-advice-param",
+    "task": true,
+    "q": "Add a required second parameter bool brief to adcAdvice. When brief is true it returns only the first sentence of the advice; when false the result is as it is now. Update the declaration, the definition and every caller in the repository; existing callers pass false.",
+    "expect": ["bool brief", "core/include/dect2/source/gain.h", "core/src/source/gain.cpp", "app/ui/statusbar.cpp", "app/dab/dab_ui.cpp", "app/fm/fm_ui.cpp", "app/atsc3/atsc3_ui.cpp", "app/dtmb/dtmb_ui.cpp", "app/isdbt/isdbt_ui.cpp"],
+    "mustNotRemain": "adcAdvice\\([A-Za-z_:]+\\)",
+    "paths": ["core", "app", "android", "tests", "tools"]
+  }
+]

package/bench/ab/onair-survey.json

@@ -0,0 +1,17 @@
+[
+  {
+    "id": "mode-count",
+    "q": "How many receive modes does OnAir support according to its own mode list, and which source file is that list counted from? Give the number.",
+    "expect": ["333", "mode_table.cpp"]
+  },
+  {
+    "id": "receiver-count",
+    "q": "How many distinct receiver classes (a class or struct whose name ends in Receiver) are defined under core/? Give the number and five examples.",
+    "expect": ["199", "189"]
+  },
+  {
+    "id": "radio-drivers",
+    "q": "Which radio hardware has a native driver class in this code base (the classes built on the shared native source base)? List the driver classes and say how many there are.",
+    "expect": ["AirspySource", "BladeSource", "LimeSource", "PlutoSource", "RtlSource", "SdrplaySource", "UsrpSource", "Rx888Source", "20"]
+  }
+]

package/bench/ab/onair-tasks.json

@@ -0,0 +1,20 @@
+[
+  {
+    "id": "silence-source",
+    "task": true,
+    "q": "Add a new built-in IQ source called SilenceSource that delivers all-zero samples at the requested sample rate, paced in real time like the existing file and synthetic sources. Build it on the same base class those two use, put it next to them, and leave a short comment on where the device factory would create it.",
+    "expect": ["class SilenceSource : public PacedSource", "core/src/source/source.cpp", "override"]
+  },
+  {
+    "id": "adc-status-verylow",
+    "task": true,
+    "q": "Add a new ADC status value VeryLow (between NoSignal and Low) for a signal more than 12 dB below the existing low threshold. Update the enum, the classifier, the status name and the advice text so that every function that switches over the status handles it.",
+    "expect": ["VeryLow", "core/include/dect2/source/gain.h", "core/src/source/gain.cpp", "case AdcStatus::VeryLow"]
+  },
+  {
+    "id": "vfo-mode-cw",
+    "task": true,
+    "q": "Add a new multi-VFO channel mode Cw (narrow CW audio, default bandwidth 500 Hz) placed before Count in the mode enum. Update every function in the VFO engine code that maps or switches over the mode (name, default bandwidth, decoder check) so the new mode is handled; demodulate it like Usb for now.",
+    "expect": ["Cw", "core/include/dect2/engine/vfo.h", "core/src/engine/vfo.cpp", "500"]
+  }
+]

package/bench/ab/onair.json

@@ -0,0 +1,52 @@
+[
+  {
+    "id": "iqsource-subclasses",
+    "q": "Which classes derive directly from IqSource (outside tests)? List every one with its file.",
+    "expect": ["BeamIqSource", "DiversitySource", "SyntheticWideSource", "UsbHackrf", "NativeSource", "HackrfSource", "HackrfAuto", "PacedSource", "AudioSource", "HamlibSource", "SoapySource"]
+  },
+  {
+    "id": "prefs-implementers",
+    "q": "The app has an abstract Prefs interface with a pure virtual setChannels. Where is it declared and which structs implement it?",
+    "expect": ["platform.h", "DefaultsPrefs", "FilePrefs", "RxPrefs"]
+  },
+  {
+    "id": "gain-for-total",
+    "q": "Where is gainForTotal implemented, and which code outside tests calls it? Give file paths.",
+    "expect": ["core/src/source/gain.cpp", "jni_onair.cpp"]
+  },
+  {
+    "id": "vfo-default-bandwidth",
+    "q": "Where is vfoDefaultBandwidthHz defined, and where is it called (outside tests)?",
+    "expect": ["core/src/engine/vfo.cpp", "vfo_state.h"]
+  },
+  {
+    "id": "vfo-set-channels",
+    "q": "How do the multi-VFO channels reach VfoBank::setChannels from the UI? Name the Engine method in between and the UI file that calls it.",
+    "expect": ["setVfoChannels", "engine.h", "vfo_ui.cpp"]
+  },
+  {
+    "id": "mode-tunings",
+    "q": "Where is modeTunings() implemented and which non-test code calls it?",
+    "expect": ["modes.cpp", "jni_onair.cpp"]
+  },
+  {
+    "id": "crash-check",
+    "q": "Which function tells whether the previous run of the program crashed, where is it implemented, and where is it called at startup?",
+    "expect": ["lastRunCrashed", "crash_report.cpp", "main.cpp"]
+  },
+  {
+    "id": "generic-gain",
+    "q": "Which non-test files call genericGain, and in which file is it implemented?",
+    "expect": ["core/src/source/gain.cpp", "jni_onair.cpp", "statusbar.cpp"]
+  },
+  {
+    "id": "adc-advice",
+    "q": "Which function produces the one-line advice text for an ADC status, where is it implemented, and which app UI files use it?",
+    "expect": ["adcAdvice", "gain.cpp", "atsc3_ui.cpp", "dab_ui.cpp", "dtmb_ui.cpp", "fm_ui.cpp", "isdbt_ui.cpp", "statusbar.cpp"]
+  },
+  {
+    "id": "classify-adc",
+    "q": "Where is classifyAdc implemented, what are the possible AdcStatus values, and roughly how many app UI files call it?",
+    "expect": ["gain.cpp", "NoSignal", "Overload", "Good"]
+  }
+]

package/bench/ab/vscode-concept.json

@@ -0,0 +1,22 @@
+[
+  {
+    "id": "hot-exit",
+    "q": "Which class is responsible for making sure work the user has not saved survives a crash or shutdown and comes back on the next start? Name the base class and its file.",
+    "expect": ["WorkingCopyBackupTracker", "workingCopyBackupTracker.ts"]
+  },
+  {
+    "id": "extension-placement",
+    "q": "Where is it decided whether a given extension runs locally, in the web worker or on the remote machine? Name the class that keeps track of that decision and its file.",
+    "expect": ["ExtensionRunningLocationTracker", "extensionRunningLocationTracker.ts"]
+  },
+  {
+    "id": "file-event-merge",
+    "q": "When many low-level file system notifications arrive for the same path in a short time, which code merges them into fewer events before they are passed on? Name the function or class and the file.",
+    "expect": ["watcher.ts", "oalesce"]
+  },
+  {
+    "id": "indent-folding",
+    "q": "Which function works out the collapsible regions of a document from how far its lines are indented, when no language-specific provider is available? Name the function and the file.",
+    "expect": ["computeRanges", "indentRangeProvider.ts"]
+  }
+]

package/bench/ab/vscode-hard.json

@@ -0,0 +1,32 @@
+[
+  {
+    "id": "picker-subclasses-transitive",
+    "q": "List every non-test class that is a direct or indirect subclass of PickerQuickAccessProvider (follow the inheritance chain to the leaves).",
+    "expect": ["AbstractCommandsQuickAccessProvider", "AbstractEditorCommandsQuickAccessProvider", "StandaloneCommandsQuickAccessProvider", "CommandsQuickAccessProvider", "BaseEditorQuickAccessProvider", "ActiveGroupEditorsByMostRecentlyUsedQuickAccess", "AllEditorsByAppearanceQuickAccess", "AllEditorsByMostRecentlyUsedQuickAccess", "DebugConsoleQuickAccess", "StartDebugQuickAccessProvider", "InstallExtensionQuickAccessProvider", "ManageExtensionsQuickAccessProvider", "IssueQuickAccess", "ViewQuickAccessProvider", "AnythingQuickAccessProvider", "TextSearchQuickAccess", "SymbolsQuickAccessProvider", "TasksQuickAccessProvider", "TerminalQuickAccessProvider"]
+  },
+  {
+    "id": "coalesce-chain",
+    "q": "Trace upwards from the file watcher's coalesceEvents function in production code: which methods call it, and which methods call those? Give method names.",
+    "expect": ["handleFileChanges", "fileChangesAggregator", "handleParcelEvents", "startPolling", "startWatching"]
+  },
+  {
+    "id": "resolver-users-and-subclasses",
+    "q": "Which methods of AbstractKeybindingService call _getResolver(), and which classes (including tests, direct or indirect) extend AbstractKeybindingService?",
+    "expect": ["getDefaultKeybindings", "getKeybindings", "lookupKeybindings", "lookupKeybinding", "softDispatch", "_doDispatch", "StandaloneKeybindingService", "WorkbenchKeybindingService", "MobileKeybindingService", "TestKeybindingService", "TestStandaloneKeybindingService"]
+  },
+  {
+    "id": "soft-dispatch-callers",
+    "q": "Which non-test files call softDispatch on the keybinding service? List every file.",
+    "expect": ["contentHoverController.ts", "actionList.ts", "hoverService.ts", "listService.ts", "agentsWindowExtensionsWidget.ts", "developerActions.ts", "dialog.ts", "modalEditorPart.ts", "accessibleView.ts", "issueReporterKeybindings.ts", "nativeIssueFormService.ts", "markersView.ts", "terminalInstance.ts"]
+  },
+  {
+    "id": "resolve-impact",
+    "q": "I want to add a required parameter to KeybindingResolver.resolve. Which methods call it directly, and which methods of the same service call the dispatching one of those (one more level up)?",
+    "expect": ["softDispatch", "_doDispatch", "_dispatch", "_singleModifierDispatch", "dispatchByUserSettingsLabel"]
+  },
+  {
+    "id": "mobile-keybinding-chain",
+    "q": "Give the complete supertype chain of MobileKeybindingService: every class it inherits from and every interface those implement.",
+    "expect": ["WorkbenchKeybindingService", "AbstractKeybindingService", "Disposable", "IKeybindingService", "IDisposable"]
+  }
+]

package/bench/ab/vscode-impact.json

@@ -0,0 +1,17 @@
+[
+  {
+    "id": "signature-change-count",
+    "q": "IKeybindingService.softDispatch (src/vs/platform/keybinding/common/keybinding.ts) gets a new required parameter. In how many production (non-test) files are there call sites that must be updated? Give the number and list the files.",
+    "expect": ["14", "contentHoverController.ts", "modalEditorPart.ts", "terminalInstance.ts", "markersView.ts"]
+  },
+  {
+    "id": "test-for-symbol",
+    "q": "Which test file directly tests AbstractKeybindingService.softDispatch, and which other classes implement softDispatch and would have to change with it? Name the test file and the mock class.",
+    "expect": ["abstractKeybindingService.test.ts", "MockKeybindingService"]
+  },
+  {
+    "id": "dead-code-file",
+    "q": "Is there an exported function in src/vs/base/common/validation.ts that nothing in the repository uses? Name it.",
+    "expect": ["vLazy"]
+  }
+]

package/bench/ab/vscode-refactors.json

@@ -0,0 +1,18 @@
+[
+  {
+    "id": "rename-soft-dispatch",
+    "task": true,
+    "q": "Rename the keybinding service method softDispatch to softDispatchKey. Update the interface, every implementation (including test doubles and mocks) and every caller in src/ and extensions/, so that nothing refers to the old name any more.",
+    "expect": ["softDispatchKey", "src/vs/platform/keybinding/common/keybinding.ts", "src/vs/platform/keybinding/common/abstractKeybindingService.ts", "src/vs/workbench/contrib/terminal/browser/terminalInstance.ts", "src/vs/platform/list/browser/listService.ts"],
+    "mustNotRemain": "\\bsoftDispatch\\b",
+    "paths": ["src", "extensions"]
+  },
+  {
+    "id": "quick-access-param",
+    "task": true,
+    "q": "Add a required third parameter includeDisabled: boolean to getQuickAccessProvider on the quick access registry. When it is false the behaviour stays as it is now. Update the interface, the implementation and every caller including tests; existing callers pass false.",
+    "expect": ["includeDisabled: boolean", "src/vs/platform/quickinput/common/quickAccess.ts", "src/vs/platform/quickinput/browser/quickAccess.ts", "src/vs/platform/quickinput/browser/helpQuickAccess.ts", "unifiedQuickAccess.ts", "quickAccess.test.ts"],
+    "mustNotRemain": "getQuickAccessProvider\\([^,()]*(\\([^()]*\\))?[^,()]*,[^,()]*(\\([^()]*\\))?[^,()]*\\)[^:]",
+    "paths": ["src"]
+  }
+]

package/bench/ab/vscode-tasks.json

@@ -0,0 +1,20 @@
+[
+  {
+    "id": "quick-access-provider",
+    "task": true,
+    "q": "Add a new quick access provider to the workbench with the prefix \"todo \" that shows three fixed placeholder picks (\"First\", \"Second\", \"Third\"). It must build on the same base class the other pick-based quick access providers use and be registered the same way they are, with a help entry. Put it in its own file under the existing workbench quick access contribution and make sure it is loaded.",
+    "expect": ["extends PickerQuickAccessProvider", "registerQuickAccessProvider", "src/vs/workbench/contrib/quickaccess/browser", "'todo '", "helpEntries"]
+  },
+  {
+    "id": "editor-contribution",
+    "task": true,
+    "q": "Add a new editor contribution under src/vs/editor/contrib/ (its own folder) that, whenever the cursor moves to line 1, logs \"cursor at top\" through the log service. It must implement the editor contribution interface, clean up its listeners on dispose, and be registered the way other editor contributions are registered. Make sure the file is actually loaded by the editor.",
+    "expect": ["IEditorContribution", "registerEditorContribution", "onDidChangeCursorPosition", "ILogService", "editor.all.ts"]
+  },
+  {
+    "id": "terminal-copy-cwd",
+    "task": true,
+    "q": "Add a terminal command with id workbench.action.terminal.copyCwd that copies the current working directory of the active terminal instance to the clipboard. Register it the way the other terminal actions that operate on the active instance are registered, and add its id to the terminal command id enum.",
+    "expect": ["workbench.action.terminal.copyCwd", "registerActiveInstanceAction", "writeText", "terminalActions.ts", "TerminalCommandId"]
+  }
+]

package/bench/ab/vscode.json

@@ -0,0 +1,52 @@
+[
+  {
+    "id": "coalesce",
+    "q": "In the file watcher, where are raw file change events deduplicated / coalesced before they are emitted, and which production (non-test) code calls that function?",
+    "expect": ["coalesceEvents", "watcher.ts", "nodejsWatcherLib", "parcelWatcher"]
+  },
+  {
+    "id": "terminal-hierarchy",
+    "q": "What does the class TerminalInstance extend and implement? Give the full chain of supertypes (classes and interfaces, including what those extend).",
+    "expect": ["Disposable", "ITerminalInstance", "IBaseTerminalInstance"]
+  },
+  {
+    "id": "quick-access",
+    "q": "Which method picks the quick access provider for a typed prefix, in which class and file is it implemented, and what does it return when no registered prefix matches?",
+    "expect": ["QuickAccessRegistry", "quickAccess.ts", "defaultProvider"]
+  },
+  {
+    "id": "keybinding",
+    "q": "Which method decides what command a keypress maps to given the current context and pending chords, where is it defined, and which production file calls it?",
+    "expect": ["KeybindingResolver", "keybindingResolver.ts", "abstractKeybindingService"]
+  },
+  {
+    "id": "folding-contrib",
+    "q": "Which class under src/vs/editor/contrib/folding implements IEditorContribution, and in which file?",
+    "expect": ["FoldingController", "folding.ts"]
+  },
+  {
+    "id": "open-code-editor",
+    "q": "I want to change the signature of ICodeEditorService.openCodeEditor. Which class implements it, and which non-test files under src/vs/editor/contrib call it?",
+    "expect": ["AbstractCodeEditorService", "gotoError.ts", "goToCommands.ts", "referencesController.ts", "symbolNavigation.ts", "inlineCompletionsModel.ts", "linkedEditing.ts", "rename.ts"]
+  },
+  {
+    "id": "activate-by-event",
+    "q": "Which classes (not interfaces) under src/vs/workbench define a method named activateByEvent? List every one.",
+    "expect": ["ExtensionsActivator", "AbstractExtensionService", "ExtensionHostManager", "LazyCreateExtensionHostManager", "NullExtensionService", "ExtensionHostProxy"]
+  },
+  {
+    "id": "code-editor-service-subclasses",
+    "q": "Which classes extend AbstractCodeEditorService, including test helpers?",
+    "expect": ["StandaloneCodeEditorService", "TestCodeEditorService", "workbench/services/editor/browser/codeEditorService.ts"]
+  },
+  {
+    "id": "show-message",
+    "q": "When an extension calls vscode.window.showInformationMessage, which main-thread class receives the request, and in which file is it?",
+    "expect": ["MainThreadMessageService", "mainThreadMessageService.ts"]
+  },
+  {
+    "id": "glob-regexp",
+    "q": "In the base glob implementation, which function turns a glob pattern string into regular expression source, and which function in the same file calls it to build the RegExp matcher?",
+    "expect": ["parseRegExp", "glob.ts", "toRegExp"]
+  }
+]

package/bench/ab.mjs

@@ -0,0 +1,213 @@
+#!/usr/bin/env node
+// A/B benchmark: the same questions answered by headless Claude Code sessions, once with the clast
+// MCP server and once with only Read / Grep / Glob. One session per question, so every number is
+// per question. Usage:
+//   node bench/ab.mjs <repo-dir> <questions.json> [--runs 3] [--parallel 4] [--model <id>]
+//                     [--arms clast,grep] [--only id1,id2] [--out results.json]
+// Requires the `claude` CLI on PATH and an index that is already built (`clast-ai index`).
+import { spawn, spawnSync } from 'node:child_process';
+import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { tmpdir } from 'node:os';
+
+const here = dirname(fileURLToPath(import.meta.url));
+const args = process.argv.slice(2);
+const flag = (name, dflt) => {
+  const i = args.indexOf(`--${name}`);
+  return i >= 0 ? args.splice(i, 2)[1] : dflt;
+};
+const runs = Number(flag('runs', '1'));
+const parallel = Number(flag('parallel', '4'));
+const model = flag('model', '');
+const arms = flag('arms', 'clast,grep').split(',');
+const only = flag('only', '');
+const out = flag('out', '');
+const noDefer = args.includes('--no-defer') ? (args.splice(args.indexOf('--no-defer'), 1), true) : false;
+const hint = flag('hint', '');
+const alwaysLoad = args.includes('--always-load') ? (args.splice(args.indexOf('--always-load'), 1), true) : false;
+const server = flag('server', join(here, '..', 'dist', 'cli.js'));
+const [repoArg, questionsArg] = args;
+if (!repoArg || !questionsArg) {
+  console.error('usage: node bench/ab.mjs <repo-dir> <questions.json> [--runs n] [--parallel n] [--model id]');
+  process.exit(2);
+}
+const repo = resolve(repoArg);
+let questions = JSON.parse(readFileSync(questionsArg, 'utf8'));
+if (only) questions = questions.filter(q => only.split(',').includes(q.id));
+
+const work = join(tmpdir(), `clast-ab-${process.pid}`);
+mkdirSync(work, { recursive: true });
+const mcpClast = join(work, 'mcp-clast.json');
+const mcpNone = join(work, 'mcp-none.json');
+writeFileSync(mcpClast, JSON.stringify({ mcpServers: { clast: { command: process.execPath, args: [resolve(server)] } } }));
+const mcpLoaded = join(work, 'mcp-loaded.json');
+writeFileSync(mcpLoaded, JSON.stringify({ mcpServers: { clast: { command: process.execPath, args: [resolve(server)], alwaysLoad: true } } }));
+writeFileSync(mcpNone, JSON.stringify({ mcpServers: {} }));
+
+const BLOCKED = 'Bash,PowerShell,NotebookEdit,Agent,Task,WebSearch,WebFetch,Skill';
+const PROMPT = q =>
+  `Answer this question about the code in the current directory. Be brief: name the symbols and ` +
+  `file paths, no long explanations. Do not modify anything.\n\n${q}`;
+// Implementation tasks ({"task": true} in the questions file): the agent edits the checkout, the
+// diff is scored against the expected strings, and the checkout is reset. Such runs must not share
+// a checkout, so they are forced to run one at a time.
+const TASK_PROMPT = q =>
+  `Implement the following change in the repository in the current directory. Find the right place ` +
+  `and the existing base classes, registries and conventions first, and follow them. Keep the change ` +
+  `focused. You cannot build or run anything here, so do not try. Finish with a two-line summary of ` +
+  `the files you changed.\n\n${q}`;
+const git = (...a) => spawnSync('git', a, { cwd: repo, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
+function collectDiff() {
+  git('add', '-A', '-N');
+  return git('diff').stdout || '';
+}
+function resetRepo() {
+  git('reset', '-q', '--hard', 'HEAD');
+  git('clean', '-fdq');
+}
+
+function runOne(question, arm) {
+  return new Promise(done => {
+    const cli = [
+      '-p', '--output-format', 'stream-json', '--verbose',
+      '--strict-mcp-config', '--mcp-config', arm === 'clast' ? mcpClast : arm === 'clast-loaded' ? mcpLoaded : mcpNone,
+      '--allowedTools',
+      (question.task ? 'Read,Grep,Glob,Edit,Write' : 'Read,Grep,Glob') + (arm === 'grep' ? '' : ',mcp__clast'),
+      '--disallowedTools', question.task ? BLOCKED : `${BLOCKED},Edit,Write`,
+      '--max-turns', question.task ? '150' : '60',
+    ];
+    if (model) cli.push('--model', model);
+    if (hint && arm !== 'grep') cli.push('--append-system-prompt', JSON.stringify(hint));
+    const env = { ...process.env };
+    for (const k of Object.keys(env)) if (/^CLAUDE_?CODE|^CLAUDECODE/i.test(k)) delete env[k];
+    if (noDefer) env.ENABLE_TOOL_SEARCH = 'false';
+    const started = Date.now();
+    const child = spawn('claude', cli, { cwd: repo, env, shell: process.platform === 'win32' });
+    let buf = '';
+    let err = '';
+    const rec = { id: question.id, arm, tools: {}, toolCalls: 0, resultChars: 0, mcpOk: null };
+    const raw = process.env.AB_RAW ? [] : null;
+    const onLine = line => {
+      if (!line.trim()) return;
+      if (raw) raw.push(line);
+      let msg;
+      try { msg = JSON.parse(line); } catch { return; }
+      if (msg.type === 'system' && msg.subtype === 'init') {
+        rec.model = msg.model;
+        const s = (msg.mcp_servers || []).find(x => x.name === 'clast');
+        rec.mcpOk = s ? s.status : null;
+      } else if (msg.type === 'assistant') {
+        for (const part of msg.message?.content || []) {
+          if (part.type === 'tool_use') {
+            rec.toolCalls++;
+            const name = part.name.replace(/^mcp__clast__/, '');
+            rec.tools[name] = (rec.tools[name] || 0) + 1;
+          }
+        }
+      } else if (msg.type === 'user') {
+        for (const part of msg.message?.content || []) {
+          if (part.type !== 'tool_result') continue;
+          const c = part.content;
+          rec.resultChars += typeof c === 'string' ? c.length : JSON.stringify(c ?? '').length;
+        }
+      } else if (msg.type === 'result') {
+        const u = msg.usage || {};
+        rec.answer = msg.result || '';
+        rec.error = msg.is_error ? msg.subtype : undefined;
+        rec.turns = msg.num_turns;
+        rec.apiMs = msg.duration_api_ms;
+        rec.cost = msg.total_cost_usd;
+        rec.inputTokens = (u.input_tokens || 0) + (u.cache_creation_input_tokens || 0) + (u.cache_read_input_tokens || 0);
+        rec.freshInput = (u.input_tokens || 0) + (u.cache_creation_input_tokens || 0);
+        rec.outputTokens = u.output_tokens || 0;
+      }
+    };
+    child.stdout.on('data', d => {
+      buf += d;
+      let i;
+      while ((i = buf.indexOf('\n')) >= 0) { onLine(buf.slice(0, i)); buf = buf.slice(i + 1); }
+    });
+    child.stderr.on('data', d => { err += d; });
+    child.stdin.end(question.task ? TASK_PROMPT(question.q) : PROMPT(question.q));
+    child.on('close', code => {
+      onLine(buf);
+      rec.wallMs = Date.now() - started;
+      if (raw) writeFileSync(join(process.env.AB_RAW, `${question.id}-${arm}-${started}.jsonl`), raw.join('\n'));
+      if (rec.answer === undefined) rec.error = `exit ${code}: ${err.slice(-300)}`;
+      if (question.task) {
+        rec.diff = collectDiff();
+        rec.diffLines = rec.diff.split('\n').filter(l => /^[+-][^+-]/.test(l)).length;
+        rec.files = [...rec.diff.matchAll(/^diff --git a\/(\S+)/gm)].map(m => m[1]);
+        // "mustNotRemain": a pattern that a complete change leaves nowhere (the old name, the old call shape).
+        if (question.mustNotRemain) {
+          const left = git('grep', '-I', '-E', '-c', question.mustNotRemain, '--', ...(question.paths || ['.'])).stdout || '';
+          rec.remaining = left.split('\n').filter(Boolean).reduce((n, l) => n + Number(l.slice(l.lastIndexOf(':') + 1)), 0);
+          rec.remainingFiles = left.split('\n').filter(Boolean).length;
+        }
+        resetRepo();
+      }
+      const answer = (question.task ? rec.diff : rec.answer || '').toLowerCase().replaceAll('\\', '/');
+      const hits = question.expect.filter(e => answer.includes(e.toLowerCase()));
+      rec.missing = question.expect.filter(e => !hits.includes(e));
+      if (question.mustNotRemain) {
+        // Half for the expected strings, half for completeness (nothing of the old form left).
+        rec.score = 0.5 * (hits.length / question.expect.length) + 0.5 * (rec.remaining === 0 ? 1 : 0);
+        if (rec.remaining) rec.missing.push(`${rec.remaining} left in ${rec.remainingFiles} files`);
+      } else {
+        rec.score = hits.length / question.expect.length;
+      }
+      done(rec);
+    });
+  });
+}
+
+const jobs = [];
+for (let r = 0; r < runs; r++) for (const q of questions) for (const arm of arms) jobs.push({ q, arm, r });
+const results = [];
+let next = 0;
+async function worker() {
+  while (next < jobs.length) {
+    const job = jobs[next++];
+    const rec = await runOne(job.q, job.arm);
+    rec.run = job.r;
+    results.push(rec);
+    console.error(
+      `[${results.length}/${jobs.length}] ${rec.arm.padEnd(12)} ${rec.id.padEnd(30)} calls ${String(rec.toolCalls).padStart(3)} ` +
+      `in ${String(rec.inputTokens ?? '?').padStart(8)} out ${String(rec.outputTokens ?? '?').padStart(6)} ` +
+      `${(rec.wallMs / 1000).toFixed(0).padStart(4)} s  score ${rec.score.toFixed(2)}${rec.error ? '  ERROR ' + rec.error : ''}`
+    );
+  }
+}
+const lanes = questions.some(q => q.task) ? 1 : Math.min(parallel, jobs.length);
+if (questions.some(q => q.task)) {
+  if (git('status', '--porcelain').stdout.trim()) {
+    console.error(`task runs reset the checkout after every session; ${repo} has uncommitted changes, refusing.`);
+    process.exit(2);
+  }
+}
+await Promise.all(Array.from({ length: lanes }, worker));
+
+const sum = (rows, f) => rows.reduce((a, r) => a + (f(r) || 0), 0);
+const median = xs => { const s = [...xs].sort((a, b) => a - b); return s.length ? s[Math.floor((s.length - 1) / 2)] : 0; };
+const summary = {};
+for (const arm of arms) {
+  const rows = results.filter(r => r.arm === arm && !r.error);
+  const n = rows.length || 1;
+  summary[arm] = {
+    sessions: rows.length,
+    errors: results.filter(r => r.arm === arm && r.error).length,
+    toolCalls: +(sum(rows, r => r.toolCalls) / n).toFixed(1),
+    toolResultChars: Math.round(sum(rows, r => r.resultChars) / n),
+    inputTokens: Math.round(sum(rows, r => r.inputTokens) / n),
+    freshInputTokens: Math.round(sum(rows, r => r.freshInput) / n),
+    outputTokens: Math.round(sum(rows, r => r.outputTokens) / n),
+    costUsd: +(sum(rows, r => r.cost) / n).toFixed(4),
+    wallSecMean: +(sum(rows, r => r.wallMs) / n / 1000).toFixed(1),
+    wallSecMedian: +(median(rows.map(r => r.wallMs)) / 1000).toFixed(1),
+    score: +(sum(rows, r => r.score) / n).toFixed(3),
+    fullyCorrect: rows.filter(r => r.score === 1).length,
+  };
+}
+console.log(JSON.stringify({ repo, model: results[0]?.model, runs, questions: questions.length, summary }, null, 2));
+if (out) writeFileSync(out, JSON.stringify({ repo, runs, summary, results }, null, 2));