@mcoda/mswarm 0.1.57 → 0.1.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -0
- package/dist/codali-executor.d.ts +278 -0
- package/dist/codali-executor.d.ts.map +1 -0
- package/dist/codali-executor.js +243 -0
- package/dist/codali-executor.js.map +1 -0
- package/dist/runtime.d.ts +46 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +298 -30
- package/dist/runtime.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +66 -1
- package/dist/server.js.map +1 -1
- package/dist/vendor/codali/agents/AgentProtocol.d.ts +287 -0
- package/dist/vendor/codali/agents/AgentProtocol.d.ts.map +1 -0
- package/dist/vendor/codali/agents/AgentProtocol.js +365 -0
- package/dist/vendor/codali/agents/AgentResolver.d.ts +23 -0
- package/dist/vendor/codali/agents/AgentResolver.d.ts.map +1 -0
- package/dist/vendor/codali/agents/AgentResolver.js +77 -0
- package/dist/vendor/codali/agents/PhaseAgentSelector.d.ts +23 -0
- package/dist/vendor/codali/agents/PhaseAgentSelector.d.ts.map +1 -0
- package/dist/vendor/codali/agents/PhaseAgentSelector.js +287 -0
- package/dist/vendor/codali/cli/EvalCommand.d.ts +37 -0
- package/dist/vendor/codali/cli/EvalCommand.d.ts.map +1 -0
- package/dist/vendor/codali/cli/EvalCommand.js +333 -0
- package/dist/vendor/codali/cli/FeedbackCommand.d.ts +22 -0
- package/dist/vendor/codali/cli/FeedbackCommand.d.ts.map +1 -0
- package/dist/vendor/codali/cli/FeedbackCommand.js +163 -0
- package/dist/vendor/codali/cli/RunCommand.d.ts +78 -0
- package/dist/vendor/codali/cli/RunCommand.d.ts.map +1 -0
- package/dist/vendor/codali/cli/RunCommand.js +2261 -0
- package/dist/vendor/codali/cli.d.ts +3 -0
- package/dist/vendor/codali/cli.d.ts.map +1 -0
- package/dist/vendor/codali/cli.js +109 -0
- package/dist/vendor/codali/cognitive/ArchitectPlanner.d.ts +107 -0
- package/dist/vendor/codali/cognitive/ArchitectPlanner.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/ArchitectPlanner.js +1726 -0
- package/dist/vendor/codali/cognitive/BuilderOutputParser.d.ts +25 -0
- package/dist/vendor/codali/cognitive/BuilderOutputParser.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/BuilderOutputParser.js +164 -0
- package/dist/vendor/codali/cognitive/BuilderRunner.d.ts +76 -0
- package/dist/vendor/codali/cognitive/BuilderRunner.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/BuilderRunner.js +1159 -0
- package/dist/vendor/codali/cognitive/ContextAssembler.d.ts +91 -0
- package/dist/vendor/codali/cognitive/ContextAssembler.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/ContextAssembler.js +4547 -0
- package/dist/vendor/codali/cognitive/ContextBudget.d.ts +19 -0
- package/dist/vendor/codali/cognitive/ContextBudget.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/ContextBudget.js +35 -0
- package/dist/vendor/codali/cognitive/ContextFileLoader.d.ts +30 -0
- package/dist/vendor/codali/cognitive/ContextFileLoader.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/ContextFileLoader.js +307 -0
- package/dist/vendor/codali/cognitive/ContextManager.d.ts +47 -0
- package/dist/vendor/codali/cognitive/ContextManager.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/ContextManager.js +272 -0
- package/dist/vendor/codali/cognitive/ContextRedactor.d.ts +18 -0
- package/dist/vendor/codali/cognitive/ContextRedactor.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/ContextRedactor.js +53 -0
- package/dist/vendor/codali/cognitive/ContextSelector.d.ts +22 -0
- package/dist/vendor/codali/cognitive/ContextSelector.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/ContextSelector.js +431 -0
- package/dist/vendor/codali/cognitive/ContextSerializer.d.ts +8 -0
- package/dist/vendor/codali/cognitive/ContextSerializer.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/ContextSerializer.js +882 -0
- package/dist/vendor/codali/cognitive/ContextStore.d.ts +27 -0
- package/dist/vendor/codali/cognitive/ContextStore.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/ContextStore.js +79 -0
- package/dist/vendor/codali/cognitive/ContextSummarizer.d.ts +16 -0
- package/dist/vendor/codali/cognitive/ContextSummarizer.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/ContextSummarizer.js +45 -0
- package/dist/vendor/codali/cognitive/CostEstimator.d.ts +31 -0
- package/dist/vendor/codali/cognitive/CostEstimator.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/CostEstimator.js +66 -0
- package/dist/vendor/codali/cognitive/CriticEvaluator.d.ts +32 -0
- package/dist/vendor/codali/cognitive/CriticEvaluator.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/CriticEvaluator.js +297 -0
- package/dist/vendor/codali/cognitive/EvidenceGate.d.ts +9 -0
- package/dist/vendor/codali/cognitive/EvidenceGate.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/EvidenceGate.js +75 -0
- package/dist/vendor/codali/cognitive/GoldenExampleIndexer.d.ts +12 -0
- package/dist/vendor/codali/cognitive/GoldenExampleIndexer.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/GoldenExampleIndexer.js +34 -0
- package/dist/vendor/codali/cognitive/GoldenSetStore.d.ts +33 -0
- package/dist/vendor/codali/cognitive/GoldenSetStore.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/GoldenSetStore.js +159 -0
- package/dist/vendor/codali/cognitive/IntentSignals.d.ts +7 -0
- package/dist/vendor/codali/cognitive/IntentSignals.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/IntentSignals.js +285 -0
- package/dist/vendor/codali/cognitive/LearningGovernance.d.ts +100 -0
- package/dist/vendor/codali/cognitive/LearningGovernance.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/LearningGovernance.js +276 -0
- package/dist/vendor/codali/cognitive/MemoryWriteback.d.ts +64 -0
- package/dist/vendor/codali/cognitive/MemoryWriteback.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/MemoryWriteback.js +287 -0
- package/dist/vendor/codali/cognitive/PatchApplier.d.ts +49 -0
- package/dist/vendor/codali/cognitive/PatchApplier.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/PatchApplier.js +199 -0
- package/dist/vendor/codali/cognitive/PatchInterpreter.d.ts +35 -0
- package/dist/vendor/codali/cognitive/PatchInterpreter.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/PatchInterpreter.js +100 -0
- package/dist/vendor/codali/cognitive/PatchOutputNormalizer.d.ts +7 -0
- package/dist/vendor/codali/cognitive/PatchOutputNormalizer.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/PatchOutputNormalizer.js +59 -0
- package/dist/vendor/codali/cognitive/PostMortemAnalyzer.d.ts +17 -0
- package/dist/vendor/codali/cognitive/PostMortemAnalyzer.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/PostMortemAnalyzer.js +131 -0
- package/dist/vendor/codali/cognitive/PreferenceExtraction.d.ts +3 -0
- package/dist/vendor/codali/cognitive/PreferenceExtraction.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/PreferenceExtraction.js +85 -0
- package/dist/vendor/codali/cognitive/Prompts.d.ts +15 -0
- package/dist/vendor/codali/cognitive/Prompts.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/Prompts.js +326 -0
- package/dist/vendor/codali/cognitive/ProviderRouting.d.ts +16 -0
- package/dist/vendor/codali/cognitive/ProviderRouting.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/ProviderRouting.js +24 -0
- package/dist/vendor/codali/cognitive/QueryExtraction.d.ts +12 -0
- package/dist/vendor/codali/cognitive/QueryExtraction.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/QueryExtraction.js +262 -0
- package/dist/vendor/codali/cognitive/RunHistoryIndexer.d.ts +13 -0
- package/dist/vendor/codali/cognitive/RunHistoryIndexer.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/RunHistoryIndexer.js +125 -0
- package/dist/vendor/codali/cognitive/SmartPipeline.d.ts +92 -0
- package/dist/vendor/codali/cognitive/SmartPipeline.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/SmartPipeline.js +4804 -0
- package/dist/vendor/codali/cognitive/Types.d.ts +474 -0
- package/dist/vendor/codali/cognitive/Types.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/Types.js +7 -0
- package/dist/vendor/codali/cognitive/ValidationRunner.d.ts +57 -0
- package/dist/vendor/codali/cognitive/ValidationRunner.d.ts.map +1 -0
- package/dist/vendor/codali/cognitive/ValidationRunner.js +515 -0
- package/dist/vendor/codali/config/Config.d.ts +249 -0
- package/dist/vendor/codali/config/Config.d.ts.map +1 -0
- package/dist/vendor/codali/config/Config.js +200 -0
- package/dist/vendor/codali/config/ConfigLoader.d.ts +56 -0
- package/dist/vendor/codali/config/ConfigLoader.d.ts.map +1 -0
- package/dist/vendor/codali/config/ConfigLoader.js +1246 -0
- package/dist/vendor/codali/docdex/DocdexClient.d.ts +158 -0
- package/dist/vendor/codali/docdex/DocdexClient.d.ts.map +1 -0
- package/dist/vendor/codali/docdex/DocdexClient.js +785 -0
- package/dist/vendor/codali/eval/EvalRunner.d.ts +35 -0
- package/dist/vendor/codali/eval/EvalRunner.d.ts.map +1 -0
- package/dist/vendor/codali/eval/EvalRunner.js +38 -0
- package/dist/vendor/codali/eval/EvalTaskExecutor.d.ts +81 -0
- package/dist/vendor/codali/eval/EvalTaskExecutor.d.ts.map +1 -0
- package/dist/vendor/codali/eval/EvalTaskExecutor.js +371 -0
- package/dist/vendor/codali/eval/GateEvaluator.d.ts +31 -0
- package/dist/vendor/codali/eval/GateEvaluator.d.ts.map +1 -0
- package/dist/vendor/codali/eval/GateEvaluator.js +134 -0
- package/dist/vendor/codali/eval/MetricTypes.d.ts +28 -0
- package/dist/vendor/codali/eval/MetricTypes.d.ts.map +1 -0
- package/dist/vendor/codali/eval/MetricTypes.js +1 -0
- package/dist/vendor/codali/eval/MetricsAggregator.d.ts +4 -0
- package/dist/vendor/codali/eval/MetricsAggregator.d.ts.map +1 -0
- package/dist/vendor/codali/eval/MetricsAggregator.js +97 -0
- package/dist/vendor/codali/eval/RegressionComparator.d.ts +29 -0
- package/dist/vendor/codali/eval/RegressionComparator.d.ts.map +1 -0
- package/dist/vendor/codali/eval/RegressionComparator.js +155 -0
- package/dist/vendor/codali/eval/ReportInputAdapter.d.ts +52 -0
- package/dist/vendor/codali/eval/ReportInputAdapter.d.ts.map +1 -0
- package/dist/vendor/codali/eval/ReportInputAdapter.js +229 -0
- package/dist/vendor/codali/eval/ReportSerializer.d.ts +32 -0
- package/dist/vendor/codali/eval/ReportSerializer.d.ts.map +1 -0
- package/dist/vendor/codali/eval/ReportSerializer.js +33 -0
- package/dist/vendor/codali/eval/ReportStore.d.ts +18 -0
- package/dist/vendor/codali/eval/ReportStore.d.ts.map +1 -0
- package/dist/vendor/codali/eval/ReportStore.js +96 -0
- package/dist/vendor/codali/eval/SuiteLoader.d.ts +12 -0
- package/dist/vendor/codali/eval/SuiteLoader.d.ts.map +1 -0
- package/dist/vendor/codali/eval/SuiteLoader.js +51 -0
- package/dist/vendor/codali/eval/SuiteSchema.d.ts +56 -0
- package/dist/vendor/codali/eval/SuiteSchema.d.ts.map +1 -0
- package/dist/vendor/codali/eval/SuiteSchema.js +357 -0
- package/dist/vendor/codali/index.d.ts +11 -0
- package/dist/vendor/codali/index.d.ts.map +1 -0
- package/dist/vendor/codali/index.js +5 -0
- package/dist/vendor/codali/providers/CodexCliProvider.d.ts +8 -0
- package/dist/vendor/codali/providers/CodexCliProvider.d.ts.map +1 -0
- package/dist/vendor/codali/providers/CodexCliProvider.js +282 -0
- package/dist/vendor/codali/providers/OllamaRemoteProvider.d.ts +8 -0
- package/dist/vendor/codali/providers/OllamaRemoteProvider.d.ts.map +1 -0
- package/dist/vendor/codali/providers/OllamaRemoteProvider.js +300 -0
- package/dist/vendor/codali/providers/OpenAiCompatibleProvider.d.ts +8 -0
- package/dist/vendor/codali/providers/OpenAiCompatibleProvider.d.ts.map +1 -0
- package/dist/vendor/codali/providers/OpenAiCompatibleProvider.js +192 -0
- package/dist/vendor/codali/providers/ProviderRegistry.d.ts +12 -0
- package/dist/vendor/codali/providers/ProviderRegistry.d.ts.map +1 -0
- package/dist/vendor/codali/providers/ProviderRegistry.js +28 -0
- package/dist/vendor/codali/providers/ProviderTypes.d.ts +81 -0
- package/dist/vendor/codali/providers/ProviderTypes.d.ts.map +1 -0
- package/dist/vendor/codali/providers/ProviderTypes.js +1 -0
- package/dist/vendor/codali/runtime/CodaliRuntime.d.ts +189 -0
- package/dist/vendor/codali/runtime/CodaliRuntime.d.ts.map +1 -0
- package/dist/vendor/codali/runtime/CodaliRuntime.js +1435 -0
- package/dist/vendor/codali/runtime/DeepInvestigationErrors.d.ts +39 -0
- package/dist/vendor/codali/runtime/DeepInvestigationErrors.d.ts.map +1 -0
- package/dist/vendor/codali/runtime/DeepInvestigationErrors.js +57 -0
- package/dist/vendor/codali/runtime/RunContext.d.ts +27 -0
- package/dist/vendor/codali/runtime/RunContext.d.ts.map +1 -0
- package/dist/vendor/codali/runtime/RunContext.js +51 -0
- package/dist/vendor/codali/runtime/RunLogQuery.d.ts +48 -0
- package/dist/vendor/codali/runtime/RunLogQuery.d.ts.map +1 -0
- package/dist/vendor/codali/runtime/RunLogQuery.js +36 -0
- package/dist/vendor/codali/runtime/RunLogReader.d.ts +19 -0
- package/dist/vendor/codali/runtime/RunLogReader.d.ts.map +1 -0
- package/dist/vendor/codali/runtime/RunLogReader.js +361 -0
- package/dist/vendor/codali/runtime/RunLogger.d.ts +71 -0
- package/dist/vendor/codali/runtime/RunLogger.d.ts.map +1 -0
- package/dist/vendor/codali/runtime/RunLogger.js +100 -0
- package/dist/vendor/codali/runtime/RunTelemetryTypes.d.ts +117 -0
- package/dist/vendor/codali/runtime/RunTelemetryTypes.d.ts.map +1 -0
- package/dist/vendor/codali/runtime/RunTelemetryTypes.js +299 -0
- package/dist/vendor/codali/runtime/Runner.d.ts +66 -0
- package/dist/vendor/codali/runtime/Runner.d.ts.map +1 -0
- package/dist/vendor/codali/runtime/Runner.js +215 -0
- package/dist/vendor/codali/runtime/StoragePaths.d.ts +3 -0
- package/dist/vendor/codali/runtime/StoragePaths.d.ts.map +1 -0
- package/dist/vendor/codali/runtime/StoragePaths.js +19 -0
- package/dist/vendor/codali/runtime/WorkspaceLock.d.ts +30 -0
- package/dist/vendor/codali/runtime/WorkspaceLock.d.ts.map +1 -0
- package/dist/vendor/codali/runtime/WorkspaceLock.js +141 -0
- package/dist/vendor/codali/session/InstructionLoader.d.ts +14 -0
- package/dist/vendor/codali/session/InstructionLoader.d.ts.map +1 -0
- package/dist/vendor/codali/session/InstructionLoader.js +107 -0
- package/dist/vendor/codali/session/SessionStore.d.ts +81 -0
- package/dist/vendor/codali/session/SessionStore.d.ts.map +1 -0
- package/dist/vendor/codali/session/SessionStore.js +244 -0
- package/dist/vendor/codali/subagents/SubagentOrchestrator.d.ts +68 -0
- package/dist/vendor/codali/subagents/SubagentOrchestrator.d.ts.map +1 -0
- package/dist/vendor/codali/subagents/SubagentOrchestrator.js +150 -0
- package/dist/vendor/codali/tools/ToolRegistry.d.ts +9 -0
- package/dist/vendor/codali/tools/ToolRegistry.d.ts.map +1 -0
- package/dist/vendor/codali/tools/ToolRegistry.js +293 -0
- package/dist/vendor/codali/tools/ToolTypes.d.ts +66 -0
- package/dist/vendor/codali/tools/ToolTypes.d.ts.map +1 -0
- package/dist/vendor/codali/tools/ToolTypes.js +40 -0
- package/dist/vendor/codali/tools/diff/DiffTool.d.ts +3 -0
- package/dist/vendor/codali/tools/diff/DiffTool.d.ts.map +1 -0
- package/dist/vendor/codali/tools/diff/DiffTool.js +34 -0
- package/dist/vendor/codali/tools/docdex/DocdexTools.d.ts +4 -0
- package/dist/vendor/codali/tools/docdex/DocdexTools.d.ts.map +1 -0
- package/dist/vendor/codali/tools/docdex/DocdexTools.js +490 -0
- package/dist/vendor/codali/tools/filesystem/FileTools.d.ts +3 -0
- package/dist/vendor/codali/tools/filesystem/FileTools.d.ts.map +1 -0
- package/dist/vendor/codali/tools/filesystem/FileTools.js +141 -0
- package/dist/vendor/codali/tools/search/SearchTool.d.ts +3 -0
- package/dist/vendor/codali/tools/search/SearchTool.d.ts.map +1 -0
- package/dist/vendor/codali/tools/search/SearchTool.js +46 -0
- package/dist/vendor/codali/tools/shell/ShellTool.d.ts +3 -0
- package/dist/vendor/codali/tools/shell/ShellTool.d.ts.map +1 -0
- package/dist/vendor/codali/tools/shell/ShellTool.js +104 -0
- package/package.json +5 -3
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import process from "node:process";
|
|
4
|
+
import { loadConfig } from "../config/ConfigLoader.js";
|
|
5
|
+
import { EvalTaskExecutor } from "../eval/EvalTaskExecutor.js";
|
|
6
|
+
import { EvalRunner } from "../eval/EvalRunner.js";
|
|
7
|
+
import { evaluateGates, resolveGateThresholds } from "../eval/GateEvaluator.js";
|
|
8
|
+
import { aggregateMetrics } from "../eval/MetricsAggregator.js";
|
|
9
|
+
import { compareAgainstBaseline } from "../eval/RegressionComparator.js";
|
|
10
|
+
import { serializeEvalReport } from "../eval/ReportSerializer.js";
|
|
11
|
+
import { ReportStore } from "../eval/ReportStore.js";
|
|
12
|
+
import { loadSuiteFromFile } from "../eval/SuiteLoader.js";
|
|
13
|
+
import { SuiteValidationError } from "../eval/SuiteSchema.js";
|
|
14
|
+
import { resolveWorkspaceRoot } from "./RunCommand.js";
|
|
15
|
+
export const EVAL_EXIT_CODES = {
|
|
16
|
+
usage_error: 2,
|
|
17
|
+
suite_validation_error: 3,
|
|
18
|
+
run_failure: 4,
|
|
19
|
+
gate_failure: 5,
|
|
20
|
+
};
|
|
21
|
+
export class EvalCommandError extends Error {
|
|
22
|
+
constructor(message, exitCode) {
|
|
23
|
+
super(message);
|
|
24
|
+
this.name = "EvalCommandError";
|
|
25
|
+
this.exitCode = exitCode;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
const HELP_TEXT = "Usage: codali eval --suite <path> [options]\n\n"
|
|
29
|
+
+ "Options:\n"
|
|
30
|
+
+ " --suite <path> Path to eval suite JSON (required)\n"
|
|
31
|
+
+ " --output <text|json> Output mode (default: text)\n"
|
|
32
|
+
+ " --baseline <path> Optional baseline report for regression diff\n"
|
|
33
|
+
+ " --report-dir <path> Override eval report output directory\n"
|
|
34
|
+
+ " --workspace-root <path> Workspace root for task execution\n"
|
|
35
|
+
+ " --provider <name> Provider override passed to task runs\n"
|
|
36
|
+
+ " --model <name> Model override passed to task runs\n"
|
|
37
|
+
+ " --api-key <token> API key override passed to task runs\n"
|
|
38
|
+
+ " --base-url <url> Base URL override passed to task runs\n"
|
|
39
|
+
+ " --agent <slug> Agent override passed to task runs\n"
|
|
40
|
+
+ " --agent-id <id> Agent id override passed to task runs\n"
|
|
41
|
+
+ " --agent-slug <slug> Agent slug override passed to task runs\n"
|
|
42
|
+
+ " --profile <name> Workflow profile override for task runs\n"
|
|
43
|
+
+ " --smart Force smart mode for task runs\n"
|
|
44
|
+
+ " --no-deep-investigation Disable deep investigation for task runs\n"
|
|
45
|
+
+ " --config <path> Config file path\n"
|
|
46
|
+
+ " --help Show help\n";
|
|
47
|
+
const expectValue = (argv, index, flag) => {
|
|
48
|
+
const value = argv[index + 1];
|
|
49
|
+
if (!value || value.startsWith("--")) {
|
|
50
|
+
throw new EvalCommandError(`Missing value for ${flag}.`, EVAL_EXIT_CODES.usage_error);
|
|
51
|
+
}
|
|
52
|
+
return value;
|
|
53
|
+
};
|
|
54
|
+
export const parseEvalArgs = (argv) => {
|
|
55
|
+
const parsed = {
|
|
56
|
+
output: "text",
|
|
57
|
+
};
|
|
58
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
59
|
+
const arg = argv[index];
|
|
60
|
+
if (arg === "--help" || arg === "-h") {
|
|
61
|
+
parsed.help = true;
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
if (arg === "--suite") {
|
|
65
|
+
parsed.suite_path = expectValue(argv, index, "--suite");
|
|
66
|
+
index += 1;
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
if (arg === "--output") {
|
|
70
|
+
const value = expectValue(argv, index, "--output").trim().toLowerCase();
|
|
71
|
+
if (value !== "text" && value !== "json") {
|
|
72
|
+
throw new EvalCommandError("Invalid --output value. Expected text|json.", EVAL_EXIT_CODES.usage_error);
|
|
73
|
+
}
|
|
74
|
+
parsed.output = value;
|
|
75
|
+
index += 1;
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
if (arg === "--baseline") {
|
|
79
|
+
parsed.baseline_path = expectValue(argv, index, "--baseline");
|
|
80
|
+
index += 1;
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
if (arg === "--report-dir") {
|
|
84
|
+
parsed.report_dir = expectValue(argv, index, "--report-dir");
|
|
85
|
+
index += 1;
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
if (arg === "--workspace-root") {
|
|
89
|
+
parsed.workspace_root = expectValue(argv, index, "--workspace-root");
|
|
90
|
+
index += 1;
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
if (arg === "--config") {
|
|
94
|
+
parsed.config_path = expectValue(argv, index, "--config");
|
|
95
|
+
index += 1;
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
if (arg === "--provider") {
|
|
99
|
+
parsed.provider = expectValue(argv, index, "--provider");
|
|
100
|
+
index += 1;
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
if (arg === "--model") {
|
|
104
|
+
parsed.model = expectValue(argv, index, "--model");
|
|
105
|
+
index += 1;
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
if (arg === "--api-key") {
|
|
109
|
+
parsed.api_key = expectValue(argv, index, "--api-key");
|
|
110
|
+
index += 1;
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
if (arg === "--base-url") {
|
|
114
|
+
parsed.base_url = expectValue(argv, index, "--base-url");
|
|
115
|
+
index += 1;
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
if (arg === "--agent") {
|
|
119
|
+
parsed.agent = expectValue(argv, index, "--agent");
|
|
120
|
+
index += 1;
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
if (arg === "--agent-id") {
|
|
124
|
+
parsed.agent_id = expectValue(argv, index, "--agent-id");
|
|
125
|
+
index += 1;
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
if (arg === "--agent-slug") {
|
|
129
|
+
parsed.agent_slug = expectValue(argv, index, "--agent-slug");
|
|
130
|
+
index += 1;
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
if (arg === "--profile" || arg === "--workflow-profile") {
|
|
134
|
+
parsed.workflow_profile = expectValue(argv, index, arg);
|
|
135
|
+
index += 1;
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
if (arg === "--smart") {
|
|
139
|
+
parsed.smart = true;
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
if (arg === "--no-deep-investigation") {
|
|
143
|
+
parsed.no_deep_investigation = true;
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
throw new EvalCommandError(`Unknown eval flag: ${arg}`, EVAL_EXIT_CODES.usage_error);
|
|
147
|
+
}
|
|
148
|
+
return parsed;
|
|
149
|
+
};
|
|
150
|
+
const formatRate = (value) => value === null ? "n/a" : `${(value * 100).toFixed(2)}%`;
|
|
151
|
+
const formatNullable = (value, unit = "") => value === null ? "n/a" : `${value.toFixed(2)}${unit}`;
|
|
152
|
+
const printTextReport = (report, reportPath) => {
|
|
153
|
+
const metrics = report.metrics;
|
|
154
|
+
const gateSummary = report.gates.passed
|
|
155
|
+
? "passed"
|
|
156
|
+
: `failed (${report.gates.failures.map((failure) => failure.code).join(", ")})`;
|
|
157
|
+
const lines = [
|
|
158
|
+
`Eval suite: ${report.suite.suite_name} (${report.suite.suite_id})`,
|
|
159
|
+
`Tasks: ${report.summary.task_passed}/${report.summary.task_total} passed`,
|
|
160
|
+
`Execution errors: ${report.summary.execution_errors}`,
|
|
161
|
+
`M-001 task success: ${formatRate(metrics.m001_task_success_rate.value)}`,
|
|
162
|
+
`M-002 first-pass success: ${formatRate(metrics.m002_first_pass_success_rate.value)}`,
|
|
163
|
+
`M-003 patch apply success: ${formatRate(metrics.m003_patch_apply_success_rate.value)}`,
|
|
164
|
+
`M-004 verification pass: ${formatRate(metrics.m004_verification_pass_rate.value)}`,
|
|
165
|
+
`M-005 hallucination rate: ${formatRate(metrics.m005_hallucination_rate.value)}`,
|
|
166
|
+
`M-006 scope violation rate: ${formatRate(metrics.m006_scope_violation_rate.value)}`,
|
|
167
|
+
`M-007 latency median/p95: ${formatNullable(metrics.m007_latency_ms.median, "ms")}/${formatNullable(metrics.m007_latency_ms.p95, "ms")}`,
|
|
168
|
+
`M-008 success tokens median/p95: ${formatNullable(metrics.m008_success_tokens.median)}/${formatNullable(metrics.m008_success_tokens.p95)}`,
|
|
169
|
+
`M-008 success cost median/p95: ${formatNullable(metrics.m008_success_cost_usd.median, " USD")}/${formatNullable(metrics.m008_success_cost_usd.p95, " USD")}`,
|
|
170
|
+
`Regression gates: ${gateSummary}`,
|
|
171
|
+
`Report: ${reportPath}`,
|
|
172
|
+
];
|
|
173
|
+
// eslint-disable-next-line no-console
|
|
174
|
+
console.log(lines.join("\n"));
|
|
175
|
+
};
|
|
176
|
+
const loadBaselineReport = async (params) => {
|
|
177
|
+
const { parsed, suitePath, suiteBaselinePath, store } = params;
|
|
178
|
+
const suiteDir = path.dirname(suitePath);
|
|
179
|
+
if (parsed.baseline_path) {
|
|
180
|
+
const baselinePath = await store.resolvePath(parsed.baseline_path, process.cwd());
|
|
181
|
+
return { path: baselinePath, report: await store.read(baselinePath) };
|
|
182
|
+
}
|
|
183
|
+
if (suiteBaselinePath) {
|
|
184
|
+
const baselinePath = await store.resolvePath(suiteBaselinePath, suiteDir);
|
|
185
|
+
return { path: baselinePath, report: await store.read(baselinePath) };
|
|
186
|
+
}
|
|
187
|
+
const latest = await store.findLatestForSuite({
|
|
188
|
+
suite_fingerprint: params.suiteFingerprint,
|
|
189
|
+
exclude_report_id: params.reportId,
|
|
190
|
+
});
|
|
191
|
+
if (!latest)
|
|
192
|
+
return {};
|
|
193
|
+
return { path: latest.path, report: latest.report };
|
|
194
|
+
};
|
|
195
|
+
export class EvalCommand {
|
|
196
|
+
static helpText() {
|
|
197
|
+
return HELP_TEXT;
|
|
198
|
+
}
|
|
199
|
+
static async run(argv) {
|
|
200
|
+
const parsed = parseEvalArgs(argv);
|
|
201
|
+
if (parsed.help) {
|
|
202
|
+
// eslint-disable-next-line no-console
|
|
203
|
+
console.log(HELP_TEXT);
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
if (!parsed.suite_path) {
|
|
207
|
+
throw new EvalCommandError("Missing required --suite <path>.", EVAL_EXIT_CODES.usage_error);
|
|
208
|
+
}
|
|
209
|
+
const resolvedWorkspaceRoot = resolveWorkspaceRoot(process.cwd(), parsed.workspace_root);
|
|
210
|
+
const cliConfig = {};
|
|
211
|
+
if (parsed.workspace_root)
|
|
212
|
+
cliConfig.workspaceRoot = parsed.workspace_root;
|
|
213
|
+
if (parsed.provider)
|
|
214
|
+
cliConfig.provider = parsed.provider;
|
|
215
|
+
if (parsed.model)
|
|
216
|
+
cliConfig.model = parsed.model;
|
|
217
|
+
if (parsed.api_key)
|
|
218
|
+
cliConfig.apiKey = parsed.api_key;
|
|
219
|
+
if (parsed.base_url)
|
|
220
|
+
cliConfig.baseUrl = parsed.base_url;
|
|
221
|
+
if (parsed.workflow_profile)
|
|
222
|
+
cliConfig.workflow = { profile: parsed.workflow_profile };
|
|
223
|
+
if (parsed.smart !== undefined)
|
|
224
|
+
cliConfig.smart = parsed.smart;
|
|
225
|
+
if (parsed.no_deep_investigation !== undefined) {
|
|
226
|
+
cliConfig.deepInvestigation = { enabled: !parsed.no_deep_investigation };
|
|
227
|
+
}
|
|
228
|
+
const config = await loadConfig({
|
|
229
|
+
cli: cliConfig,
|
|
230
|
+
cwd: resolvedWorkspaceRoot,
|
|
231
|
+
configPath: parsed.config_path,
|
|
232
|
+
});
|
|
233
|
+
let loadedSuite;
|
|
234
|
+
try {
|
|
235
|
+
loadedSuite = await loadSuiteFromFile(parsed.suite_path, process.cwd());
|
|
236
|
+
}
|
|
237
|
+
catch (error) {
|
|
238
|
+
if (error instanceof SuiteValidationError) {
|
|
239
|
+
const issueSummary = error.issues
|
|
240
|
+
.map((issue) => `${issue.path}:${issue.code}`)
|
|
241
|
+
.join(", ");
|
|
242
|
+
throw new EvalCommandError(`Suite validation failed: ${issueSummary}`, EVAL_EXIT_CODES.suite_validation_error);
|
|
243
|
+
}
|
|
244
|
+
throw error;
|
|
245
|
+
}
|
|
246
|
+
const executor = new EvalTaskExecutor({
|
|
247
|
+
workspace_root: config.workspaceRoot,
|
|
248
|
+
suite_dir: loadedSuite.suite_dir,
|
|
249
|
+
provider: parsed.provider ?? (config.provider || undefined),
|
|
250
|
+
model: parsed.model ?? (config.model || undefined),
|
|
251
|
+
api_key: parsed.api_key ?? config.apiKey,
|
|
252
|
+
base_url: parsed.base_url ?? config.baseUrl,
|
|
253
|
+
agent: parsed.agent,
|
|
254
|
+
agent_id: parsed.agent_id,
|
|
255
|
+
agent_slug: parsed.agent_slug,
|
|
256
|
+
workflow_profile: parsed.workflow_profile,
|
|
257
|
+
smart: parsed.smart,
|
|
258
|
+
no_deep_investigation: parsed.no_deep_investigation,
|
|
259
|
+
});
|
|
260
|
+
const runner = new EvalRunner({
|
|
261
|
+
suite_id: loadedSuite.suite.suite_id,
|
|
262
|
+
suite_fingerprint: loadedSuite.suite_fingerprint,
|
|
263
|
+
tasks: loadedSuite.suite.tasks,
|
|
264
|
+
executor,
|
|
265
|
+
});
|
|
266
|
+
const runResult = await runner.run();
|
|
267
|
+
const metrics = aggregateMetrics(runResult);
|
|
268
|
+
const reportId = randomUUID();
|
|
269
|
+
const reportStore = new ReportStore(config.workspaceRoot, parsed.report_dir ?? config.eval?.report_dir);
|
|
270
|
+
const suiteBaselinePath = loadedSuite.suite.baseline?.mode === "none"
|
|
271
|
+
? undefined
|
|
272
|
+
: loadedSuite.suite.baseline?.report_path;
|
|
273
|
+
const baseline = await loadBaselineReport({
|
|
274
|
+
parsed,
|
|
275
|
+
suitePath: loadedSuite.suite_path,
|
|
276
|
+
suiteBaselinePath,
|
|
277
|
+
store: reportStore,
|
|
278
|
+
suiteFingerprint: loadedSuite.suite_fingerprint,
|
|
279
|
+
reportId,
|
|
280
|
+
});
|
|
281
|
+
const regression = compareAgainstBaseline({
|
|
282
|
+
current: metrics,
|
|
283
|
+
baseline: baseline.report?.metrics,
|
|
284
|
+
baseline_report_id: baseline.report?.report_id,
|
|
285
|
+
baseline_created_at: baseline.report?.created_at,
|
|
286
|
+
});
|
|
287
|
+
const thresholds = resolveGateThresholds(config.eval?.gates, loadedSuite.suite.thresholds);
|
|
288
|
+
const gates = evaluateGates({ metrics, thresholds, comparison: regression });
|
|
289
|
+
const hasRunFailures = runResult.summary.failed > 0 || runResult.summary.execution_errors > 0;
|
|
290
|
+
const exitCode = hasRunFailures
|
|
291
|
+
? EVAL_EXIT_CODES.run_failure
|
|
292
|
+
: (!gates.passed ? EVAL_EXIT_CODES.gate_failure : 0);
|
|
293
|
+
const report = {
|
|
294
|
+
schema_version: 1,
|
|
295
|
+
report_id: reportId,
|
|
296
|
+
created_at: new Date().toISOString(),
|
|
297
|
+
suite: {
|
|
298
|
+
suite_id: loadedSuite.suite.suite_id,
|
|
299
|
+
suite_name: loadedSuite.suite.name,
|
|
300
|
+
suite_path: loadedSuite.suite_path,
|
|
301
|
+
suite_fingerprint: loadedSuite.suite_fingerprint,
|
|
302
|
+
task_count: loadedSuite.suite.tasks.length,
|
|
303
|
+
},
|
|
304
|
+
summary: {
|
|
305
|
+
exit_code: exitCode,
|
|
306
|
+
passed: exitCode === 0,
|
|
307
|
+
gate_passed: gates.passed,
|
|
308
|
+
task_total: runResult.summary.total,
|
|
309
|
+
task_passed: runResult.summary.passed,
|
|
310
|
+
task_failed: runResult.summary.failed,
|
|
311
|
+
execution_errors: runResult.summary.execution_errors,
|
|
312
|
+
},
|
|
313
|
+
run: runResult,
|
|
314
|
+
metrics,
|
|
315
|
+
regression,
|
|
316
|
+
gates,
|
|
317
|
+
};
|
|
318
|
+
const reportPath = await reportStore.save(report);
|
|
319
|
+
if (parsed.output === "json") {
|
|
320
|
+
// eslint-disable-next-line no-console
|
|
321
|
+
console.log(serializeEvalReport(report, true));
|
|
322
|
+
}
|
|
323
|
+
else {
|
|
324
|
+
printTextReport(report, reportPath);
|
|
325
|
+
}
|
|
326
|
+
if (exitCode === EVAL_EXIT_CODES.run_failure) {
|
|
327
|
+
throw new EvalCommandError("Eval run completed with failed tasks or execution errors.", EVAL_EXIT_CODES.run_failure);
|
|
328
|
+
}
|
|
329
|
+
if (exitCode === EVAL_EXIT_CODES.gate_failure) {
|
|
330
|
+
throw new EvalCommandError("Eval regression gates failed.", EVAL_EXIT_CODES.gate_failure);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { MemoryWriteback } from "../cognitive/MemoryWriteback.js";
|
|
2
|
+
import { PostMortemAnalyzer } from "../cognitive/PostMortemAnalyzer.js";
|
|
3
|
+
import type { CodaliConfig } from "../config/Config.js";
|
|
4
|
+
import { DocdexClient } from "../docdex/DocdexClient.js";
|
|
5
|
+
import { createProvider } from "../providers/ProviderRegistry.js";
|
|
6
|
+
import type { Provider } from "../providers/ProviderTypes.js";
|
|
7
|
+
interface FeedbackCommandDependencies {
|
|
8
|
+
cwd?: () => string;
|
|
9
|
+
loadConfig?: (options: {
|
|
10
|
+
cwd: string;
|
|
11
|
+
}) => Promise<CodaliConfig>;
|
|
12
|
+
createDocdexClient?: (config: CodaliConfig) => DocdexClient;
|
|
13
|
+
createProvider?: typeof createProvider;
|
|
14
|
+
createMemoryWriteback?: (client: DocdexClient, options: ConstructorParameters<typeof MemoryWriteback>[1]) => Pick<MemoryWriteback, "persist">;
|
|
15
|
+
createAnalyzer?: (provider: Provider, workspaceRoot: string) => Pick<PostMortemAnalyzer, "analyze">;
|
|
16
|
+
log?: (line: string) => void;
|
|
17
|
+
}
|
|
18
|
+
export declare class FeedbackCommand {
|
|
19
|
+
static run(argv: string[], deps?: FeedbackCommandDependencies): Promise<void>;
|
|
20
|
+
}
|
|
21
|
+
export {};
|
|
22
|
+
//# sourceMappingURL=FeedbackCommand.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FeedbackCommand.d.ts","sourceRoot":"","sources":["../../src/cli/FeedbackCommand.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAA6B,MAAM,iCAAiC,CAAC;AAC7F,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAExD,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAClE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AAsB9D,UAAU,2BAA2B;IACnC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC;IACjE,kBAAkB,CAAC,EAAE,CAAC,MAAM,EAAE,YAAY,KAAK,YAAY,CAAC;IAC5D,cAAc,CAAC,EAAE,OAAO,cAAc,CAAC;IACvC,qBAAqB,CAAC,EAAE,CACtB,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,eAAe,CAAC,CAAC,CAAC,CAAC,KACtD,IAAI,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;IACtC,cAAc,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,KAAK,IAAI,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAC;IACpG,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CAC9B;AA2DD,qBAAa,eAAe;WACb,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,GAAE,2BAAgC,GAAG,OAAO,CAAC,IAAI,CAAC;CAqGxF"}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import process from "node:process";
|
|
3
|
+
import { MemoryWriteback } from "../cognitive/MemoryWriteback.js";
|
|
4
|
+
import { PostMortemAnalyzer } from "../cognitive/PostMortemAnalyzer.js";
|
|
5
|
+
import { loadConfig } from "../config/ConfigLoader.js";
|
|
6
|
+
import { DocdexClient } from "../docdex/DocdexClient.js";
|
|
7
|
+
import { createProvider } from "../providers/ProviderRegistry.js";
|
|
8
|
+
import { resolveWorkspaceRoot } from "./RunCommand.js";
|
|
9
|
+
const USAGE = "Usage: codali learn --file <path/to/file> [--confirm <dedupe_key> ...]\n"
|
|
10
|
+
+ " or: codali learn --confirm <dedupe_key> [--confirm <dedupe_key> ...]";
|
|
11
|
+
class FeedbackCommandError extends Error {
|
|
12
|
+
constructor(message, exitCode = 1) {
|
|
13
|
+
super(message);
|
|
14
|
+
this.name = "FeedbackCommandError";
|
|
15
|
+
this.exitCode = exitCode;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
const parseArgs = (argv) => {
|
|
19
|
+
const parsed = { confirmKeys: [] };
|
|
20
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
21
|
+
const arg = argv[index];
|
|
22
|
+
const next = argv[index + 1];
|
|
23
|
+
if (arg === "--file" && next) {
|
|
24
|
+
parsed.filePath = next;
|
|
25
|
+
index += 1;
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
28
|
+
if (arg === "--confirm" && next) {
|
|
29
|
+
parsed.confirmKeys.push(next);
|
|
30
|
+
index += 1;
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
if (!arg.startsWith("-") && !parsed.filePath) {
|
|
34
|
+
parsed.filePath = arg;
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
parsed.confirmKeys = Array.from(new Set(parsed.confirmKeys
|
|
39
|
+
.map((entry) => entry.trim())
|
|
40
|
+
.filter(Boolean)));
|
|
41
|
+
return parsed;
|
|
42
|
+
};
|
|
43
|
+
const formatOutcome = (outcome) => {
|
|
44
|
+
const key = outcome.dedupe_key ? ` (${outcome.dedupe_key})` : "";
|
|
45
|
+
const score = outcome.confidence_score !== undefined
|
|
46
|
+
? ` score=${outcome.confidence_score.toFixed(2)}`
|
|
47
|
+
: "";
|
|
48
|
+
const lifecycle = outcome.lifecycle_state ? ` lifecycle=${outcome.lifecycle_state}` : "";
|
|
49
|
+
return `- ${outcome.status}:${outcome.code}${key}${lifecycle}${score} - ${outcome.message}`;
|
|
50
|
+
};
|
|
51
|
+
const summarizeOutcomes = (outcomes) => {
|
|
52
|
+
const summary = {
|
|
53
|
+
accepted: 0,
|
|
54
|
+
promoted: 0,
|
|
55
|
+
deferred: 0,
|
|
56
|
+
suppressed: 0,
|
|
57
|
+
rejected: 0,
|
|
58
|
+
};
|
|
59
|
+
for (const outcome of outcomes) {
|
|
60
|
+
if (outcome.status === "accepted")
|
|
61
|
+
summary.accepted += 1;
|
|
62
|
+
if (outcome.status === "promoted")
|
|
63
|
+
summary.promoted += 1;
|
|
64
|
+
if (outcome.status === "deferred")
|
|
65
|
+
summary.deferred += 1;
|
|
66
|
+
if (outcome.status === "suppressed")
|
|
67
|
+
summary.suppressed += 1;
|
|
68
|
+
if (outcome.status === "rejected")
|
|
69
|
+
summary.rejected += 1;
|
|
70
|
+
}
|
|
71
|
+
return summary;
|
|
72
|
+
};
|
|
73
|
+
export class FeedbackCommand {
|
|
74
|
+
static async run(argv, deps = {}) {
|
|
75
|
+
const args = parseArgs(argv);
|
|
76
|
+
if (!args.filePath && args.confirmKeys.length === 0) {
|
|
77
|
+
throw new FeedbackCommandError(USAGE, 2);
|
|
78
|
+
}
|
|
79
|
+
const log = deps.log ?? ((line) => console.log(line)); // eslint-disable-line no-console
|
|
80
|
+
const cwd = deps.cwd?.() ?? process.cwd();
|
|
81
|
+
const workspaceRoot = resolveWorkspaceRoot(cwd);
|
|
82
|
+
const configLoader = deps.loadConfig ?? loadConfig;
|
|
83
|
+
const config = await configLoader({ cwd: workspaceRoot });
|
|
84
|
+
const docdexClient = deps.createDocdexClient
|
|
85
|
+
? deps.createDocdexClient(config)
|
|
86
|
+
: new DocdexClient({
|
|
87
|
+
baseUrl: config.docdex.baseUrl,
|
|
88
|
+
repoId: config.docdex.repoId,
|
|
89
|
+
repoRoot: config.docdex.repoRoot ?? config.workspaceRoot,
|
|
90
|
+
});
|
|
91
|
+
const memoryWriteback = deps.createMemoryWriteback
|
|
92
|
+
? deps.createMemoryWriteback(docdexClient, {
|
|
93
|
+
agentId: config.agentId ?? "codali",
|
|
94
|
+
workspaceRoot,
|
|
95
|
+
learning: config.learning,
|
|
96
|
+
})
|
|
97
|
+
: new MemoryWriteback(docdexClient, {
|
|
98
|
+
agentId: config.agentId ?? "codali",
|
|
99
|
+
workspaceRoot,
|
|
100
|
+
learning: config.learning,
|
|
101
|
+
});
|
|
102
|
+
if (args.confirmKeys.length > 0) {
|
|
103
|
+
const promotionResult = await memoryWriteback.persist({
|
|
104
|
+
failures: 0,
|
|
105
|
+
maxRetries: 0,
|
|
106
|
+
lesson: "",
|
|
107
|
+
promotions: args.confirmKeys.map((dedupe_key) => ({
|
|
108
|
+
dedupe_key,
|
|
109
|
+
agentId: config.agentId ?? "codali",
|
|
110
|
+
})),
|
|
111
|
+
});
|
|
112
|
+
const promotionSummary = summarizeOutcomes(promotionResult.outcomes);
|
|
113
|
+
log(`Promotion outcomes (${promotionResult.outcomes.length}):`);
|
|
114
|
+
for (const outcome of promotionResult.outcomes) {
|
|
115
|
+
log(formatOutcome(outcome));
|
|
116
|
+
}
|
|
117
|
+
log(`Summary: promoted=${promotionSummary.promoted} suppressed=${promotionSummary.suppressed} rejected=${promotionSummary.rejected}`);
|
|
118
|
+
if (promotionSummary.rejected > 0) {
|
|
119
|
+
throw new FeedbackCommandError("One or more promotions failed.", 3);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (!args.filePath) {
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
const providerName = config.routing?.architect?.provider ?? config.provider;
|
|
126
|
+
const providerModel = config.routing?.architect?.model ?? config.model;
|
|
127
|
+
if (!providerName || !providerModel) {
|
|
128
|
+
throw new FeedbackCommandError("No provider/model configured. Use --provider/--model or configure codali.");
|
|
129
|
+
}
|
|
130
|
+
const providerFactory = deps.createProvider ?? createProvider;
|
|
131
|
+
const provider = providerFactory(providerName, {
|
|
132
|
+
model: providerModel,
|
|
133
|
+
apiKey: config.apiKey,
|
|
134
|
+
baseUrl: config.baseUrl,
|
|
135
|
+
});
|
|
136
|
+
const analyzer = deps.createAnalyzer
|
|
137
|
+
? deps.createAnalyzer(provider, workspaceRoot)
|
|
138
|
+
: new PostMortemAnalyzer(provider, workspaceRoot);
|
|
139
|
+
const absoluteFilePath = path.resolve(workspaceRoot, args.filePath);
|
|
140
|
+
log(`Analyzing history for ${absoluteFilePath}...`);
|
|
141
|
+
const analysis = await analyzer.analyze(absoluteFilePath);
|
|
142
|
+
if (analysis.status === "no_change" || analysis.rules.length === 0) {
|
|
143
|
+
log(`No rule persisted: ${analysis.message}`);
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
const writebackResult = await memoryWriteback.persist({
|
|
147
|
+
failures: 0,
|
|
148
|
+
maxRetries: config.limits.maxRetries,
|
|
149
|
+
lesson: "",
|
|
150
|
+
rules: analysis.rules,
|
|
151
|
+
});
|
|
152
|
+
const summary = summarizeOutcomes(writebackResult.outcomes);
|
|
153
|
+
log(`Extracted rule: "${analysis.rule ?? analysis.rules[0]?.content ?? ""}"`);
|
|
154
|
+
log(`Learning outcomes (${writebackResult.outcomes.length}):`);
|
|
155
|
+
for (const outcome of writebackResult.outcomes) {
|
|
156
|
+
log(formatOutcome(outcome));
|
|
157
|
+
}
|
|
158
|
+
log(`Summary: accepted=${summary.accepted} deferred=${summary.deferred} suppressed=${summary.suppressed} rejected=${summary.rejected}`);
|
|
159
|
+
if (summary.rejected > 0) {
|
|
160
|
+
throw new FeedbackCommandError("Learning writeback failed for one or more rules.", 4);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import type { ToolConfig } from "../config/Config.js";
|
|
2
|
+
import { type PipelinePhase } from "../cognitive/ProviderRouting.js";
|
|
3
|
+
import { type PhaseAgentSelection } from "../agents/PhaseAgentSelector.js";
|
|
4
|
+
interface ParsedArgs {
|
|
5
|
+
workspaceRoot?: string;
|
|
6
|
+
project?: string;
|
|
7
|
+
command?: string;
|
|
8
|
+
commandRunId?: string;
|
|
9
|
+
jobId?: string;
|
|
10
|
+
runId?: string;
|
|
11
|
+
taskId?: string;
|
|
12
|
+
taskKey?: string;
|
|
13
|
+
agent?: string;
|
|
14
|
+
agentId?: string;
|
|
15
|
+
agentSlug?: string;
|
|
16
|
+
agentLibrarian?: string;
|
|
17
|
+
agentArchitect?: string;
|
|
18
|
+
agentBuilder?: string;
|
|
19
|
+
agentCritic?: string;
|
|
20
|
+
agentInterpreter?: string;
|
|
21
|
+
smart?: boolean;
|
|
22
|
+
deepInvestigationEnabled?: boolean;
|
|
23
|
+
planHint?: string;
|
|
24
|
+
provider?: string;
|
|
25
|
+
model?: string;
|
|
26
|
+
apiKey?: string;
|
|
27
|
+
baseUrl?: string;
|
|
28
|
+
workflowProfile?: string;
|
|
29
|
+
taskFile?: string;
|
|
30
|
+
configPath?: string;
|
|
31
|
+
docdexBaseUrl?: string;
|
|
32
|
+
docdexRepoId?: string;
|
|
33
|
+
docdexRepoRoot?: string;
|
|
34
|
+
contextMode?: "bundle_text" | "json";
|
|
35
|
+
contextMaxFiles?: number;
|
|
36
|
+
contextMaxTotalBytes?: number;
|
|
37
|
+
contextTokenBudget?: number;
|
|
38
|
+
contextFocusMaxBytes?: number;
|
|
39
|
+
contextPeripheryMaxBytes?: number;
|
|
40
|
+
contextIncludeRepoMap?: boolean;
|
|
41
|
+
contextIncludeImpact?: boolean;
|
|
42
|
+
contextIncludeSnippets?: boolean;
|
|
43
|
+
contextReadStrategy?: "docdex" | "fs";
|
|
44
|
+
contextMaxRefreshes?: number;
|
|
45
|
+
contextSkeletonize?: boolean;
|
|
46
|
+
contextRedactSecrets?: boolean;
|
|
47
|
+
contextIgnoreFilesFrom?: string[];
|
|
48
|
+
securityRedactPatterns?: string[];
|
|
49
|
+
builderMode?: "tool_calls" | "patch_json" | "freeform";
|
|
50
|
+
builderPatchFormat?: "search_replace" | "file_writes";
|
|
51
|
+
interpreterProvider?: string;
|
|
52
|
+
interpreterModel?: string;
|
|
53
|
+
interpreterFormat?: string;
|
|
54
|
+
interpreterGrammar?: string;
|
|
55
|
+
interpreterMaxRetries?: number;
|
|
56
|
+
interpreterTimeoutMs?: number;
|
|
57
|
+
streamingFlushMs?: number;
|
|
58
|
+
costMaxPerRun?: number;
|
|
59
|
+
costCharPerToken?: number;
|
|
60
|
+
costPricingOverrides?: string;
|
|
61
|
+
inlineTask?: string;
|
|
62
|
+
}
|
|
63
|
+
export declare const parseArgs: (argv: string[]) => ParsedArgs;
|
|
64
|
+
export declare const resolveWorkspaceRoot: (cwd: string, explicitRoot?: string) => string;
|
|
65
|
+
export declare const isToolEnabled: (name: string, tools: ToolConfig) => boolean;
|
|
66
|
+
export declare const assessPhaseFallbackSuitability: (phase: PipelinePhase, builderMode: "tool_calls" | "patch_json" | "freeform", selection: Pick<PhaseAgentSelection, "capabilities"> & {
|
|
67
|
+
supportsTools?: boolean;
|
|
68
|
+
}) => {
|
|
69
|
+
ok: boolean;
|
|
70
|
+
reason: string;
|
|
71
|
+
details?: Record<string, number | string>;
|
|
72
|
+
builderMode?: "tool_calls" | "patch_json" | "freeform";
|
|
73
|
+
};
|
|
74
|
+
export declare class RunCommand {
|
|
75
|
+
static run(argv: string[]): Promise<void>;
|
|
76
|
+
}
|
|
77
|
+
export {};
|
|
78
|
+
//# sourceMappingURL=RunCommand.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RunCommand.d.ts","sourceRoot":"","sources":["../../src/cli/RunCommand.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAA2B,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAsC/E,OAAO,EAAuB,KAAK,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAc1F,OAAO,EAAqB,KAAK,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AAQ9F,UAAU,UAAU;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC;IACrC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,mBAAmB,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC;IACtC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,sBAAsB,CAAC,EAAE,MAAM,EAAE,CAAC;IAClC,sBAAsB,CAAC,EAAE,MAAM,EAAE,CAAC;IAClC,WAAW,CAAC,EAAE,YAAY,GAAG,YAAY,GAAG,UAAU,CAAC;IACvD,kBAAkB,CAAC,EAAE,gBAAgB,GAAG,aAAa,CAAC;IACtD,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,eAAO,MAAM,SAAS,GAAI,MAAM,MAAM,EAAE,KAAG,UA2T1C,CAAC;AAqBF,eAAO,MAAM,oBAAoB,GAC/B,KAAK,MAAM,EACX,eAAe,MAAM,KACpB,MAgBF,CAAC;AAkPF,eAAO,MAAM,aAAa,GAAI,MAAM,MAAM,EAAE,OAAO,UAAU,KAAG,OAO/D,CAAC;AAsBF,eAAO,MAAM,8BAA8B,GACzC,OAAO,aAAa,EACpB,aAAa,YAAY,GAAG,YAAY,GAAG,UAAU,EACrD,WAAW,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,GAAG;IAAE,aAAa,CAAC,EAAE,OAAO,CAAA;CAAE,KACjF;IACD,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IAC1C,WAAW,CAAC,EAAE,YAAY,GAAG,YAAY,GAAG,UAAU,CAAC;CAyCxD,CAAC;AA8WF,qBAAa,UAAU;WACR,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAi0ChD"}
|