@mhingston5/lasso 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +707 -0
- package/docs/agent-wrangling.png +0 -0
- package/package.json +26 -0
- package/src/capabilities/matcher.ts +25 -0
- package/src/capabilities/registry.ts +103 -0
- package/src/capabilities/types.ts +15 -0
- package/src/cir/lower.ts +253 -0
- package/src/cir/optimize.ts +251 -0
- package/src/cir/types.ts +131 -0
- package/src/cir/validate.ts +265 -0
- package/src/compiler/compile.ts +601 -0
- package/src/compiler/feedback.ts +471 -0
- package/src/compiler/runtime-helpers.ts +455 -0
- package/src/composition/chain.ts +58 -0
- package/src/composition/conditional.ts +76 -0
- package/src/composition/parallel.ts +75 -0
- package/src/composition/types.ts +105 -0
- package/src/environment/analyzer.ts +56 -0
- package/src/environment/discovery.ts +179 -0
- package/src/environment/types.ts +68 -0
- package/src/failures/classifiers.ts +134 -0
- package/src/failures/generator.ts +421 -0
- package/src/failures/map-reference-failures.ts +23 -0
- package/src/failures/ontology.ts +210 -0
- package/src/failures/recovery.ts +214 -0
- package/src/failures/types.ts +14 -0
- package/src/index.ts +67 -0
- package/src/memory/advisor.ts +132 -0
- package/src/memory/extractor.ts +166 -0
- package/src/memory/store.ts +107 -0
- package/src/memory/types.ts +53 -0
- package/src/metaharness/engine.ts +256 -0
- package/src/metaharness/predictor.ts +168 -0
- package/src/metaharness/types.ts +40 -0
- package/src/mutation/derive.ts +308 -0
- package/src/mutation/diff.ts +52 -0
- package/src/mutation/engine.ts +256 -0
- package/src/mutation/types.ts +84 -0
- package/src/pi/command-input.ts +209 -0
- package/src/pi/commands.ts +351 -0
- package/src/pi/extension.ts +16 -0
- package/src/planner/synthesize.ts +83 -0
- package/src/planner/template-rules.ts +183 -0
- package/src/planner/types.ts +42 -0
- package/src/reference/catalog.ts +128 -0
- package/src/reference/patch-validation-strategies.ts +170 -0
- package/src/reference/patch-validation.ts +174 -0
- package/src/reference/pr-review-merge.ts +155 -0
- package/src/reference/strategies.ts +126 -0
- package/src/reference/types.ts +33 -0
- package/src/replanner/risk-rules.ts +161 -0
- package/src/replanner/runtime.ts +308 -0
- package/src/replanner/synthesize.ts +619 -0
- package/src/replanner/types.ts +73 -0
- package/src/spec/schema.ts +254 -0
- package/src/spec/types.ts +319 -0
- package/src/spec/validate.ts +296 -0
- package/src/state/snapshots.ts +43 -0
- package/src/state/types.ts +12 -0
- package/src/synthesis/graph-builder.ts +267 -0
- package/src/synthesis/harness-builder.ts +113 -0
- package/src/synthesis/intent-ir.ts +63 -0
- package/src/synthesis/policy-builder.ts +320 -0
- package/src/synthesis/risk-analyzer.ts +182 -0
- package/src/synthesis/skill-parser.ts +441 -0
- package/src/verification/engine.ts +230 -0
- package/src/versioning/file-store.ts +103 -0
- package/src/versioning/history.ts +43 -0
- package/src/versioning/store.ts +16 -0
- package/src/versioning/types.ts +31 -0
- package/test/capabilities/matcher.test.ts +67 -0
- package/test/capabilities/registry.test.ts +136 -0
- package/test/capabilities/synthesis.test.ts +264 -0
- package/test/cir/lower.test.ts +417 -0
- package/test/cir/optimize.test.ts +266 -0
- package/test/cir/validate.test.ts +368 -0
- package/test/compiler/adaptive-runtime.test.ts +157 -0
- package/test/compiler/compile.test.ts +1198 -0
- package/test/compiler/feedback.test.ts +784 -0
- package/test/compiler/guardrails.test.ts +191 -0
- package/test/compiler/trace.test.ts +404 -0
- package/test/composition/chain.test.ts +328 -0
- package/test/composition/conditional.test.ts +241 -0
- package/test/composition/parallel.test.ts +215 -0
- package/test/environment/analyzer.test.ts +204 -0
- package/test/environment/discovery.test.ts +149 -0
- package/test/failures/classifiers.test.ts +287 -0
- package/test/failures/generator.test.ts +203 -0
- package/test/failures/ontology.test.ts +439 -0
- package/test/failures/recovery.test.ts +300 -0
- package/test/helpers/createFixtureRepo.ts +84 -0
- package/test/helpers/createPatchValidationFixture.ts +144 -0
- package/test/helpers/runCompiledWorkflow.ts +208 -0
- package/test/memory/advisor.test.ts +332 -0
- package/test/memory/extractor.test.ts +295 -0
- package/test/memory/store.test.ts +244 -0
- package/test/metaharness/engine.test.ts +575 -0
- package/test/metaharness/predictor.test.ts +436 -0
- package/test/mutation/derive-failure.test.ts +209 -0
- package/test/mutation/engine.test.ts +622 -0
- package/test/package-smoke.test.ts +29 -0
- package/test/pi/command-input.test.ts +153 -0
- package/test/pi/commands.test.ts +623 -0
- package/test/planner/classify-template.test.ts +32 -0
- package/test/planner/synthesize.test.ts +901 -0
- package/test/reference/PatchValidation.failures.test.ts +137 -0
- package/test/reference/PatchValidation.test.ts +326 -0
- package/test/reference/PrReviewMerge.failures.test.ts +121 -0
- package/test/reference/PrReviewMerge.test.ts +55 -0
- package/test/reference/catalog-open.test.ts +70 -0
- package/test/replanner/runtime.test.ts +207 -0
- package/test/replanner/synthesize.test.ts +303 -0
- package/test/spec/validate.test.ts +1056 -0
- package/test/state/snapshots.test.ts +264 -0
- package/test/synthesis/custom-workflow.test.ts +264 -0
- package/test/synthesis/graph-builder.test.ts +370 -0
- package/test/synthesis/harness-builder.test.ts +128 -0
- package/test/synthesis/policy-builder.test.ts +149 -0
- package/test/synthesis/risk-analyzer.test.ts +230 -0
- package/test/synthesis/skill-parser.test.ts +796 -0
- package/test/verification/engine.test.ts +509 -0
- package/test/versioning/history.test.ts +144 -0
- package/test/versioning/store.test.ts +254 -0
- package/vitest.config.ts +9 -0
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import type { FailureSignature, FailureContext } from "./ontology.js";
|
|
2
|
+
|
|
3
|
+
export interface RecoveryStep {
|
|
4
|
+
action: string;
|
|
5
|
+
description: string;
|
|
6
|
+
automated: boolean;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export interface RecoveryPlan {
|
|
10
|
+
steps: RecoveryStep[];
|
|
11
|
+
estimatedSuccessRate: number;
|
|
12
|
+
requiresHumanApproval: boolean;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
const AUTH_RECOVERY_STEPS: RecoveryStep[] = [
|
|
16
|
+
{
|
|
17
|
+
action: "Refresh authentication credentials",
|
|
18
|
+
description: "Attempt to refresh or re-obtain the expired/invalid credentials",
|
|
19
|
+
automated: false,
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
action: "Verify API key configuration",
|
|
23
|
+
description: "Check that API keys and secrets are correctly set in the environment",
|
|
24
|
+
automated: true,
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
action: "Check token expiry",
|
|
28
|
+
description: "Inspect the current token's expiry time and renew if necessary",
|
|
29
|
+
automated: true,
|
|
30
|
+
},
|
|
31
|
+
];
|
|
32
|
+
|
|
33
|
+
const TOOL_RECOVERY_STEPS: RecoveryStep[] = [
|
|
34
|
+
{
|
|
35
|
+
action: "Check tool availability",
|
|
36
|
+
description: "Verify the required tool is installed and accessible in PATH",
|
|
37
|
+
automated: true,
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
action: "Install missing tool",
|
|
41
|
+
description: "Install the required tool if it is not present",
|
|
42
|
+
automated: false,
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
action: "Verify tool version",
|
|
46
|
+
description: "Check that the tool version is compatible with expected version",
|
|
47
|
+
automated: true,
|
|
48
|
+
},
|
|
49
|
+
];
|
|
50
|
+
|
|
51
|
+
const RESOURCE_RECOVERY_STEPS: RecoveryStep[] = [
|
|
52
|
+
{
|
|
53
|
+
action: "Check disk space",
|
|
54
|
+
description: "Verify available disk space and clean up temporary files if needed",
|
|
55
|
+
automated: true,
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
action: "Free up memory",
|
|
59
|
+
description: "Release unused memory or increase memory limits",
|
|
60
|
+
automated: false,
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
action: "Apply rate limiting",
|
|
64
|
+
description: "Implement backoff strategy to avoid hitting rate limits",
|
|
65
|
+
automated: true,
|
|
66
|
+
},
|
|
67
|
+
];
|
|
68
|
+
|
|
69
|
+
const SEMANTIC_RECOVERY_STEPS: RecoveryStep[] = [
|
|
70
|
+
{
|
|
71
|
+
action: "Validate output format",
|
|
72
|
+
description: "Check the output against the expected schema or format",
|
|
73
|
+
automated: true,
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
action: "Review input data",
|
|
77
|
+
description: "Verify input data is correct and complete",
|
|
78
|
+
automated: true,
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
action: "Add validation step",
|
|
82
|
+
description: "Insert a validation node before processing to catch format errors early",
|
|
83
|
+
automated: false,
|
|
84
|
+
},
|
|
85
|
+
];
|
|
86
|
+
|
|
87
|
+
const HUMAN_RECOVERY_STEPS: RecoveryStep[] = [
|
|
88
|
+
{
|
|
89
|
+
action: "Escalate to human operator",
|
|
90
|
+
description: "Notify a human operator to review and make a decision",
|
|
91
|
+
automated: false,
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
action: "Provide additional context",
|
|
95
|
+
description: "Gather more information to help the human make an informed decision",
|
|
96
|
+
automated: true,
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
action: "Reconsider proposed changes",
|
|
100
|
+
description: "Review and potentially modify the changes that were rejected",
|
|
101
|
+
automated: false,
|
|
102
|
+
},
|
|
103
|
+
];
|
|
104
|
+
|
|
105
|
+
const ENVIRONMENT_DRIFT_RECOVERY_STEPS: RecoveryStep[] = [
|
|
106
|
+
{
|
|
107
|
+
action: "Sync environment dependencies",
|
|
108
|
+
description: "Install or update dependencies to match expected versions",
|
|
109
|
+
automated: false,
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
action: "Verify configuration",
|
|
113
|
+
description: "Check that environment configuration matches expected state",
|
|
114
|
+
automated: true,
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
action: "Update version requirements",
|
|
118
|
+
description: "Align version requirements with the actual environment",
|
|
119
|
+
automated: false,
|
|
120
|
+
},
|
|
121
|
+
];
|
|
122
|
+
|
|
123
|
+
const NETWORK_RECOVERY_STEPS: RecoveryStep[] = [
|
|
124
|
+
{
|
|
125
|
+
action: "Retry with exponential backoff",
|
|
126
|
+
description: "Retry the failed operation with increasing delays between attempts",
|
|
127
|
+
automated: true,
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
action: "Check network connectivity",
|
|
131
|
+
description: "Verify DNS resolution and network reachability to the target",
|
|
132
|
+
automated: true,
|
|
133
|
+
},
|
|
134
|
+
{
|
|
135
|
+
action: "Verify target service health",
|
|
136
|
+
description: "Check if the target service is up and responding",
|
|
137
|
+
automated: true,
|
|
138
|
+
},
|
|
139
|
+
];
|
|
140
|
+
|
|
141
|
+
const UNKNOWN_RECOVERY_STEPS: RecoveryStep[] = [
|
|
142
|
+
{
|
|
143
|
+
action: "Collect diagnostic information",
|
|
144
|
+
description: "Gather logs, metrics, and traces to understand the failure",
|
|
145
|
+
automated: true,
|
|
146
|
+
},
|
|
147
|
+
{
|
|
148
|
+
action: "Review error logs",
|
|
149
|
+
description: "Examine detailed error logs for root cause clues",
|
|
150
|
+
automated: true,
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
action: "Escalate to human operator",
|
|
154
|
+
description: "If automated diagnostics fail, escalate for human investigation",
|
|
155
|
+
automated: false,
|
|
156
|
+
},
|
|
157
|
+
];
|
|
158
|
+
|
|
159
|
+
const CLASS_RECOVERY_MAP: Record<FailureSignature["class"], RecoveryStep[]> = {
|
|
160
|
+
auth: AUTH_RECOVERY_STEPS,
|
|
161
|
+
tool: TOOL_RECOVERY_STEPS,
|
|
162
|
+
resource: RESOURCE_RECOVERY_STEPS,
|
|
163
|
+
semantic: SEMANTIC_RECOVERY_STEPS,
|
|
164
|
+
human: HUMAN_RECOVERY_STEPS,
|
|
165
|
+
"environment-drift": ENVIRONMENT_DRIFT_RECOVERY_STEPS,
|
|
166
|
+
network: NETWORK_RECOVERY_STEPS,
|
|
167
|
+
unknown: UNKNOWN_RECOVERY_STEPS,
|
|
168
|
+
};
|
|
169
|
+
|
|
170
|
+
const CLASS_BASE_SUCCESS_RATE: Record<FailureSignature["class"], number> = {
|
|
171
|
+
auth: 0.3,
|
|
172
|
+
tool: 0.5,
|
|
173
|
+
resource: 0.4,
|
|
174
|
+
semantic: 0.3,
|
|
175
|
+
human: 0.2,
|
|
176
|
+
"environment-drift": 0.4,
|
|
177
|
+
network: 0.7,
|
|
178
|
+
unknown: 0.1,
|
|
179
|
+
};
|
|
180
|
+
|
|
181
|
+
export function suggestRecovery(
|
|
182
|
+
signature: FailureSignature,
|
|
183
|
+
context?: FailureContext,
|
|
184
|
+
): RecoveryPlan {
|
|
185
|
+
const baseSteps = CLASS_RECOVERY_MAP[signature.class];
|
|
186
|
+
const steps = [...baseSteps];
|
|
187
|
+
|
|
188
|
+
if (context?.attemptNumber !== undefined && context.attemptNumber > 3) {
|
|
189
|
+
steps.push({
|
|
190
|
+
action: "Escalate after repeated failures",
|
|
191
|
+
description: `Failure has occurred ${context.attemptNumber} times, escalate to human`,
|
|
192
|
+
automated: false,
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
const baseRate = CLASS_BASE_SUCCESS_RATE[signature.class];
|
|
197
|
+
const confidenceAdjustment = signature.confidence * 0.2;
|
|
198
|
+
const retryableBonus = signature.retryable ? 0.15 : 0;
|
|
199
|
+
const estimatedSuccessRate = Math.min(1, baseRate + confidenceAdjustment + retryableBonus);
|
|
200
|
+
|
|
201
|
+
const requiresHumanApproval =
|
|
202
|
+
signature.requiresHumanIntervention ||
|
|
203
|
+
signature.class === "semantic" ||
|
|
204
|
+
signature.class === "environment-drift" ||
|
|
205
|
+
signature.class === "unknown" ||
|
|
206
|
+
(signature.class === "resource" &&
|
|
207
|
+
signature.evidence.some((e) => e.toLowerCase().includes("memory")));
|
|
208
|
+
|
|
209
|
+
return {
|
|
210
|
+
steps,
|
|
211
|
+
estimatedSuccessRate: Math.round(estimatedSuccessRate * 100) / 100,
|
|
212
|
+
requiresHumanApproval,
|
|
213
|
+
};
|
|
214
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export interface FailureRecord {
|
|
2
|
+
domainType: string;
|
|
3
|
+
rootCause:
|
|
4
|
+
| "tool_timeout"
|
|
5
|
+
| "auth_required"
|
|
6
|
+
| "rate_limited"
|
|
7
|
+
| "invalid_output"
|
|
8
|
+
| "dependency_failure"
|
|
9
|
+
| "verification_failed"
|
|
10
|
+
| "human_block"
|
|
11
|
+
| "unknown";
|
|
12
|
+
nodeId?: string;
|
|
13
|
+
message: string;
|
|
14
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
export type { HarnessSpec, TaskNode, TaskGraph, TaskEdge, ExecutionPolicy, RetryPolicy, VerificationPolicy, HumanPolicy, ObservabilityPolicy } from "./spec/types.js";
|
|
2
|
+
export type { CirWorkflow, CirNode, CirTransition, CirExecutionPolicy } from "./cir/types.js";
|
|
3
|
+
export type { CompiledHarnessWorkflow, CompiledHarnessResult } from "./compiler/compile.js";
|
|
4
|
+
export type { LocalPrBundle } from "./reference/types.js";
|
|
5
|
+
export type { PlannerResult, WorkflowTemplate, ExtractionResult } from "./planner/types.js";
|
|
6
|
+
export type {
|
|
7
|
+
ReplanAbortReason,
|
|
8
|
+
ReplanRequest,
|
|
9
|
+
ReplanResult,
|
|
10
|
+
ReplanTrigger,
|
|
11
|
+
ReplanWorkflow,
|
|
12
|
+
RiskLevel,
|
|
13
|
+
} from "./replanner/types.js";
|
|
14
|
+
export type { FailureRecord } from "./failures/types.js";
|
|
15
|
+
export type { HarnessState } from "./state/types.js";
|
|
16
|
+
export type { IntentIR, IntentParseResult, SupportedWorkflowFamily } from "./synthesis/intent-ir.js";
|
|
17
|
+
export type { TaskGraph as SynthesisTaskGraph, WorkflowStage } from "./synthesis/graph-builder.js";
|
|
18
|
+
export type { RiskModel, StageRisk } from "./synthesis/risk-analyzer.js";
|
|
19
|
+
export type { PolicyBundle, PolicyResult } from "./synthesis/policy-builder.js";
|
|
20
|
+
export type { HarnessVersion, LineageEntry, HarnessExecutionTrace } from "./versioning/types.js";
|
|
21
|
+
export type { LineageStore, LineageFilter } from "./versioning/store.js";
|
|
22
|
+
export { FileLineageStore } from "./versioning/file-store.js";
|
|
23
|
+
export type { AdaptiveRuntimeMetadata, AdaptiveRuntimeInput, RuntimeReplanDecision } from "./replanner/runtime.js";
|
|
24
|
+
export { validateHarnessSpec } from "./spec/validate.js";
|
|
25
|
+
export { lowerHarnessSpecToCir } from "./cir/lower.js";
|
|
26
|
+
export { compileHarnessSpec } from "./compiler/compile.js";
|
|
27
|
+
export { planWorkflowRequest } from "./planner/synthesize.js";
|
|
28
|
+
export { parseReplanRequest, replanWorkflowRequest } from "./replanner/synthesize.js";
|
|
29
|
+
export { buildPrReviewMergeHarnessSpec } from "./reference/pr-review-merge.js";
|
|
30
|
+
export { createLassoCommands, clearCompiledHarnesses } from "./pi/commands.js";
|
|
31
|
+
export { classifyFailureRecord, isRetryableFailure } from "./failures/ontology.js";
|
|
32
|
+
export { mapReferenceFailure } from "./failures/map-reference-failures.js";
|
|
33
|
+
export { createHarnessState, addFailure, recordNodeResult, updateMetrics, captureSnapshot } from "./state/snapshots.js";
|
|
34
|
+
export { parsePromptOrSkill } from "./synthesis/skill-parser.js";
|
|
35
|
+
export { buildTaskGraph } from "./synthesis/graph-builder.js";
|
|
36
|
+
export { analyzeRisks } from "./synthesis/risk-analyzer.js";
|
|
37
|
+
export { synthesizePolicy } from "./synthesis/policy-builder.js";
|
|
38
|
+
export { synthesizeHarness } from "./synthesis/harness-builder.js";
|
|
39
|
+
export { createInitialVersion, createNextVersion, createLineageEntry } from "./versioning/history.js";
|
|
40
|
+
export { prepareInitialAdaptiveInput, unwrapAdaptiveInput, prepareRuntimeReplan, MAX_ADAPTIVE_VERSIONS } from "./replanner/runtime.js";
|
|
41
|
+
export type { MetaHarnessConfig, MetaHarnessResult, MetaHarness } from "./metaharness/types.js";
|
|
42
|
+
export { DefaultMetaHarness } from "./metaharness/engine.js";
|
|
43
|
+
export { predictFailuresFromEnvironment } from "./metaharness/predictor.js";
|
|
44
|
+
export { discoverEnvironment } from "./environment/discovery.js";
|
|
45
|
+
export { analyzeEnvironment } from "./environment/analyzer.js";
|
|
46
|
+
export type { EnvironmentModel, EnvironmentAnalysis, ToolCapability, Constraint } from "./environment/types.js";
|
|
47
|
+
export { classifyFailure } from "./failures/ontology.js";
|
|
48
|
+
export { suggestRecovery } from "./failures/recovery.js";
|
|
49
|
+
export type { FailureSignature, FailureClass } from "./failures/ontology.js";
|
|
50
|
+
export type { RecoveryPlan } from "./failures/recovery.js";
|
|
51
|
+
export { FileMemoryStore } from "./memory/store.js";
|
|
52
|
+
export { adviseFromMemory } from "./memory/advisor.js";
|
|
53
|
+
export { extractPatternsFromTrace } from "./memory/extractor.js";
|
|
54
|
+
export type { HarnessMemory, MemoryStore, MemoryAdvice } from "./memory/types.js";
|
|
55
|
+
export { analyzeCompiledWorkflow, applyCompilerSuggestions } from "./compiler/feedback.js";
|
|
56
|
+
export type { CostEstimate, CompilerAnalysis, CompilerSuggestion } from "./compiler/feedback.js";
|
|
57
|
+
export type { HarnessMutation, MutationType, MutationTrigger, MutationPolicy, MutationResult, SpecDiff } from "./mutation/types.js";
|
|
58
|
+
export { mutateHarness } from "./mutation/engine.js";
|
|
59
|
+
export { deriveMutationsFromTrace, deriveMutationsFromFailure } from "./mutation/derive.js";
|
|
60
|
+
export { diffSpecs } from "./mutation/diff.js";
|
|
61
|
+
export { chainHarnesses } from "./composition/chain.js";
|
|
62
|
+
export { parallelHarnesses } from "./composition/parallel.js";
|
|
63
|
+
export { conditionalHarness } from "./composition/conditional.js";
|
|
64
|
+
export type { HarnessStage, CompositionResult, HarnessComposer } from "./composition/types.js";
|
|
65
|
+
export { DefaultCapabilityRegistry } from "./capabilities/registry.js";
|
|
66
|
+
export type { Capability, CapabilityRegistry } from "./capabilities/types.js";
|
|
67
|
+
export { default } from "./pi/extension.js";
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import type { MemoryStore, MemoryQuery, MemoryAdvice, TaskSignatureOptions } from "./types.js";
|
|
2
|
+
import type { HarnessSpec } from "../spec/types.js";
|
|
3
|
+
|
|
4
|
+
export async function adviseFromMemory(
|
|
5
|
+
taskSignature: string,
|
|
6
|
+
memoryStore: MemoryStore,
|
|
7
|
+
options?: TaskSignatureOptions | HarnessSpec,
|
|
8
|
+
currentSpec?: HarnessSpec,
|
|
9
|
+
): Promise<MemoryAdvice> {
|
|
10
|
+
let queryOptions: TaskSignatureOptions | undefined;
|
|
11
|
+
let spec: HarnessSpec | undefined;
|
|
12
|
+
|
|
13
|
+
if (options && isHarnessSpec(options)) {
|
|
14
|
+
spec = options;
|
|
15
|
+
} else {
|
|
16
|
+
queryOptions = options;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
if (currentSpec) {
|
|
20
|
+
spec = currentSpec;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const query: MemoryQuery = {
|
|
24
|
+
taskSignature: queryOptions?.taskSignature,
|
|
25
|
+
minEffectiveness: queryOptions?.minEffectiveness,
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
const memories = await memoryStore.searchMemories(query);
|
|
29
|
+
|
|
30
|
+
if (memories.length === 0) {
|
|
31
|
+
return {
|
|
32
|
+
suggestions: [],
|
|
33
|
+
warnings: [],
|
|
34
|
+
sourceTaskIds: [],
|
|
35
|
+
aggregateEffectiveness: 0,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const suggestions: string[] = [];
|
|
40
|
+
const warnings: string[] = [];
|
|
41
|
+
const sourceTaskIds: string[] = [];
|
|
42
|
+
let totalEffectiveness = 0;
|
|
43
|
+
|
|
44
|
+
const existingNodeIds = spec ? new Set(spec.graph.nodes.map((n) => n.id)) : new Set<string>();
|
|
45
|
+
|
|
46
|
+
for (const memory of memories) {
|
|
47
|
+
sourceTaskIds.push(memory.taskId);
|
|
48
|
+
totalEffectiveness += memory.effectivenessScore;
|
|
49
|
+
|
|
50
|
+
for (const pattern of memory.successfulPatterns) {
|
|
51
|
+
const patternNodes = extractNodeIdsFromPattern(pattern);
|
|
52
|
+
const hasAllNodes = patternNodes.every((id) => existingNodeIds.has(id));
|
|
53
|
+
|
|
54
|
+
if (hasAllNodes && patternNodes.length > 0) {
|
|
55
|
+
suggestions.push(`Pattern "${pattern}" already exists in current spec`);
|
|
56
|
+
} else {
|
|
57
|
+
const effectivenessPct = Math.round(memory.effectivenessScore * 100);
|
|
58
|
+
suggestions.push(
|
|
59
|
+
`Previously, "${pattern}" improved success rate (${effectivenessPct}% effectiveness in task ${memory.taskId})`,
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
for (const pattern of memory.failedPatterns) {
|
|
65
|
+
warnings.push(
|
|
66
|
+
`Pattern "${pattern}" failed in task ${memory.taskId} (effectiveness: ${Math.round(memory.effectivenessScore * 100)}%)`,
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
for (const record of memory.mutationHistory) {
|
|
71
|
+
if (record.outcome === "improved") {
|
|
72
|
+
suggestions.push(
|
|
73
|
+
`Mutation "${record.mutation}" previously improved outcomes (triggered by: ${record.triggeredBy})`,
|
|
74
|
+
);
|
|
75
|
+
} else if (record.outcome === "worse") {
|
|
76
|
+
warnings.push(
|
|
77
|
+
`Mutation "${record.mutation}" previously worsened outcomes (triggered by: ${record.triggeredBy}) - avoid this approach`,
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const aggregateEffectiveness = memories.length > 0
|
|
84
|
+
? totalEffectiveness / memories.length
|
|
85
|
+
: 0;
|
|
86
|
+
|
|
87
|
+
const uniqueSuggestions = [...new Set(suggestions)];
|
|
88
|
+
const uniqueWarnings = [...new Set(warnings)];
|
|
89
|
+
|
|
90
|
+
uniqueSuggestions.sort((a, b) => {
|
|
91
|
+
const aHasAlready = a.includes("already");
|
|
92
|
+
const bHasAlready = b.includes("already");
|
|
93
|
+
if (aHasAlready && !bHasAlready) return 1;
|
|
94
|
+
if (!aHasAlready && bHasAlready) return -1;
|
|
95
|
+
return 0;
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
return {
|
|
99
|
+
suggestions: uniqueSuggestions,
|
|
100
|
+
warnings: uniqueWarnings,
|
|
101
|
+
sourceTaskIds,
|
|
102
|
+
aggregateEffectiveness,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function isHarnessSpec(value: unknown): value is HarnessSpec {
|
|
107
|
+
return (
|
|
108
|
+
value !== null &&
|
|
109
|
+
typeof value === "object" &&
|
|
110
|
+
"name" in value &&
|
|
111
|
+
"graph" in value &&
|
|
112
|
+
typeof (value as Record<string, unknown>).graph === "object" &&
|
|
113
|
+
"nodes" in ((value as Record<string, unknown>).graph as Record<string, unknown>)
|
|
114
|
+
);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function extractNodeIdsFromPattern(pattern: string): string[] {
|
|
118
|
+
const nodePattern = /([a-zA-Z0-9_-]+)-before-([a-zA-Z0-9_-]+)/;
|
|
119
|
+
const match = pattern.match(nodePattern);
|
|
120
|
+
|
|
121
|
+
if (match) {
|
|
122
|
+
return [match[1], match[2]];
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const singleNodePattern = /^([a-zA-Z0-9_-]+)-/;
|
|
126
|
+
const singleMatch = pattern.match(singleNodePattern);
|
|
127
|
+
if (singleMatch) {
|
|
128
|
+
return [singleMatch[1]];
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
return [];
|
|
132
|
+
}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import type { HarnessExecutionTrace } from "../versioning/types.js";
|
|
2
|
+
import type { HarnessSpec } from "../spec/types.js";
|
|
3
|
+
import type { ExecutionTraceEntry } from "../compiler/runtime-helpers.js";
|
|
4
|
+
|
|
5
|
+
interface NodeSequence {
|
|
6
|
+
nodeId: string;
|
|
7
|
+
phases: ExecutionTraceEntry["phase"][];
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export function extractPatternsFromTrace(
|
|
11
|
+
trace: HarnessExecutionTrace,
|
|
12
|
+
spec: HarnessSpec,
|
|
13
|
+
): { successful: string[]; failed: string[] } {
|
|
14
|
+
if (trace.entries.length === 0) {
|
|
15
|
+
return { successful: [], failed: [] };
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const successful: string[] = [];
|
|
19
|
+
const failed: string[] = [];
|
|
20
|
+
|
|
21
|
+
const nodeSequences = buildNodeSequences(trace.entries);
|
|
22
|
+
const nodeOrder = extractNodeOrder(trace.entries);
|
|
23
|
+
|
|
24
|
+
for (const sequence of nodeSequences) {
|
|
25
|
+
const hasSuccess = sequence.phases.includes("success");
|
|
26
|
+
const hasFailure = sequence.phases.includes("failure");
|
|
27
|
+
const hasRetry = sequence.phases.includes("retry");
|
|
28
|
+
const hasVerificationFail = sequence.phases.includes("verification-fail");
|
|
29
|
+
|
|
30
|
+
if (hasSuccess && !hasFailure) {
|
|
31
|
+
successful.push(`${sequence.nodeId}-succeeds`);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
if (hasFailure) {
|
|
35
|
+
failed.push(`${sequence.nodeId}-fails`);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
if (hasRetry && hasSuccess) {
|
|
39
|
+
successful.push(`${sequence.nodeId}-retry-succeeds`);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
if (hasVerificationFail) {
|
|
43
|
+
failed.push(`${sequence.nodeId}-verification-fails`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const orderingPatterns = extractOrderingPatterns(nodeOrder, trace.entries);
|
|
48
|
+
successful.push(...orderingPatterns.successful);
|
|
49
|
+
failed.push(...orderingPatterns.failed);
|
|
50
|
+
|
|
51
|
+
const specOrderingPatterns = extractSpecOrderingPatterns(nodeOrder, spec, trace.entries);
|
|
52
|
+
successful.push(...specOrderingPatterns);
|
|
53
|
+
|
|
54
|
+
return {
|
|
55
|
+
successful: deduplicate(successful),
|
|
56
|
+
failed: deduplicate(failed),
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function buildNodeSequences(entries: ExecutionTraceEntry[]): NodeSequence[] {
|
|
61
|
+
const sequences = new Map<string, NodeSequence>();
|
|
62
|
+
|
|
63
|
+
for (const entry of entries) {
|
|
64
|
+
if (!sequences.has(entry.nodeId)) {
|
|
65
|
+
sequences.set(entry.nodeId, { nodeId: entry.nodeId, phases: [] });
|
|
66
|
+
}
|
|
67
|
+
sequences.get(entry.nodeId)!.phases.push(entry.phase);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
return Array.from(sequences.values());
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function extractNodeOrder(entries: ExecutionTraceEntry[]): string[] {
|
|
74
|
+
const seen = new Set<string>();
|
|
75
|
+
const order: string[] = [];
|
|
76
|
+
|
|
77
|
+
for (const entry of entries) {
|
|
78
|
+
if (!seen.has(entry.nodeId)) {
|
|
79
|
+
seen.add(entry.nodeId);
|
|
80
|
+
order.push(entry.nodeId);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
return order;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function extractOrderingPatterns(
|
|
88
|
+
nodeOrder: string[],
|
|
89
|
+
entries: ExecutionTraceEntry[],
|
|
90
|
+
): { successful: string[]; failed: string[] } {
|
|
91
|
+
const successful: string[] = [];
|
|
92
|
+
const failed: string[] = [];
|
|
93
|
+
|
|
94
|
+
const nodeOutcomes = new Map<string, { success: boolean; failure: boolean }>();
|
|
95
|
+
|
|
96
|
+
for (const entry of entries) {
|
|
97
|
+
if (!nodeOutcomes.has(entry.nodeId)) {
|
|
98
|
+
nodeOutcomes.set(entry.nodeId, { success: false, failure: false });
|
|
99
|
+
}
|
|
100
|
+
const outcome = nodeOutcomes.get(entry.nodeId)!;
|
|
101
|
+
if (entry.phase === "success") outcome.success = true;
|
|
102
|
+
if (entry.phase === "failure") outcome.failure = true;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
for (let i = 0; i < nodeOrder.length - 1; i++) {
|
|
106
|
+
const current = nodeOrder[i];
|
|
107
|
+
const next = nodeOrder[i + 1];
|
|
108
|
+
const currentOutcome = nodeOutcomes.get(current);
|
|
109
|
+
const nextOutcome = nodeOutcomes.get(next);
|
|
110
|
+
|
|
111
|
+
if (!currentOutcome || !nextOutcome) continue;
|
|
112
|
+
|
|
113
|
+
if (currentOutcome.success && nextOutcome.success) {
|
|
114
|
+
successful.push(`${current}-before-${next}-succeeds`);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
if (currentOutcome.success && nextOutcome.failure) {
|
|
118
|
+
failed.push(`${current}-before-${next}-fails`);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
if (currentOutcome.failure && nextOutcome.success) {
|
|
122
|
+
successful.push(`${current}-fails-but-${next}-succeeds`);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
return { successful, failed };
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function extractSpecOrderingPatterns(
|
|
130
|
+
nodeOrder: string[],
|
|
131
|
+
spec: HarnessSpec,
|
|
132
|
+
entries: ExecutionTraceEntry[],
|
|
133
|
+
): string[] {
|
|
134
|
+
const patterns: string[] = [];
|
|
135
|
+
|
|
136
|
+
const hasTerminalPhases = entries.some(
|
|
137
|
+
(e) => e.phase === "success" || e.phase === "failure" || e.phase === "verification-fail",
|
|
138
|
+
);
|
|
139
|
+
|
|
140
|
+
if (!hasTerminalPhases) {
|
|
141
|
+
return patterns;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
for (const edge of spec.graph.edges) {
|
|
145
|
+
const fromIdx = nodeOrder.indexOf(edge.from);
|
|
146
|
+
const toIdx = nodeOrder.indexOf(edge.to);
|
|
147
|
+
|
|
148
|
+
if (fromIdx !== -1 && toIdx !== -1 && fromIdx < toIdx) {
|
|
149
|
+
const fromNode = spec.graph.nodes.find((n) => n.id === edge.from);
|
|
150
|
+
const toNode = spec.graph.nodes.find((n) => n.id === edge.to);
|
|
151
|
+
|
|
152
|
+
if (fromNode && toNode) {
|
|
153
|
+
const fromKind = fromNode.kind;
|
|
154
|
+
if (fromKind === "tool" || fromKind === "condition") {
|
|
155
|
+
patterns.push(`${edge.from}-before-${edge.to}`);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
return patterns;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function deduplicate(arr: string[]): string[] {
|
|
165
|
+
return [...new Set(arr)];
|
|
166
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { mkdir, readFile, readdir, rename, writeFile } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import type { HarnessMemory, MemoryStore, MemoryUpdate, MemoryQuery } from "./types.js";
|
|
4
|
+
|
|
5
|
+
export class FileMemoryStore implements MemoryStore {
|
|
6
|
+
private readonly memoriesDir: string;
|
|
7
|
+
|
|
8
|
+
constructor(private readonly storeDir: string) {
|
|
9
|
+
this.memoriesDir = join(storeDir, "memories");
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
async saveMemory(memory: HarnessMemory): Promise<void> {
|
|
13
|
+
await mkdir(this.memoriesDir, { recursive: true });
|
|
14
|
+
const data = structuredClone(memory);
|
|
15
|
+
const filePath = join(this.memoriesDir, `${memory.taskId}.json`);
|
|
16
|
+
const tmpPath = `${filePath}.tmp`;
|
|
17
|
+
await writeFile(tmpPath, JSON.stringify(data, null, 2), "utf-8");
|
|
18
|
+
await rename(tmpPath, filePath);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async getMemory(taskId: string): Promise<HarnessMemory | null> {
|
|
22
|
+
try {
|
|
23
|
+
const filePath = join(this.memoriesDir, `${taskId}.json`);
|
|
24
|
+
const raw = await readFile(filePath, "utf-8");
|
|
25
|
+
return JSON.parse(raw) as HarnessMemory;
|
|
26
|
+
} catch (err: unknown) {
|
|
27
|
+
const code = (err as NodeJS.ErrnoException).code;
|
|
28
|
+
if (code === "ENOENT") return null;
|
|
29
|
+
throw err;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async updateMemory(taskId: string, update: MemoryUpdate): Promise<HarnessMemory> {
|
|
34
|
+
const existing = await this.getMemory(taskId);
|
|
35
|
+
if (!existing) {
|
|
36
|
+
throw new Error(`Memory not found for task: ${taskId}`);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const updated: HarnessMemory = structuredClone(existing);
|
|
40
|
+
|
|
41
|
+
if (update.successfulPattern) {
|
|
42
|
+
if (!updated.successfulPatterns.includes(update.successfulPattern)) {
|
|
43
|
+
updated.successfulPatterns.push(update.successfulPattern);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (update.failedPattern) {
|
|
48
|
+
if (!updated.failedPatterns.includes(update.failedPattern)) {
|
|
49
|
+
updated.failedPatterns.push(update.failedPattern);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
if (update.mutation) {
|
|
54
|
+
updated.mutationHistory.push(structuredClone(update.mutation));
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
if (update.effectivenessDelta !== undefined) {
|
|
58
|
+
updated.effectivenessScore = Math.max(
|
|
59
|
+
0,
|
|
60
|
+
Math.min(1, updated.effectivenessScore + update.effectivenessDelta),
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
updated.lastUpdated = Date.now();
|
|
65
|
+
|
|
66
|
+
await this.saveMemory(updated);
|
|
67
|
+
return updated;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async searchMemories(query: MemoryQuery): Promise<HarnessMemory[]> {
|
|
71
|
+
let memories = await this.loadAllMemories();
|
|
72
|
+
|
|
73
|
+
if (query.taskSignature) {
|
|
74
|
+
memories = memories.filter(
|
|
75
|
+
(m) => m.taskEmbedding && m.taskEmbedding.includes(query.taskSignature!),
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (query.minEffectiveness !== undefined) {
|
|
80
|
+
memories = memories.filter(
|
|
81
|
+
(m) => m.effectivenessScore >= query.minEffectiveness!,
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
if (query.limit !== undefined) {
|
|
86
|
+
memories = memories.slice(0, query.limit);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
return memories;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
private async loadAllMemories(): Promise<HarnessMemory[]> {
|
|
93
|
+
try {
|
|
94
|
+
const files = await readdir(this.memoriesDir);
|
|
95
|
+
const jsonFiles = files.filter((f) => f.endsWith(".json"));
|
|
96
|
+
const memories = await Promise.all(
|
|
97
|
+
jsonFiles.map(async (file) => {
|
|
98
|
+
const raw = await readFile(join(this.memoriesDir, file), "utf-8");
|
|
99
|
+
return JSON.parse(raw) as HarnessMemory;
|
|
100
|
+
}),
|
|
101
|
+
);
|
|
102
|
+
return memories.sort((a, b) => b.effectivenessScore - a.effectivenessScore);
|
|
103
|
+
} catch {
|
|
104
|
+
return [];
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|