@mhingston5/lasso 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.lean-ctx/graph.db +0 -0
- package/.lean-ctx/graph.db-shm +0 -0
- package/.lean-ctx/graph.db-wal +0 -0
- package/README.md +150 -5
- package/package.json +1 -1
- package/src/cir/lower.ts +2 -0
- package/src/cir/types.ts +6 -0
- package/src/compiler/compile.ts +272 -2
- package/src/failures/generator.ts +78 -2
- package/src/failures/types.ts +21 -0
- package/src/index.ts +1 -0
- package/src/metaharness/engine.ts +146 -3
- package/src/metaharness/trace-adapter.ts +34 -0
- package/src/metaharness/types.ts +41 -0
- package/src/replanner/runtime.ts +181 -0
- package/src/spec/schema.ts +46 -6
- package/src/spec/types.ts +39 -0
- package/test/compiler/per-node-harness.test.ts +955 -0
- package/test/failures/risk.test.ts +285 -0
- package/test/metaharness/synthesize-from-trace.test.ts +372 -0
- package/test/replanner/runtime.test.ts +134 -0
package/src/failures/types.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import type { HarnessMutation } from "../mutation/types.js";
|
|
2
|
+
import type { FailureClass } from "./ontology.js";
|
|
3
|
+
|
|
1
4
|
export interface FailureRecord {
|
|
2
5
|
domainType: string;
|
|
3
6
|
rootCause:
|
|
@@ -12,3 +15,21 @@ export interface FailureRecord {
|
|
|
12
15
|
nodeId?: string;
|
|
13
16
|
message: string;
|
|
14
17
|
}
|
|
18
|
+
|
|
19
|
+
export interface Risk {
|
|
20
|
+
id: string;
|
|
21
|
+
probability: number;
|
|
22
|
+
impact: number;
|
|
23
|
+
score: number;
|
|
24
|
+
signals: string[];
|
|
25
|
+
mitigations: HarnessMutation[];
|
|
26
|
+
failureClass: FailureClass;
|
|
27
|
+
description: string;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface RiskAssessment {
|
|
31
|
+
risks: Risk[];
|
|
32
|
+
overallScore: number;
|
|
33
|
+
highRiskThreshold: number;
|
|
34
|
+
risksAboveThreshold: Risk[];
|
|
35
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -39,6 +39,7 @@ export { synthesizeHarness } from "./synthesis/harness-builder.js";
|
|
|
39
39
|
export { createInitialVersion, createNextVersion, createLineageEntry } from "./versioning/history.js";
|
|
40
40
|
export { prepareInitialAdaptiveInput, unwrapAdaptiveInput, prepareRuntimeReplan, MAX_ADAPTIVE_VERSIONS } from "./replanner/runtime.js";
|
|
41
41
|
export type { MetaHarnessConfig, MetaHarnessResult, MetaHarness } from "./metaharness/types.js";
|
|
42
|
+
export type { ExecutionTrace, CompletedNode, FailedNode, HarnessSynthesisResult } from "./metaharness/types.js";
|
|
42
43
|
export { DefaultMetaHarness } from "./metaharness/engine.js";
|
|
43
44
|
export { predictFailuresFromEnvironment } from "./metaharness/predictor.js";
|
|
44
45
|
export { discoverEnvironment } from "./environment/discovery.js";
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import type { HarnessSpec } from "../spec/types.js";
|
|
2
2
|
import type { EnvironmentModel, EnvironmentAnalysis } from "../environment/types.js";
|
|
3
3
|
import type { FailureSignature } from "../failures/ontology.js";
|
|
4
|
-
import type { MemoryAdvice
|
|
4
|
+
import type { MemoryAdvice } from "../memory/types.js";
|
|
5
5
|
import type { CapabilityRegistry } from "../capabilities/types.js";
|
|
6
6
|
import type { MutationPolicy, HarnessMutation } from "../mutation/types.js";
|
|
7
|
-
import type { MetaHarnessConfig, MetaHarnessResult, MetaHarness } from "./types.js";
|
|
7
|
+
import type { MetaHarnessConfig, MetaHarnessResult, MetaHarness, ExecutionTrace, HarnessSynthesisResult } from "./types.js";
|
|
8
|
+
import { buildTraceEntries } from "./trace-adapter.js";
|
|
8
9
|
import type { CompilerAnalysis } from "../compiler/feedback.js";
|
|
9
10
|
import type { HarnessStage, CompositionResult } from "../composition/types.js";
|
|
10
11
|
import { discoverEnvironment } from "../environment/discovery.js";
|
|
@@ -19,11 +20,12 @@ import { compileHarnessSpec } from "../compiler/compile.js";
|
|
|
19
20
|
import { analyzeCompiledWorkflow } from "../compiler/feedback.js";
|
|
20
21
|
import { predictFailuresFromEnvironment } from "./predictor.js";
|
|
21
22
|
import { generateFailureModes } from "../failures/generator.js";
|
|
22
|
-
import { deriveMutationsFromFailure } from "../mutation/derive.js";
|
|
23
|
+
import { deriveMutationsFromFailure, deriveMutationsFromTrace } from "../mutation/derive.js";
|
|
23
24
|
import { mutateHarness } from "../mutation/engine.js";
|
|
24
25
|
import { chainHarnesses } from "../composition/chain.js";
|
|
25
26
|
import { parallelHarnesses } from "../composition/parallel.js";
|
|
26
27
|
import { conditionalHarness } from "../composition/conditional.js";
|
|
28
|
+
import { classifyFailure } from "../failures/ontology.js";
|
|
27
29
|
|
|
28
30
|
export class DefaultMetaHarness implements MetaHarness {
|
|
29
31
|
constructor(private config: MetaHarnessConfig) {}
|
|
@@ -189,6 +191,89 @@ export class DefaultMetaHarness implements MetaHarness {
|
|
|
189
191
|
};
|
|
190
192
|
}
|
|
191
193
|
|
|
194
|
+
async synthesizeFromTrace(
|
|
195
|
+
trace: ExecutionTrace,
|
|
196
|
+
currentSpec: HarnessSpec,
|
|
197
|
+
environment: EnvironmentModel,
|
|
198
|
+
): Promise<HarnessSynthesisResult> {
|
|
199
|
+
const rationale: string[] = [];
|
|
200
|
+
const allMutations: HarnessMutation[] = [];
|
|
201
|
+
|
|
202
|
+
// 1. Analyze trace for patterns
|
|
203
|
+
const repeatedFailures = findRepeatedFailures(trace.failedNodes);
|
|
204
|
+
const slowNodes = findSlowNodes(trace.completedNodes);
|
|
205
|
+
const costSpike = detectCostSpike(trace);
|
|
206
|
+
|
|
207
|
+
for (const [nodeId, count] of repeatedFailures) {
|
|
208
|
+
rationale.push(`Repeated failures (${count}x) on node "${nodeId}"`);
|
|
209
|
+
}
|
|
210
|
+
for (const [nodeId, durationMs] of slowNodes) {
|
|
211
|
+
rationale.push(`Slow node "${nodeId}": ${durationMs}ms duration`);
|
|
212
|
+
}
|
|
213
|
+
if (costSpike) {
|
|
214
|
+
rationale.push(`cost spike detected: $${trace.totalCostUsd?.toFixed(2) ?? "0.00"} total`);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// 2. Classify each failure and derive mutations
|
|
218
|
+
for (const failedNode of trace.failedNodes) {
|
|
219
|
+
const signature = classifyFailure(failedNode.error, { nodeId: failedNode.nodeId });
|
|
220
|
+
const mutations = deriveMutationsFromFailure(signature, currentSpec, { nodeId: failedNode.nodeId });
|
|
221
|
+
allMutations.push(...mutations);
|
|
222
|
+
|
|
223
|
+
if (mutations.length > 0) {
|
|
224
|
+
rationale.push(
|
|
225
|
+
`Classified failure on "${failedNode.nodeId}" as ${signature.class}, derived ${mutations.length} mutation(s)`,
|
|
226
|
+
);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// 3. Also derive mutations from the full trace using existing trace analysis
|
|
231
|
+
const traceEntries = buildTraceEntries(trace);
|
|
232
|
+
const traceMutations = deriveMutationsFromTrace(
|
|
233
|
+
{
|
|
234
|
+
entries: traceEntries,
|
|
235
|
+
totalDurationMs: trace.completedNodes.reduce(
|
|
236
|
+
(sum, n) => sum + (n.completedAt - n.startedAt), 0,
|
|
237
|
+
),
|
|
238
|
+
nodeCount: trace.completedNodes.length + trace.failedNodes.length,
|
|
239
|
+
failureCount: trace.failedNodes.length,
|
|
240
|
+
startTimeMs: Math.min(
|
|
241
|
+
...[...trace.completedNodes, ...trace.failedNodes].map(n => n.startedAt),
|
|
242
|
+
Date.now(),
|
|
243
|
+
),
|
|
244
|
+
endTimeMs: trace.capturedAt,
|
|
245
|
+
},
|
|
246
|
+
currentSpec,
|
|
247
|
+
);
|
|
248
|
+
allMutations.push(...traceMutations);
|
|
249
|
+
|
|
250
|
+
if (traceMutations.length > 0) {
|
|
251
|
+
rationale.push(`Derived ${traceMutations.length} mutation(s) from execution trace patterns`);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// 4. Apply mutation policy if configured
|
|
255
|
+
const limitedMutations = this.config.mutationPolicy
|
|
256
|
+
? enforceMutationPolicy(allMutations, this.config.mutationPolicy)
|
|
257
|
+
: allMutations;
|
|
258
|
+
|
|
259
|
+
// 5. Apply mutations
|
|
260
|
+
let updatedSpec = currentSpec;
|
|
261
|
+
if (limitedMutations.length > 0) {
|
|
262
|
+
const result = mutateHarness(currentSpec, limitedMutations);
|
|
263
|
+
updatedSpec = result.spec;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
// 6. Determine decision
|
|
267
|
+
const decision = determineDecision(trace, rationale);
|
|
268
|
+
|
|
269
|
+
return {
|
|
270
|
+
mutations: limitedMutations,
|
|
271
|
+
spec: updatedSpec,
|
|
272
|
+
rationale,
|
|
273
|
+
decision,
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
|
|
192
277
|
composeHarnesses(stages: HarnessStage[]): CompositionResult {
|
|
193
278
|
return chainHarnesses(stages);
|
|
194
279
|
}
|
|
@@ -254,3 +339,61 @@ function calculateReadinessScore(
|
|
|
254
339
|
|
|
255
340
|
return Math.min(100, Math.max(0, score));
|
|
256
341
|
}
|
|
342
|
+
|
|
343
|
+
function findRepeatedFailures(failedNodes: ExecutionTrace["failedNodes"]): Map<string, number> {
|
|
344
|
+
const counts = new Map<string, number>();
|
|
345
|
+
for (const node of failedNodes) {
|
|
346
|
+
counts.set(node.nodeId, (counts.get(node.nodeId) ?? 0) + 1);
|
|
347
|
+
}
|
|
348
|
+
const repeated = new Map<string, number>();
|
|
349
|
+
for (const [nodeId, count] of counts) {
|
|
350
|
+
if (count >= 2) {
|
|
351
|
+
repeated.set(nodeId, count);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
return repeated;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
const SLOW_NODE_THRESHOLD_MS = 30_000;
|
|
358
|
+
|
|
359
|
+
function findSlowNodes(
|
|
360
|
+
completedNodes: ExecutionTrace["completedNodes"],
|
|
361
|
+
): Map<string, number> {
|
|
362
|
+
const slow = new Map<string, number>();
|
|
363
|
+
for (const node of completedNodes) {
|
|
364
|
+
const duration = node.completedAt - node.startedAt;
|
|
365
|
+
if (duration >= SLOW_NODE_THRESHOLD_MS) {
|
|
366
|
+
slow.set(node.nodeId, duration);
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
return slow;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
const COST_SPIKE_THRESHOLD_USD = 5.0;
|
|
373
|
+
|
|
374
|
+
function detectCostSpike(trace: ExecutionTrace): boolean {
|
|
375
|
+
return (trace.totalCostUsd ?? 0) >= COST_SPIKE_THRESHOLD_USD;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
function determineDecision(
|
|
379
|
+
trace: ExecutionTrace,
|
|
380
|
+
rationale: string[],
|
|
381
|
+
): HarnessSynthesisResult["decision"] {
|
|
382
|
+
const totalNodes = trace.completedNodes.length + trace.failedNodes.length;
|
|
383
|
+
|
|
384
|
+
const requiresHuman = trace.failedNodes.some(
|
|
385
|
+
n => n.error.toLowerCase().includes("approval required")
|
|
386
|
+
|| n.error.toLowerCase().includes("human intervention"),
|
|
387
|
+
);
|
|
388
|
+
if (requiresHuman) {
|
|
389
|
+
rationale.push("Failure requires human intervention");
|
|
390
|
+
return "needs_operator_input";
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
if (totalNodes > 0 && trace.failedNodes.length === totalNodes) {
|
|
394
|
+
rationale.push("All nodes have failed — stopping execution");
|
|
395
|
+
return "stop";
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
return "continue";
|
|
399
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { ExecutionTraceEntry } from "../compiler/runtime-helpers.js";
|
|
2
|
+
import type { ExecutionTrace } from "./types.js";
|
|
3
|
+
|
|
4
|
+
export function buildTraceEntries(trace: ExecutionTrace): ExecutionTraceEntry[] {
|
|
5
|
+
const entries: ExecutionTraceEntry[] = [];
|
|
6
|
+
|
|
7
|
+
for (const node of trace.completedNodes) {
|
|
8
|
+
entries.push({
|
|
9
|
+
nodeId: node.nodeId,
|
|
10
|
+
source: undefined,
|
|
11
|
+
phase: "success",
|
|
12
|
+
startedAt: node.startedAt,
|
|
13
|
+
completedAt: node.completedAt,
|
|
14
|
+
outputSnapshot: node.output,
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
for (const node of trace.failedNodes) {
|
|
19
|
+
entries.push({
|
|
20
|
+
nodeId: node.nodeId,
|
|
21
|
+
source: undefined,
|
|
22
|
+
phase: "failure",
|
|
23
|
+
startedAt: node.startedAt,
|
|
24
|
+
completedAt: node.failedAt,
|
|
25
|
+
details: {
|
|
26
|
+
error: node.error,
|
|
27
|
+
category: node.failureClass ?? "unknown",
|
|
28
|
+
retryCount: node.retryCount,
|
|
29
|
+
},
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
return entries;
|
|
34
|
+
}
|
package/src/metaharness/types.ts
CHANGED
|
@@ -8,6 +8,42 @@ import type { MutationPolicy, HarnessMutation } from "../mutation/types.js";
|
|
|
8
8
|
import type { CompilerAnalysis, CompilerSuggestion } from "../compiler/feedback.js";
|
|
9
9
|
import type { HarnessStage, CompositionResult } from "../composition/types.js";
|
|
10
10
|
|
|
11
|
+
// ============================================================================
|
|
12
|
+
// Mid-execution trace synthesis types
|
|
13
|
+
// ============================================================================
|
|
14
|
+
|
|
15
|
+
export interface CompletedNode {
|
|
16
|
+
nodeId: string;
|
|
17
|
+
startedAt: number;
|
|
18
|
+
completedAt: number;
|
|
19
|
+
output?: unknown;
|
|
20
|
+
costUsd?: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface FailedNode {
|
|
24
|
+
nodeId: string;
|
|
25
|
+
startedAt: number;
|
|
26
|
+
failedAt: number;
|
|
27
|
+
error: string;
|
|
28
|
+
failureClass?: string;
|
|
29
|
+
retryCount: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface ExecutionTrace {
|
|
33
|
+
completedNodes: CompletedNode[];
|
|
34
|
+
failedNodes: FailedNode[];
|
|
35
|
+
currentNodeId?: string;
|
|
36
|
+
capturedAt: number;
|
|
37
|
+
totalCostUsd?: number;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface HarnessSynthesisResult {
|
|
41
|
+
mutations: HarnessMutation[];
|
|
42
|
+
spec: HarnessSpec;
|
|
43
|
+
rationale: string[];
|
|
44
|
+
decision: "continue" | "needs_operator_input" | "stop";
|
|
45
|
+
}
|
|
46
|
+
|
|
11
47
|
export interface MetaHarnessConfig {
|
|
12
48
|
environmentModel?: EnvironmentModel;
|
|
13
49
|
memoryStore?: MemoryStore;
|
|
@@ -34,6 +70,11 @@ export interface MetaHarness {
|
|
|
34
70
|
predictFailures(spec: HarnessSpec, env: EnvironmentModel): Promise<FailureSignature[]>;
|
|
35
71
|
synthesizePolicies(spec: HarnessSpec, failures: FailureSignature[]): HarnessSpec;
|
|
36
72
|
generateHarness(intent: string, config?: MetaHarnessConfig): Promise<MetaHarnessResult>;
|
|
73
|
+
synthesizeFromTrace(
|
|
74
|
+
trace: ExecutionTrace,
|
|
75
|
+
currentSpec: HarnessSpec,
|
|
76
|
+
environment: EnvironmentModel,
|
|
77
|
+
): Promise<HarnessSynthesisResult>;
|
|
37
78
|
composeHarnesses(stages: HarnessStage[]): CompositionResult;
|
|
38
79
|
composeParallel(harnesses: HarnessSpec[]): CompositionResult;
|
|
39
80
|
composeConditional(condition: string, trueHarness: HarnessSpec, falseHarness?: HarnessSpec): CompositionResult;
|
package/src/replanner/runtime.ts
CHANGED
|
@@ -10,6 +10,8 @@ import type { HarnessMutation, MutationResult } from "../mutation/types.js";
|
|
|
10
10
|
import type { MemoryStore, HarnessMemory, MemoryAdvice, MemoryUpdate } from "../memory/types.js";
|
|
11
11
|
import { extractPatternsFromTrace } from "../memory/extractor.js";
|
|
12
12
|
import { adviseFromMemory } from "../memory/advisor.js";
|
|
13
|
+
import type { ExecutionTrace, HarnessSynthesisResult } from "../metaharness/types.js";
|
|
14
|
+
import type { EnvironmentModel } from "../environment/types.js";
|
|
13
15
|
|
|
14
16
|
export const MAX_ADAPTIVE_VERSIONS = 5;
|
|
15
17
|
|
|
@@ -46,6 +48,14 @@ export type RuntimeReplanDecision =
|
|
|
46
48
|
decision: "stop";
|
|
47
49
|
lineageEntry: LineageEntry;
|
|
48
50
|
replanResult: ReplanResult;
|
|
51
|
+
}
|
|
52
|
+
| {
|
|
53
|
+
decision: "trace_synthesis";
|
|
54
|
+
lineageEntry: LineageEntry;
|
|
55
|
+
synthesisResult: HarnessSynthesisResult;
|
|
56
|
+
nextRequest?: ReferenceWorkflowRequest;
|
|
57
|
+
nextVersion?: HarnessVersion;
|
|
58
|
+
nextInput?: AdaptiveRuntimeInput;
|
|
49
59
|
};
|
|
50
60
|
|
|
51
61
|
export function prepareInitialAdaptiveInput(
|
|
@@ -195,6 +205,177 @@ export function prepareRuntimeReplanWithMutations(
|
|
|
195
205
|
};
|
|
196
206
|
}
|
|
197
207
|
|
|
208
|
+
export interface TraceSynthesisReplanInput {
|
|
209
|
+
adaptive: AdaptiveRuntimeMetadata;
|
|
210
|
+
runtimeInput: unknown;
|
|
211
|
+
result: CompiledHarnessResult;
|
|
212
|
+
environment: EnvironmentModel;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
export async function prepareRuntimeReplanWithTraceSynthesis(
|
|
216
|
+
input: TraceSynthesisReplanInput,
|
|
217
|
+
): Promise<RuntimeReplanDecision> {
|
|
218
|
+
const { adaptive, runtimeInput, result, environment } = input;
|
|
219
|
+
const lineageEntry = createLineageEntry(adaptive.currentVersion, result);
|
|
220
|
+
|
|
221
|
+
if (adaptive.currentVersion.version >= MAX_ADAPTIVE_VERSIONS) {
|
|
222
|
+
return {
|
|
223
|
+
decision: "stop",
|
|
224
|
+
lineageEntry,
|
|
225
|
+
replanResult: {
|
|
226
|
+
status: "stop",
|
|
227
|
+
workflow: adaptive.currentRequest.workflow,
|
|
228
|
+
riskLevel: "high",
|
|
229
|
+
reasons: ["Max adaptive version limit reached"],
|
|
230
|
+
guidance: ["Manual intervention required to continue workflow evolution"],
|
|
231
|
+
},
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const executionTrace: ExecutionTrace = buildExecutionTrace(result);
|
|
236
|
+
|
|
237
|
+
const { DefaultMetaHarness } = await import("../metaharness/engine.js");
|
|
238
|
+
const metaHarness = new DefaultMetaHarness({ environmentModel: environment });
|
|
239
|
+
|
|
240
|
+
const synthesisResult = await metaHarness.synthesizeFromTrace(
|
|
241
|
+
executionTrace,
|
|
242
|
+
adaptive.currentVersion.spec,
|
|
243
|
+
environment,
|
|
244
|
+
);
|
|
245
|
+
|
|
246
|
+
switch (synthesisResult.decision) {
|
|
247
|
+
case "stop":
|
|
248
|
+
return {
|
|
249
|
+
decision: "stop",
|
|
250
|
+
lineageEntry,
|
|
251
|
+
replanResult: {
|
|
252
|
+
status: "stop",
|
|
253
|
+
workflow: adaptive.currentRequest.workflow,
|
|
254
|
+
riskLevel: "high",
|
|
255
|
+
reasons: synthesisResult.rationale,
|
|
256
|
+
guidance: ["Trace synthesis determined execution should stop"],
|
|
257
|
+
},
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
case "needs_operator_input":
|
|
261
|
+
return {
|
|
262
|
+
decision: "needs_operator_input",
|
|
263
|
+
lineageEntry,
|
|
264
|
+
replanResult: {
|
|
265
|
+
status: "needs_operator_input",
|
|
266
|
+
workflow: adaptive.currentRequest.workflow,
|
|
267
|
+
riskLevel: "medium",
|
|
268
|
+
reasons: synthesisResult.rationale,
|
|
269
|
+
guidance: ["Trace synthesis requires operator input"],
|
|
270
|
+
},
|
|
271
|
+
};
|
|
272
|
+
|
|
273
|
+
case "continue": {
|
|
274
|
+
if (synthesisResult.mutations.length === 0) {
|
|
275
|
+
const replanRequest = buildReplanRequest(
|
|
276
|
+
adaptive.currentRequest,
|
|
277
|
+
result.terminalNodeId,
|
|
278
|
+
result.harnessState,
|
|
279
|
+
);
|
|
280
|
+
const replanResult = replanWorkflowRequest(replanRequest);
|
|
281
|
+
|
|
282
|
+
if (replanResult.status === "draft_request") {
|
|
283
|
+
const baseSpec = buildReferenceHarnessSpec(replanResult.request);
|
|
284
|
+
const nextVersion = createNextVersion(
|
|
285
|
+
adaptive.currentVersion,
|
|
286
|
+
baseSpec,
|
|
287
|
+
`trace-synthesis: ${synthesisResult.rationale[0] || "no mutations needed"}`,
|
|
288
|
+
);
|
|
289
|
+
const nextInput: AdaptiveRuntimeInput = {
|
|
290
|
+
input: structuredClone(runtimeInput),
|
|
291
|
+
__lassoAdaptiveRuntime: {
|
|
292
|
+
currentRequest: structuredClone(replanResult.request),
|
|
293
|
+
currentVersion: nextVersion,
|
|
294
|
+
lineage: [...adaptive.lineage, lineageEntry],
|
|
295
|
+
},
|
|
296
|
+
};
|
|
297
|
+
|
|
298
|
+
return {
|
|
299
|
+
decision: "trace_synthesis",
|
|
300
|
+
lineageEntry,
|
|
301
|
+
synthesisResult,
|
|
302
|
+
nextRequest: replanResult.request,
|
|
303
|
+
nextVersion,
|
|
304
|
+
nextInput,
|
|
305
|
+
};
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
return {
|
|
309
|
+
decision: "stop",
|
|
310
|
+
lineageEntry,
|
|
311
|
+
replanResult,
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
const nextVersion = createNextVersion(
|
|
316
|
+
adaptive.currentVersion,
|
|
317
|
+
synthesisResult.spec,
|
|
318
|
+
`trace-synthesis: ${synthesisResult.rationale.join("; ")}`,
|
|
319
|
+
);
|
|
320
|
+
|
|
321
|
+
const nextInput: AdaptiveRuntimeInput = {
|
|
322
|
+
input: structuredClone(runtimeInput),
|
|
323
|
+
__lassoAdaptiveRuntime: {
|
|
324
|
+
currentRequest: structuredClone(adaptive.currentRequest),
|
|
325
|
+
currentVersion: nextVersion,
|
|
326
|
+
lineage: [...adaptive.lineage, lineageEntry],
|
|
327
|
+
pendingMutations: synthesisResult.mutations,
|
|
328
|
+
},
|
|
329
|
+
};
|
|
330
|
+
|
|
331
|
+
return {
|
|
332
|
+
decision: "trace_synthesis",
|
|
333
|
+
lineageEntry,
|
|
334
|
+
synthesisResult,
|
|
335
|
+
nextRequest: adaptive.currentRequest,
|
|
336
|
+
nextVersion,
|
|
337
|
+
nextInput,
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
// NOTE: Timestamps here are approximations — the total harness duration is
|
|
344
|
+
// applied uniformly to all nodes. Per-node timing should be sourced from
|
|
345
|
+
// trace entries when available.
|
|
346
|
+
function buildExecutionTrace(result: CompiledHarnessResult): ExecutionTrace {
|
|
347
|
+
const completedNodes: ExecutionTrace["completedNodes"] = [];
|
|
348
|
+
const failedNodes: ExecutionTrace["failedNodes"] = [];
|
|
349
|
+
|
|
350
|
+
for (const failure of result.harnessState.failures) {
|
|
351
|
+
failedNodes.push({
|
|
352
|
+
nodeId: failure.nodeId ?? "unknown",
|
|
353
|
+
startedAt: Date.now() - (result.harnessState.metrics.durationMs ?? 0),
|
|
354
|
+
failedAt: Date.now(),
|
|
355
|
+
error: failure.message,
|
|
356
|
+
retryCount: 0,
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
for (const [nodeId, output] of Object.entries(result.harnessState.nodeResults ?? {})) {
|
|
361
|
+
if (!failedNodes.some(f => f.nodeId === nodeId)) {
|
|
362
|
+
completedNodes.push({
|
|
363
|
+
nodeId,
|
|
364
|
+
startedAt: Date.now() - (result.harnessState.metrics.durationMs ?? 0),
|
|
365
|
+
completedAt: Date.now(),
|
|
366
|
+
output,
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
return {
|
|
372
|
+
completedNodes,
|
|
373
|
+
failedNodes,
|
|
374
|
+
currentNodeId: result.terminalNodeId,
|
|
375
|
+
capturedAt: Date.now(),
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
|
|
198
379
|
function buildReplanRequest(
|
|
199
380
|
originalRequest: ReferenceWorkflowRequest,
|
|
200
381
|
terminalNodeId: string,
|
package/src/spec/schema.ts
CHANGED
|
@@ -35,7 +35,9 @@ export const harnessSpecSchema = {
|
|
|
35
35
|
label: { type: "string" },
|
|
36
36
|
executionPolicy: { $ref: "#/$defs/executionPolicy" },
|
|
37
37
|
retryPolicy: { $ref: "#/$defs/retryPolicy" },
|
|
38
|
-
verificationPolicy: { $ref: "#/$defs/verificationPolicy" }
|
|
38
|
+
verificationPolicy: { $ref: "#/$defs/verificationPolicy" },
|
|
39
|
+
guardrails: { $ref: "#/$defs/nodeGuardrails" },
|
|
40
|
+
verificationHooks: { $ref: "#/$defs/verificationHooks" }
|
|
39
41
|
}
|
|
40
42
|
},
|
|
41
43
|
{
|
|
@@ -54,7 +56,9 @@ export const harnessSpecSchema = {
|
|
|
54
56
|
label: { type: "string" },
|
|
55
57
|
executionPolicy: { $ref: "#/$defs/executionPolicy" },
|
|
56
58
|
retryPolicy: { $ref: "#/$defs/retryPolicy" },
|
|
57
|
-
verificationPolicy: { $ref: "#/$defs/verificationPolicy" }
|
|
59
|
+
verificationPolicy: { $ref: "#/$defs/verificationPolicy" },
|
|
60
|
+
guardrails: { $ref: "#/$defs/nodeGuardrails" },
|
|
61
|
+
verificationHooks: { $ref: "#/$defs/verificationHooks" }
|
|
58
62
|
}
|
|
59
63
|
},
|
|
60
64
|
{
|
|
@@ -70,7 +74,9 @@ export const harnessSpecSchema = {
|
|
|
70
74
|
timeout: { type: "number" },
|
|
71
75
|
label: { type: "string" },
|
|
72
76
|
executionPolicy: { $ref: "#/$defs/executionPolicy" },
|
|
73
|
-
verificationPolicy: { $ref: "#/$defs/verificationPolicy" }
|
|
77
|
+
verificationPolicy: { $ref: "#/$defs/verificationPolicy" },
|
|
78
|
+
guardrails: { $ref: "#/$defs/nodeGuardrails" },
|
|
79
|
+
verificationHooks: { $ref: "#/$defs/verificationHooks" }
|
|
74
80
|
}
|
|
75
81
|
},
|
|
76
82
|
{
|
|
@@ -85,7 +91,9 @@ export const harnessSpecSchema = {
|
|
|
85
91
|
elseNodeId: { type: "string", minLength: 1 },
|
|
86
92
|
label: { type: "string" },
|
|
87
93
|
executionPolicy: { $ref: "#/$defs/executionPolicy" },
|
|
88
|
-
verificationPolicy: { $ref: "#/$defs/verificationPolicy" }
|
|
94
|
+
verificationPolicy: { $ref: "#/$defs/verificationPolicy" },
|
|
95
|
+
guardrails: { $ref: "#/$defs/nodeGuardrails" },
|
|
96
|
+
verificationHooks: { $ref: "#/$defs/verificationHooks" }
|
|
89
97
|
}
|
|
90
98
|
},
|
|
91
99
|
{
|
|
@@ -99,7 +107,9 @@ export const harnessSpecSchema = {
|
|
|
99
107
|
strategy: { enum: ["all", "any", "majority"] },
|
|
100
108
|
label: { type: "string" },
|
|
101
109
|
executionPolicy: { $ref: "#/$defs/executionPolicy" },
|
|
102
|
-
verificationPolicy: { $ref: "#/$defs/verificationPolicy" }
|
|
110
|
+
verificationPolicy: { $ref: "#/$defs/verificationPolicy" },
|
|
111
|
+
guardrails: { $ref: "#/$defs/nodeGuardrails" },
|
|
112
|
+
verificationHooks: { $ref: "#/$defs/verificationHooks" }
|
|
103
113
|
}
|
|
104
114
|
},
|
|
105
115
|
{
|
|
@@ -114,7 +124,9 @@ export const harnessSpecSchema = {
|
|
|
114
124
|
label: { type: "string" },
|
|
115
125
|
executionPolicy: { $ref: "#/$defs/executionPolicy" },
|
|
116
126
|
retryPolicy: { $ref: "#/$defs/retryPolicy" },
|
|
117
|
-
verificationPolicy: { $ref: "#/$defs/verificationPolicy" }
|
|
127
|
+
verificationPolicy: { $ref: "#/$defs/verificationPolicy" },
|
|
128
|
+
guardrails: { $ref: "#/$defs/nodeGuardrails" },
|
|
129
|
+
verificationHooks: { $ref: "#/$defs/verificationHooks" }
|
|
118
130
|
}
|
|
119
131
|
}
|
|
120
132
|
]
|
|
@@ -249,6 +261,34 @@ export const harnessSpecSchema = {
|
|
|
249
261
|
items: { type: "string" }
|
|
250
262
|
}
|
|
251
263
|
}
|
|
264
|
+
},
|
|
265
|
+
nodeGuardrails: {
|
|
266
|
+
type: "object",
|
|
267
|
+
additionalProperties: false,
|
|
268
|
+
properties: {
|
|
269
|
+
timeoutSeconds: { type: "number", exclusiveMinimum: 0 },
|
|
270
|
+
maxRetries: { type: "number", minimum: 0 },
|
|
271
|
+
maxCostUsd: { type: "number", exclusiveMinimum: 0 },
|
|
272
|
+
constraints: {
|
|
273
|
+
type: "array",
|
|
274
|
+
items: { type: "string", minLength: 1 }
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
},
|
|
278
|
+
verificationHooks: {
|
|
279
|
+
type: "array",
|
|
280
|
+
items: {
|
|
281
|
+
type: "object",
|
|
282
|
+
required: ["name", "kind", "check", "onFail"],
|
|
283
|
+
additionalProperties: false,
|
|
284
|
+
properties: {
|
|
285
|
+
name: { type: "string", minLength: 1 },
|
|
286
|
+
kind: { enum: ["tool", "llm", "expression"] },
|
|
287
|
+
check: { type: "string", minLength: 1 },
|
|
288
|
+
onFail: { enum: ["block", "warn", "retry"] },
|
|
289
|
+
maxAttempts: { type: "number", minimum: 1 }
|
|
290
|
+
}
|
|
291
|
+
}
|
|
252
292
|
}
|
|
253
293
|
}
|
|
254
294
|
};
|
package/src/spec/types.ts
CHANGED
|
@@ -75,6 +75,45 @@ export interface BaseNode {
|
|
|
75
75
|
|
|
76
76
|
/** Verification policy for this node */
|
|
77
77
|
verificationPolicy?: VerificationPolicy;
|
|
78
|
+
|
|
79
|
+
/** Per-node guardrails — limits that apply only during this node's execution */
|
|
80
|
+
guardrails?: NodeGuardrails;
|
|
81
|
+
|
|
82
|
+
/** Per-node verification hooks — run after this specific node completes */
|
|
83
|
+
verificationHooks?: VerificationHook[];
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export interface NodeGuardrails {
|
|
87
|
+
/** Max execution time for this node in seconds */
|
|
88
|
+
timeoutSeconds?: number;
|
|
89
|
+
/** Max retries for this node (overrides global retryPolicy) */
|
|
90
|
+
maxRetries?: number;
|
|
91
|
+
/** Max cost in USD for this node (for LLM nodes) */
|
|
92
|
+
maxCostUsd?: number;
|
|
93
|
+
/** Custom guardrail expressions that must hold true */
|
|
94
|
+
constraints?: string[];
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Inline verification hook that runs after the declaring node completes.
|
|
99
|
+
* Unlike {@link VerificationRule}, which references a separate verifier node
|
|
100
|
+
* in the graph (node-based verification), a VerificationHook defines an
|
|
101
|
+
* inline check (string expression, LLM prompt, or tool call) that executes
|
|
102
|
+
* as part of the same node's lifecycle. Both serve similar purposes through
|
|
103
|
+
* different mechanisms: hooks are lightweight and co-located, while rules
|
|
104
|
+
* allow full graph-node verification logic with independent inputs/outputs.
|
|
105
|
+
*/
|
|
106
|
+
export interface VerificationHook {
|
|
107
|
+
/** Hook name for identification */
|
|
108
|
+
name: string;
|
|
109
|
+
/** Kind of verification */
|
|
110
|
+
kind: "tool" | "llm" | "expression";
|
|
111
|
+
/** The check to run (tool name, prompt, or expression) */
|
|
112
|
+
check: string;
|
|
113
|
+
/** What to do on failure */
|
|
114
|
+
onFail: "block" | "warn" | "retry";
|
|
115
|
+
/** Max verification attempts (optional) */
|
|
116
|
+
maxAttempts?: number;
|
|
78
117
|
}
|
|
79
118
|
|
|
80
119
|
/** Execute a tool command */
|