@kontourai/flow-agents 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/actions/trust-verify/action.yml +4 -2
- package/.github/workflows/ci.yml +12 -0
- package/.github/workflows/runtime-compat.yml +1 -1
- package/CHANGELOG.md +29 -0
- package/README.md +3 -3
- package/build/src/cli/workflow-sidecar.d.ts +16 -0
- package/build/src/cli/workflow-sidecar.js +72 -12
- package/build/src/lib/flow-resolver.d.ts +29 -0
- package/build/src/lib/flow-resolver.js +71 -0
- package/context/scripts/telemetry/lib/config.sh +15 -0
- package/context/scripts/telemetry/telemetry.conf +4 -0
- package/context/scripts/telemetry/telemetry.sh +23 -1
- package/docs/design/flowrun-eventsourcing-design.md +216 -0
- package/docs/design/workflowrun-observability-design.md +431 -0
- package/evals/ci/antigaming-suite.sh +2 -0
- package/evals/ci/run-baseline.sh +2 -0
- package/evals/integration/test_command_log_concurrency.sh +114 -0
- package/evals/integration/test_command_log_fork_classification.sh +134 -0
- package/evals/integration/test_kit_identity_trust.sh +393 -0
- package/evals/integration/test_usage_cost.sh +119 -0
- package/evals/integration/test_verify_cli.sh +23 -0
- package/evals/run.sh +2 -0
- package/integrations/strands/flow_agents_strands/hooks.py +126 -1
- package/integrations/strands/flow_agents_strands/telemetry.py +172 -0
- package/integrations/strands/tests/test_usage.py +129 -0
- package/integrations/strands-ts/src/hooks.ts +135 -1
- package/integrations/strands-ts/src/telemetry.ts +170 -0
- package/integrations/strands-ts/test/test-usage.ts +85 -0
- package/package.json +5 -5
- package/scripts/hooks/evidence-capture.js +75 -13
- package/scripts/hooks/stop-goal-fit.js +76 -23
- package/scripts/repair-command-log.js +115 -0
- package/scripts/telemetry/lib/config.sh +15 -0
- package/scripts/telemetry/lib/pricing.sh +42 -0
- package/scripts/telemetry/lib/usage.sh +108 -0
- package/scripts/telemetry/pricing.golden.json +15 -0
- package/scripts/telemetry/pricing.json +31 -0
- package/scripts/telemetry/telemetry.conf +4 -0
- package/scripts/telemetry/telemetry.sh +23 -1
- package/src/cli/workflow-sidecar.ts +73 -11
- package/src/lib/flow-resolver.ts +85 -0
|
@@ -6,7 +6,7 @@ import { createHash } from "node:crypto";
|
|
|
6
6
|
import { createRequire } from "node:module";
|
|
7
7
|
import { fileURLToPath } from "node:url";
|
|
8
8
|
// ADR 0016 Abstraction A: shared FlowDefinition resolver (P-a)
|
|
9
|
-
import { resolveActiveFlowStep, resolveFlowFilePath, resolvePhaseMap, type ActiveFlowStep } from "../lib/flow-resolver.js";
|
|
9
|
+
import { resolveActiveFlowStep, resolveFlowFilePath, resolvePhaseMap, resolveRouteBackPolicy, type ActiveFlowStep } from "../lib/flow-resolver.js";
|
|
10
10
|
|
|
11
11
|
type AnyObj = Record<string, any>;
|
|
12
12
|
|
|
@@ -19,11 +19,17 @@ export const verdicts = new Set(["pass", "partial", "fail", "not_verified"]);
|
|
|
19
19
|
function now(): string { return new Date().toISOString().replace(/\.\d{3}Z$/, "Z"); }
|
|
20
20
|
function read(file: string): string { return fs.readFileSync(file, "utf8"); }
|
|
21
21
|
export function writeJson(file: string, payload: AnyObj): void { fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, `${JSON.stringify(payload, null, 2)}\n`); }
|
|
22
|
-
|
|
22
|
+
// Single-line but readable "key": "value" form. Built by collapsing the
|
|
23
|
+
// structural whitespace from an indented stringify — corruption-proof, unlike a
|
|
24
|
+
// regex that would also rewrite ":"/"," sequences inside string values.
|
|
25
|
+
function spacedLine(payload: AnyObj, replacer?: (string | number)[]): string {
|
|
26
|
+
return JSON.stringify(payload, replacer as never, 1).replace(/\n\s*/g, " ");
|
|
27
|
+
}
|
|
28
|
+
function printJson(payload: AnyObj): void { console.log(spacedLine(payload)); }
|
|
23
29
|
export function loadJson(file: string, fallback: AnyObj = {}): AnyObj { return fs.existsSync(file) ? JSON.parse(read(file)) : { ...fallback }; }
|
|
24
30
|
export function appendJsonl(file: string, payload: AnyObj): void {
|
|
25
31
|
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
26
|
-
const line =
|
|
32
|
+
const line = spacedLine(payload, Object.keys(payload).sort());
|
|
27
33
|
fs.appendFileSync(file, `${line}\n`);
|
|
28
34
|
}
|
|
29
35
|
function die(message: string): never { throw new Error(message); }
|
|
@@ -1026,21 +1032,71 @@ function deriveSurfaceStatus(ref: AnyObj): string {
|
|
|
1026
1032
|
if (ref.integrity?.status !== "matched") return "fail";
|
|
1027
1033
|
return "pass";
|
|
1028
1034
|
}
|
|
1035
|
+
/**
|
|
1036
|
+
* Derive kit identity from a parsed trust.bundle by structurally reading the
|
|
1037
|
+
* DECLARED primary claim (kit-typed) rather than hardcoding "builder".
|
|
1038
|
+
*
|
|
1039
|
+
* Resolution order (no fallbacks to "builder"):
|
|
1040
|
+
* 1. First non-workflow.* claim in bundle.claims[] → claimType drives kitId + subject.
|
|
1041
|
+
* 2. No kit-typed claim: try current.json active_flow_id adjacent to the bundle file
|
|
1042
|
+
* (bundle lives at <session-dir>/trust.bundle → flowAgentsDir = grandparent).
|
|
1043
|
+
* 3. Genuinely unknown: mark as "unknown" — never hardcode a kit identity.
|
|
1044
|
+
*/
|
|
1045
|
+
export function kitIdentityFromBundle(
|
|
1046
|
+
raw: AnyObj,
|
|
1047
|
+
bundleFile: string,
|
|
1048
|
+
): { claimType: string; kitId: string; subject: string; gateId: string } {
|
|
1049
|
+
// 1. Structurally read the bundle's declared kit-typed claim.
|
|
1050
|
+
const claims: AnyObj[] = Array.isArray(raw.claims) ? raw.claims : [];
|
|
1051
|
+
for (const claim of claims) {
|
|
1052
|
+
const ct = typeof claim?.claimType === "string" ? claim.claimType : "";
|
|
1053
|
+
if (ct && !ct.startsWith("workflow.")) {
|
|
1054
|
+
const kitId = ct.split(".")[0] ?? "unknown";
|
|
1055
|
+
if (kitId && kitId !== "unknown") {
|
|
1056
|
+
return { claimType: ct, kitId, subject: `${kitId}-kit`, gateId: ct };
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
// 2. No kit-typed claim in bundle — try to derive kit from current.json active_flow_id.
|
|
1061
|
+
// The bundle lives at <session-dir>/trust.bundle, so:
|
|
1062
|
+
// sessionDir = path.dirname(bundleFile)
|
|
1063
|
+
// flowAgentsDir = path.dirname(sessionDir)
|
|
1064
|
+
try {
|
|
1065
|
+
const sessionDir = path.dirname(bundleFile);
|
|
1066
|
+
const flowAgentsDir = path.dirname(sessionDir);
|
|
1067
|
+
const currentFile = path.join(flowAgentsDir, "current.json");
|
|
1068
|
+
const current = JSON.parse(fs.readFileSync(currentFile, "utf8")) as Record<string, unknown>;
|
|
1069
|
+
const flowId = typeof current["active_flow_id"] === "string" ? current["active_flow_id"] : null;
|
|
1070
|
+
if (flowId && flowId.includes(".")) {
|
|
1071
|
+
const kitId = flowId.split(".")[0]!;
|
|
1072
|
+
if (kitId) {
|
|
1073
|
+
const derivedClaimType = `${kitId}.trust.bundle`;
|
|
1074
|
+
return { claimType: derivedClaimType, kitId, subject: `${kitId}-kit`, gateId: derivedClaimType };
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
} catch {
|
|
1078
|
+
// Ignore — fall through to unknown
|
|
1079
|
+
}
|
|
1080
|
+
// 3. Genuinely unknown — never fallback to "builder".
|
|
1081
|
+
return { claimType: "unknown.trust.bundle", kitId: "unknown", subject: "unknown-kit", gateId: "unknown.trust.bundle" };
|
|
1082
|
+
}
|
|
1029
1083
|
function surfaceCheckFromArtifact(file: string, index: number): AnyObj {
|
|
1030
1084
|
const raw = JSON.parse(read(file));
|
|
1031
1085
|
const lower = JSON.stringify(raw).toLowerCase();
|
|
1086
|
+
// Structurally read kit identity from the bundle — never hardcode "builder".
|
|
1087
|
+
const { claimType: bundleClaimType, subject: bundleSubject, gateId: bundleGateId } = kitIdentityFromBundle(raw, file);
|
|
1032
1088
|
let ref: AnyObj;
|
|
1033
1089
|
if (lower.includes("provider") && lower.includes("absent")) {
|
|
1034
|
-
ref = { artifact_kind: "trust.bundle", artifact_ref: file, gate_id: "provider.unavailable", claim_type:
|
|
1090
|
+
ref = { artifact_kind: "trust.bundle", artifact_ref: file, gate_id: "provider.unavailable", claim_type: bundleClaimType, claim_status: "unknown", subject: bundleSubject, freshness: { status: "unknown", summary: "No trust provider is configured" }, authority: { producer: "unknown", summary: "No trust provider is configured" }, integrity: { status: "unknown", summary: "Unknown" }, status: "not_verified", summary: "No trust provider is configured" };
|
|
1035
1091
|
} else if (lower.includes("artifact") && lower.includes("absent")) {
|
|
1036
|
-
ref = { artifact_kind: "trust.bundle", artifact_ref: file, gate_id: "artifact.unavailable", claim_type:
|
|
1092
|
+
ref = { artifact_kind: "trust.bundle", artifact_ref: file, gate_id: "artifact.unavailable", claim_type: bundleClaimType, claim_status: "unknown", subject: bundleSubject, freshness: { status: "unknown", summary: "Artifact not readable" }, authority: { producer: "unknown", summary: "Artifact not readable" }, integrity: { status: "unknown", summary: "Artifact not readable" }, status: "not_verified", summary: "artifact not readable" };
|
|
1037
1093
|
} else {
|
|
1038
1094
|
const claimStatus = lower.includes("rejected") ? "rejected" : "accepted";
|
|
1039
1095
|
const freshness = lower.includes("stale") ? "stale" : "fresh";
|
|
1040
1096
|
const producer = lower.includes("missing-authority") ? "unknown" : "surface-local";
|
|
1041
1097
|
const integrity = lower.includes("mismatch") ? "mismatch" : "matched";
|
|
1042
1098
|
// Use trust.bundle as the canonical Hachure-aligned artifact_kind for all trust-backed evidence refs
|
|
1043
|
-
ref = { artifact_kind: "trust.bundle", artifact_ref: file, gate_id:
|
|
1099
|
+
ref = { artifact_kind: "trust.bundle", artifact_ref: file, gate_id: bundleGateId, claim_type: bundleClaimType, claim_status: claimStatus, subject: bundleSubject, freshness: { status: freshness, summary: freshness === "fresh" ? "fresh" : "not currently verifiable" }, authority: { producer, summary: producer === "unknown" ? "missing authority" : "Local Surface trust producer." }, integrity: { status: integrity, summary: integrity === "matched" ? "matched" : "integrity mismatch" } };
|
|
1044
1100
|
ref.status = deriveSurfaceStatus(ref);
|
|
1045
1101
|
ref.summary = ref.status === "pass" ? "accepted" : ref.status === "not_verified" ? "not currently verifiable" : (claimStatus === "rejected" ? "rejected" : producer === "unknown" ? "missing authority" : "integrity mismatch");
|
|
1046
1102
|
}
|
|
@@ -1279,14 +1335,20 @@ async function advanceState(p: ReturnType<typeof parseArgs>): Promise<number> {
|
|
|
1279
1335
|
const prev = loadJson(path.join(dir, "state.json"));
|
|
1280
1336
|
if ((status === "archived" || status === "accepted") && prev.phase !== "learning") diagnostic(dir, "terminal_jump_rejected", "Terminal workflow states require release and learning gates.");
|
|
1281
1337
|
const flow = opt(p, "flow-definition");
|
|
1282
|
-
|
|
1338
|
+
// Route-back guard: FlowDefinition-driven (not hardcoded to builder.build).
|
|
1339
|
+
// Fires when the active flow's gate for prev.phase declares a route_back_policy
|
|
1340
|
+
// AND the target phase maps to a step listed in on_route_back values.
|
|
1341
|
+
// builder.build verify-gate already carries this declaration — behavior preserved.
|
|
1342
|
+
const repoRoot = flow ? findRepoRootFromDir(dir) : "";
|
|
1343
|
+
const routeBack = flow ? resolveRouteBackPolicy(flow, prev.phase, phase, repoRoot) : null;
|
|
1344
|
+
if (routeBack) {
|
|
1283
1345
|
const reason = opt(p, "route-back-reason");
|
|
1284
|
-
if (!reason) diagnostic(dir, "route_back_reason_required",
|
|
1346
|
+
if (!reason) diagnostic(dir, "route_back_reason_required", `Route-back from ${prev.phase} to ${phase} requires a --route-back-reason (e.g. implementation_defect).`);
|
|
1285
1347
|
const file = path.join(dir, "transition-attempts.json");
|
|
1286
1348
|
const attempts = loadJson(file);
|
|
1287
|
-
const key =
|
|
1349
|
+
const key = `${prev.phase}->${phase}:${reason}`;
|
|
1288
1350
|
const count = attempts[key]?.count ?? 0;
|
|
1289
|
-
if (count >=
|
|
1351
|
+
if (count >= routeBack.maxAttempts) diagnostic(dir, "route_back_attempts_exceeded", `Route-back attempt limit (${routeBack.maxAttempts}) exceeded for ${prev.phase}→${phase}.`);
|
|
1290
1352
|
attempts[key] = { count: count + 1, reason, updated_at: opt(p, "timestamp", now()) };
|
|
1291
1353
|
writeJson(file, attempts);
|
|
1292
1354
|
}
|
|
@@ -1300,7 +1362,7 @@ async function advanceState(p: ReturnType<typeof parseArgs>): Promise<number> {
|
|
|
1300
1362
|
// --step-id individually. The repoRoot is derived by walking up from dir to find kits/.
|
|
1301
1363
|
if (flow) {
|
|
1302
1364
|
const root = path.resolve(opt(p, "artifact-root", path.dirname(dir)));
|
|
1303
|
-
|
|
1365
|
+
// repoRoot already computed above when flow is present
|
|
1304
1366
|
const phaseMap = resolvePhaseMap(flow, repoRoot);
|
|
1305
1367
|
const stepId = phaseMap?.[phase] ?? undefined;
|
|
1306
1368
|
if (stepId) {
|
package/src/lib/flow-resolver.ts
CHANGED
|
@@ -124,6 +124,10 @@ export type ActiveFlowStep = {
|
|
|
124
124
|
type FlowGate = {
|
|
125
125
|
step: string;
|
|
126
126
|
expects?: GateExpectation[];
|
|
127
|
+
/** Reason-to-step mapping for route-back transitions (e.g. {"implementation_defect": "execute"}). */
|
|
128
|
+
on_route_back?: Record<string, string>;
|
|
129
|
+
/** Policy governing route-back attempt limits. */
|
|
130
|
+
route_back_policy?: { max_attempts: number; on_exceeded: string };
|
|
127
131
|
};
|
|
128
132
|
|
|
129
133
|
/** Shape of a FlowDefinition JSON file. */
|
|
@@ -282,3 +286,84 @@ export function resolveActiveFlowStep(flowAgentsDir: string): ActiveFlowStep | n
|
|
|
282
286
|
const repoRoot = findRepoRoot(path.dirname(flowAgentsDir));
|
|
283
287
|
return resolveFlowStep(flowId, stepId, repoRoot);
|
|
284
288
|
}
|
|
289
|
+
|
|
290
|
+
/** The resolved route-back policy for a phase transition. */
|
|
291
|
+
export type RouteBackPolicy = {
|
|
292
|
+
/** Maximum allowed route-back attempts for this transition key. */
|
|
293
|
+
maxAttempts: number;
|
|
294
|
+
/** Action when attempts are exceeded (e.g. "block"). */
|
|
295
|
+
onExceeded: string;
|
|
296
|
+
/** The step id whose gate declared this policy (e.g. "verify"). */
|
|
297
|
+
fromStepId: string;
|
|
298
|
+
};
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* Resolve the route-back policy for a phase transition, if the active FlowDefinition
|
|
302
|
+
* declares one on the source phase's gate.
|
|
303
|
+
*
|
|
304
|
+
* A route-back is a transition where the source phase's gate declares both
|
|
305
|
+
* `route_back_policy` and `on_route_back`, and the target phase maps to a step
|
|
306
|
+
* listed as a route-back target in `on_route_back` values.
|
|
307
|
+
*
|
|
308
|
+
* This is the FlowDefinition-driven replacement for the hardcoded
|
|
309
|
+
* `flow === "builder.build" && prev.phase === "verification" && phase === "execution"`
|
|
310
|
+
* guard in advance-state. Any flow that declares `route_back_policy` on a gate
|
|
311
|
+
* automatically gets route-back enforcement without code changes.
|
|
312
|
+
*
|
|
313
|
+
* @param flowId e.g. "builder.build" — kitId is the prefix before the first ".".
|
|
314
|
+
* @param fromPhase Lifecycle phase leaving (e.g. "verification").
|
|
315
|
+
* @param toPhase Lifecycle phase entering (e.g. "execution").
|
|
316
|
+
* @param repoRoot Absolute path to the repository root (kits/ lives here).
|
|
317
|
+
* @returns RouteBackPolicy when the transition is a declared route-back, null otherwise.
|
|
318
|
+
*/
|
|
319
|
+
export function resolveRouteBackPolicy(
|
|
320
|
+
flowId: string,
|
|
321
|
+
fromPhase: string,
|
|
322
|
+
toPhase: string,
|
|
323
|
+
repoRoot: string,
|
|
324
|
+
): RouteBackPolicy | null {
|
|
325
|
+
if (!flowId || !fromPhase || !toPhase) return null;
|
|
326
|
+
const dotIdx = flowId.indexOf(".");
|
|
327
|
+
if (dotIdx < 1) return null;
|
|
328
|
+
const kitId = flowId.slice(0, dotIdx);
|
|
329
|
+
const flowName = flowId.slice(dotIdx + 1);
|
|
330
|
+
if (!kitId || !flowName) return null;
|
|
331
|
+
|
|
332
|
+
const flowFilePath = resolveFlowFilePath(kitId, flowName, flowId, repoRoot);
|
|
333
|
+
if (!flowFilePath) return null;
|
|
334
|
+
|
|
335
|
+
let flowDef: FlowDefinition;
|
|
336
|
+
try {
|
|
337
|
+
const raw = fs.readFileSync(flowFilePath, "utf8");
|
|
338
|
+
flowDef = JSON.parse(raw) as FlowDefinition;
|
|
339
|
+
} catch {
|
|
340
|
+
return null; // ENOENT, permission error, or parse error — fail-open
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
if (!flowDef || typeof flowDef !== "object") return null;
|
|
344
|
+
const phaseMap = flowDef.phase_map;
|
|
345
|
+
if (!phaseMap || typeof phaseMap !== "object" || Array.isArray(phaseMap)) return null;
|
|
346
|
+
|
|
347
|
+
const fromStep = phaseMap[fromPhase];
|
|
348
|
+
const toStep = phaseMap[toPhase];
|
|
349
|
+
if (!fromStep || !toStep) return null; // phases not in this flow
|
|
350
|
+
|
|
351
|
+
if (!flowDef.gates) return null;
|
|
352
|
+
for (const gate of Object.values(flowDef.gates)) {
|
|
353
|
+
if (!gate || gate.step !== fromStep) continue;
|
|
354
|
+
if (!gate.route_back_policy || !gate.on_route_back) return null;
|
|
355
|
+
// Check if toStep is a valid route-back target declared in on_route_back
|
|
356
|
+
const routeBackTargets = Object.values(gate.on_route_back);
|
|
357
|
+
if (!routeBackTargets.includes(toStep)) return null;
|
|
358
|
+
const maxAttempts =
|
|
359
|
+
typeof gate.route_back_policy.max_attempts === "number"
|
|
360
|
+
? gate.route_back_policy.max_attempts
|
|
361
|
+
: 3;
|
|
362
|
+
return {
|
|
363
|
+
maxAttempts,
|
|
364
|
+
onExceeded: gate.route_back_policy.on_exceeded ?? "block",
|
|
365
|
+
fromStepId: fromStep,
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
return null;
|
|
369
|
+
}
|