@kontourai/flow-agents 3.1.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +4 -0
- package/CHANGELOG.md +17 -0
- package/build/src/cli/assignment-provider.d.ts +45 -0
- package/build/src/cli/assignment-provider.js +97 -12
- package/build/src/cli/workflow-sidecar.d.ts +14 -4
- package/build/src/cli/workflow-sidecar.js +100 -10
- package/context/contracts/assignment-provider-contract.md +1 -1
- package/context/contracts/execution-contract.md +78 -0
- package/context/scripts/hooks/config-protection.js +11 -4
- package/context/scripts/hooks/stop-goal-fit.js +259 -4
- package/docs/adr/0022-fail-closed-delivery-reconciliation-with-governed-exemptions.md +111 -0
- package/evals/ci/run-baseline.sh +2 -0
- package/evals/integration/test_checkpoint_signing.sh +4 -3
- package/evals/integration/test_gate_lockdown.sh +36 -0
- package/evals/integration/test_model_routing_escalation.sh +145 -0
- package/evals/integration/test_publish_delivery.sh +14 -6
- package/evals/integration/test_stop_hook_release.sh +552 -0
- package/evals/integration/test_trust_reconcile_negatives.sh +170 -0
- package/evals/run.sh +6 -0
- package/evals/static/test_model_routing_hints.sh +107 -0
- package/kits/builder/skills/builder-shape/SKILL.md +10 -0
- package/kits/builder/skills/deliver/SKILL.md +52 -11
- package/kits/builder/skills/design-probe/SKILL.md +10 -0
- package/kits/builder/skills/execute-plan/SKILL.md +13 -0
- package/kits/builder/skills/fix-bug/SKILL.md +17 -0
- package/kits/builder/skills/idea-to-backlog/SKILL.md +10 -0
- package/kits/builder/skills/plan-work/SKILL.md +9 -0
- package/kits/builder/skills/pull-work/SKILL.md +10 -0
- package/kits/builder/skills/review-work/SKILL.md +11 -0
- package/kits/builder/skills/tdd-workflow/SKILL.md +17 -0
- package/kits/builder/skills/verify-work/SKILL.md +11 -0
- package/kits/knowledge/adapters/default-store/index.js +56 -15
- package/kits/knowledge/adapters/flow-runner/index.js +912 -16
- package/kits/knowledge/adapters/obsidian-store/index.js +29 -11
- package/kits/knowledge/adapters/shared/codec.js +124 -0
- package/kits/knowledge/docs/store-contract.md +405 -3
- package/kits/knowledge/evals/audit-freshness/suite.test.js +92 -1
- package/kits/knowledge/evals/consolidate-incremental/suite.test.js +494 -0
- package/kits/knowledge/evals/consolidation/suite.test.js +1 -1
- package/kits/knowledge/evals/contract-suite/suite.test.js +36 -0
- package/kits/knowledge/evals/freshness/suite.test.js +339 -0
- package/kits/knowledge/evals/inbound-references/suite.test.js +351 -0
- package/kits/knowledge/evals/retirement/suite.test.js +1 -1
- package/kits/knowledge/evals/supersede-propagation/suite.test.js +384 -0
- package/package.json +1 -1
- package/schemas/workflow-handoff.schema.json +6 -0
- package/scripts/ci/mint-attestation.js +33 -6
- package/scripts/ci/trust-reconcile.js +144 -26
- package/scripts/hooks/config-protection.js +11 -4
- package/scripts/hooks/stop-goal-fit.js +259 -4
- package/src/cli/assignment-provider.ts +110 -12
- package/src/cli/workflow-sidecar.ts +99 -10
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
'use strict';
|
|
26
26
|
|
|
27
27
|
const fs = require('fs');
|
|
28
|
+
const os = require('os');
|
|
28
29
|
const path = require('path');
|
|
29
30
|
const { spawnSync } = require('child_process');
|
|
30
31
|
const crypto = require('crypto');
|
|
@@ -42,7 +43,7 @@ const {
|
|
|
42
43
|
flowAgentsArtifactRoot,
|
|
43
44
|
flowAgentsArtifactRootsForRead,
|
|
44
45
|
} = require('./lib/local-artifact-paths');
|
|
45
|
-
const { resolveActor } = require('./lib/actor-identity.js');
|
|
46
|
+
const { resolveActor, isUnresolvedActor, detectRuntime } = require('./lib/actor-identity.js');
|
|
46
47
|
const { readCurrentPointer } = require('./lib/current-pointer.js');
|
|
47
48
|
|
|
48
49
|
const MAX_STDIN = 1024 * 1024;
|
|
@@ -1671,7 +1672,7 @@ async function analyze(root, now = Date.now()) {
|
|
|
1671
1672
|
artifacts = artifacts.filter(a => a && hasSidecarPresence(path.dirname(a.file)));
|
|
1672
1673
|
}
|
|
1673
1674
|
|
|
1674
|
-
if (artifacts.length === 0) return { warnings: [], blocking: false };
|
|
1675
|
+
if (artifacts.length === 0) return { warnings: [], blocking: false, latestArtifactDir: null };
|
|
1675
1676
|
|
|
1676
1677
|
const latest = artifacts[0];
|
|
1677
1678
|
const latestArtifactDir = path.dirname(latest.file);
|
|
@@ -1745,7 +1746,7 @@ async function analyze(root, now = Date.now()) {
|
|
|
1745
1746
|
if (/\[backstop in warn mode — not blocking\]/.test(w)) return false;
|
|
1746
1747
|
return blockRe.test(w);
|
|
1747
1748
|
});
|
|
1748
|
-
return { warnings, blocking, preExecution, gatePrefix: gateLabel(activeFlowStep) };
|
|
1749
|
+
return { warnings, blocking, preExecution, gatePrefix: gateLabel(activeFlowStep), latestArtifactDir };
|
|
1749
1750
|
}
|
|
1750
1751
|
|
|
1751
1752
|
/**
|
|
@@ -1844,12 +1845,266 @@ function remediationFor(warning) {
|
|
|
1844
1845
|
return null;
|
|
1845
1846
|
}
|
|
1846
1847
|
|
|
1848
|
+
// ─── #292 Wave 2: Stop-hook non-terminal release-with-handoff ────────────────
|
|
1849
|
+
//
|
|
1850
|
+
// When a session's Stop event fires and the session's state.json status is NOT
|
|
1851
|
+
// one of workflow-sidecar.ts's LIVENESS_TERMINAL statuses (delivered/accepted/
|
|
1852
|
+
// archived — the SAME set advance-state's terminal path already tests against,
|
|
1853
|
+
// imported here rather than re-declared, per AC8), the liveness claim and (for
|
|
1854
|
+
// the local-file assignment provider) the durable assignment claim would
|
|
1855
|
+
// otherwise sit held until TTL, even though the session has genuinely ended.
|
|
1856
|
+
// This closes that gap: always emit a provider-agnostic liveness release (AC1),
|
|
1857
|
+
// and for local-file additionally perform the real release inline (AC2), while
|
|
1858
|
+
// honestly disclosing (never executing) the equivalent GitHub-provider release
|
|
1859
|
+
// as a pending handoff.json intent (AC3). See the plan artifact
|
|
1860
|
+
// (.kontourai/flow-agents/kontourai-flow-agents-292/kontourai-flow-agents-292--plan-work.md,
|
|
1861
|
+
// "Wave 2") for the full design rationale and the render-don't-execute
|
|
1862
|
+
// constraint this deliberately does not cross.
|
|
1863
|
+
//
|
|
1864
|
+
// Fail-open (AC7): the ENTIRE body runs under one try/catch. Any failure —
|
|
1865
|
+
// missing build/, corrupt assignment record, unresolved actor, missing/
|
|
1866
|
+
// malformed provider settings — degrades to a single stderr diagnostic and
|
|
1867
|
+
// returns, never throwing, never affecting the Stop hook's own exit code or
|
|
1868
|
+
// goal-fit's warnings/blocking output (this function's return value is not
|
|
1869
|
+
// consumed by that contract at all — it is a pure side effect).
|
|
1870
|
+
|
|
1871
|
+
/**
|
|
1872
|
+
* require() the compiled workflow-sidecar.js the same hasBuild/fs.existsSync-guarded
|
|
1873
|
+
* way loadActiveFlowStep() already requires flow-resolver.js. Returns null (never
|
|
1874
|
+
* throws) when build/ is absent or the require fails — the caller treats null as
|
|
1875
|
+
* "the LIVENESS_TERMINAL boundary is unknown; do not guess it" and skips the whole
|
|
1876
|
+
* release (fail-open, never a re-derived duplicate Set — AC8).
|
|
1877
|
+
*/
|
|
1878
|
+
function loadWorkflowSidecarBuilt() {
|
|
1879
|
+
const packageRoot = path.resolve(__dirname, '..', '..');
|
|
1880
|
+
const builtSidecar = path.join(packageRoot, 'build', 'src', 'cli', 'workflow-sidecar.js');
|
|
1881
|
+
if (!fs.existsSync(builtSidecar)) return null; // hasBuild guard
|
|
1882
|
+
try {
|
|
1883
|
+
const mod = require(builtSidecar);
|
|
1884
|
+
if (!(mod.LIVENESS_TERMINAL instanceof Set)) return null;
|
|
1885
|
+
return mod;
|
|
1886
|
+
} catch {
|
|
1887
|
+
return null; // require failed — fail-open
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
|
|
1891
|
+
/** Same hasBuild-guarded require idiom, for build/src/cli/assignment-provider.js. */
|
|
1892
|
+
function loadAssignmentProviderBuilt() {
|
|
1893
|
+
const packageRoot = path.resolve(__dirname, '..', '..');
|
|
1894
|
+
const builtProvider = path.join(packageRoot, 'build', 'src', 'cli', 'assignment-provider.js');
|
|
1895
|
+
if (!fs.existsSync(builtProvider)) return null;
|
|
1896
|
+
try {
|
|
1897
|
+
const mod = require(builtProvider);
|
|
1898
|
+
if (typeof mod.performLocalRelease !== 'function') return null;
|
|
1899
|
+
return mod;
|
|
1900
|
+
} catch {
|
|
1901
|
+
return null;
|
|
1902
|
+
}
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
/**
|
|
1906
|
+
* Resolve the effective assignment-provider kind for this repo ('local-file' | 'github'),
|
|
1907
|
+
* or null when it cannot be determined. Prefers reading context/settings/
|
|
1908
|
+
* assignment-provider-settings.json directly (project settings first, matching
|
|
1909
|
+
* effective-assignment-provider-settings.ts's own lookup precedence) rather than shelling
|
|
1910
|
+
* out to or refactoring that CLI-shaped file (see plan Unresolved Question 1) — this task's
|
|
1911
|
+
* scope is the Stop hook, not that file. Returns null (never guesses) when no settings file
|
|
1912
|
+
* is present/parseable or names no provider kind for a project entry.
|
|
1913
|
+
*
|
|
1914
|
+
* Deliberately reads ONLY the project settings file (context/settings/
|
|
1915
|
+
* assignment-provider-settings.json relative to repo root) — the common case for this repo
|
|
1916
|
+
* and the one confirmed present. The effective() merge/repo-matching logic in
|
|
1917
|
+
* effective-assignment-provider-settings.ts is intentionally NOT reimplemented here (that
|
|
1918
|
+
* would fork a second merge algorithm); a repo whose provider kind can only be resolved via
|
|
1919
|
+
* global settings or multi-project merge degrades to "unknown, skip assignment-layer
|
|
1920
|
+
* release" rather than guessing.
|
|
1921
|
+
*/
|
|
1922
|
+
function resolveAssignmentProviderKind(root) {
|
|
1923
|
+
const settingsFile = path.join(root, 'context', 'settings', 'assignment-provider-settings.json');
|
|
1924
|
+
const settings = readJsonFile(settingsFile);
|
|
1925
|
+
if (!settings) return null;
|
|
1926
|
+
const candidates = [];
|
|
1927
|
+
if (settings.defaults && settings.defaults.provider) candidates.push(settings.defaults.provider);
|
|
1928
|
+
if (Array.isArray(settings.projects)) {
|
|
1929
|
+
for (const project of settings.projects) {
|
|
1930
|
+
if (project && project.provider) candidates.push(project.provider);
|
|
1931
|
+
}
|
|
1932
|
+
}
|
|
1933
|
+
const kind = candidates.map(p => p && p.kind).find(k => k === 'local-file' || k === 'github');
|
|
1934
|
+
return kind || null;
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
/**
|
|
1938
|
+
* Refresh handoff.json by merge (read+spread the existing file's JSON, overwrite only the
|
|
1939
|
+
* fields this task owns), mirroring advanceState's own handoff-write shape
|
|
1940
|
+
* (`{ ...loadJson(...), ...sidecarBase(slug), summary, current_state_ref: "state.json",
|
|
1941
|
+
* next_steps, blockers: [], warnings: [] }`) and current-pointer.js's writePerActorCurrent
|
|
1942
|
+
* write idiom (`fs.writeFileSync(file, JSON.stringify(payload, null, 2) + "\n")`). next_steps
|
|
1943
|
+
* is appended to (not clobbered) when a provider-release next-step is due.
|
|
1944
|
+
*/
|
|
1945
|
+
function refreshHandoffAfterRelease(artifactDir, slug, state, providerReleasePending, providerReleaseNextCommand) {
|
|
1946
|
+
const file = path.join(artifactDir, 'handoff.json');
|
|
1947
|
+
const existing = readJsonFile(file) || {};
|
|
1948
|
+
const status = state ? normalizedStatus(state.status || 'unknown') : 'unknown';
|
|
1949
|
+
const phase = state ? normalizedStatus(state.phase || 'unknown') : 'unknown';
|
|
1950
|
+
const summary = (state && state.next_action && state.next_action.summary)
|
|
1951
|
+
? String(state.next_action.summary)
|
|
1952
|
+
: `session ended at status:${status} phase:${phase}`;
|
|
1953
|
+
const existingNextSteps = Array.isArray(existing.next_steps) ? existing.next_steps.slice() : [];
|
|
1954
|
+
const nextSteps = existingNextSteps.slice();
|
|
1955
|
+
if (providerReleasePending && providerReleaseNextCommand && !nextSteps.includes(providerReleaseNextCommand)) {
|
|
1956
|
+
nextSteps.push(providerReleaseNextCommand);
|
|
1957
|
+
}
|
|
1958
|
+
const payload = {
|
|
1959
|
+
...existing,
|
|
1960
|
+
schema_version: existing.schema_version || '1.0',
|
|
1961
|
+
task_slug: existing.task_slug || slug,
|
|
1962
|
+
summary,
|
|
1963
|
+
current_state_ref: 'state.json',
|
|
1964
|
+
next_steps: nextSteps,
|
|
1965
|
+
...(providerReleasePending ? { provider_release_pending: true } : {}),
|
|
1966
|
+
...(providerReleaseNextCommand ? { provider_release_next_command: providerReleaseNextCommand } : {}),
|
|
1967
|
+
};
|
|
1968
|
+
fs.writeFileSync(file, `${JSON.stringify(payload, null, 2)}\n`);
|
|
1969
|
+
}
|
|
1970
|
+
|
|
1971
|
+
/**
|
|
1972
|
+
* #292 Wave 2: on a Stop event whose resolved session is at a NON-terminal status, release
|
|
1973
|
+
* the session's liveness claim (always, provider-agnostic — AC1) and, for the local-file
|
|
1974
|
+
* assignment provider, its durable assignment claim too (AC2), then refresh handoff.json
|
|
1975
|
+
* (AC4). A terminal-status session (already released by advance-state) is a deliberate no-op
|
|
1976
|
+
* (AC5). For the github provider, never calls render-release or gh — only records an
|
|
1977
|
+
* honestly-labeled pending intent in handoff.json (AC3). Actor-scoped: only ever releases
|
|
1978
|
+
* the CURRENT actor's own held claim (AC6, enforced inside performLocalRelease).
|
|
1979
|
+
*
|
|
1980
|
+
* Called once from run(), reusing analyze()'s already-resolved latestArtifactDir — this
|
|
1981
|
+
* function does not re-derive "what is the active session" a second time.
|
|
1982
|
+
*
|
|
1983
|
+
* Fail-open (AC7): the entire body is wrapped in one try/catch. Any failure degrades to a
|
|
1984
|
+
* single stderr diagnostic and returns, never throwing, never affecting the Stop hook's exit
|
|
1985
|
+
* code or goal-fit's warnings/blocking contract (this function's return value is unused by
|
|
1986
|
+
* that contract).
|
|
1987
|
+
*
|
|
1988
|
+
* @param {string} root - repo root (as resolved by findRepoRoot)
|
|
1989
|
+
* @param {string|null} artifactDir - analyze()'s already-resolved latestArtifactDir
|
|
1990
|
+
*/
|
|
1991
|
+
function releaseOnNonTerminalStop(root, artifactDir) {
|
|
1992
|
+
try {
|
|
1993
|
+
if (!artifactDir) return; // no active session resolved — nothing to release.
|
|
1994
|
+
|
|
1995
|
+
const state = readJsonFile(path.join(artifactDir, 'state.json'));
|
|
1996
|
+
if (!state) return; // AC5: no state.json — nothing to gate a release decision on.
|
|
1997
|
+
|
|
1998
|
+
const sidecar = loadWorkflowSidecarBuilt();
|
|
1999
|
+
if (!sidecar) {
|
|
2000
|
+
process.stderr.write('[Hook] Goal Fit: stop-hook release skipped — build/src/cli/workflow-sidecar.js not available (LIVENESS_TERMINAL boundary unknown).\n');
|
|
2001
|
+
return;
|
|
2002
|
+
}
|
|
2003
|
+
|
|
2004
|
+
const status = normalizedStatus(state.status || 'unknown');
|
|
2005
|
+
if (sidecar.LIVENESS_TERMINAL.has(status)) return; // AC5: terminal — advance-state already released.
|
|
2006
|
+
|
|
2007
|
+
// Slug: artifact dir's basename, or state.json's own task_slug field — reuse whichever
|
|
2008
|
+
// this file already has, never a new slug-derivation rule.
|
|
2009
|
+
const slug = (state.task_slug && String(state.task_slug).trim()) || path.basename(artifactDir);
|
|
2010
|
+
|
|
2011
|
+
const { actor } = resolveActor(process.env);
|
|
2012
|
+
if (isUnresolvedActor(actor)) {
|
|
2013
|
+
process.stderr.write(`[Hook] Goal Fit: stop-hook release skipped for "${safeOneLine(slug, 80)}" — actor identity is unresolved; never releasing under a synthetic identity.\n`);
|
|
2014
|
+
return;
|
|
2015
|
+
}
|
|
2016
|
+
|
|
2017
|
+
const flowAgentsDir = path.dirname(artifactDir);
|
|
2018
|
+
const nowIso = new Date().toISOString();
|
|
2019
|
+
|
|
2020
|
+
// AC1: ALWAYS emit the liveness release, regardless of provider kind — liveness is
|
|
2021
|
+
// provider-agnostic and this is the ONE shared writer every other liveness emitter uses.
|
|
2022
|
+
try {
|
|
2023
|
+
require('./lib/liveness-write.js').appendLivenessEvent(flowAgentsDir, {
|
|
2024
|
+
type: 'release',
|
|
2025
|
+
subjectId: slug,
|
|
2026
|
+
actor,
|
|
2027
|
+
at: nowIso,
|
|
2028
|
+
source: 'stop-hook',
|
|
2029
|
+
});
|
|
2030
|
+
} catch (err) {
|
|
2031
|
+
process.stderr.write(`[Hook] Goal Fit: stop-hook liveness release failed for "${safeOneLine(slug, 80)}": ${safeOneLine(err && err.message || err, 160)}\n`);
|
|
2032
|
+
}
|
|
2033
|
+
|
|
2034
|
+
// Resolve the effective assignment-provider kind — unknown means "do not guess",
|
|
2035
|
+
// skip the assignment-layer release entirely (fail-open).
|
|
2036
|
+
const providerKind = resolveAssignmentProviderKind(root);
|
|
2037
|
+
if (!providerKind) {
|
|
2038
|
+
process.stderr.write(`[Hook] Goal Fit: stop-hook release — provider kind unknown for "${safeOneLine(slug, 80)}", skipping assignment-layer release (liveness release already emitted).\n`);
|
|
2039
|
+
refreshHandoffAfterRelease(artifactDir, slug, state, false, null);
|
|
2040
|
+
return;
|
|
2041
|
+
}
|
|
2042
|
+
|
|
2043
|
+
let providerReleasePending = false;
|
|
2044
|
+
let providerReleaseNextCommand = null;
|
|
2045
|
+
|
|
2046
|
+
if (providerKind === 'local-file') {
|
|
2047
|
+
const provider = loadAssignmentProviderBuilt();
|
|
2048
|
+
if (!provider) {
|
|
2049
|
+
process.stderr.write('[Hook] Goal Fit: stop-hook local-file release skipped — build/src/cli/assignment-provider.js not available.\n');
|
|
2050
|
+
} else {
|
|
2051
|
+
try {
|
|
2052
|
+
// Mirror assignment-provider.ts's loadActorStruct() exactly: runtime is derived via
|
|
2053
|
+
// detectRuntime (not a literal 'unknown'), since serializeActor()'s comparison key
|
|
2054
|
+
// includes runtime — a mismatched runtime segment here would make performLocalRelease's
|
|
2055
|
+
// AC6 actor-match check wrongly treat this session's own claim as foreign.
|
|
2056
|
+
//
|
|
2057
|
+
// actorKey: opts.actorKey MUST be the same canonical `resolveActor(env).actor` string
|
|
2058
|
+
// already resolved above (`actor`) — the identical bare-or-triple form
|
|
2059
|
+
// computeEffectiveState()'s holderActorKey preference (`record.actor_key ||
|
|
2060
|
+
// serializeActor(record.actor)`) and performLocalRelease's mirrored ownership check
|
|
2061
|
+
// both key on. Without this, an explicit-override actor's release-side comparison
|
|
2062
|
+
// would fall back to serializeActor(releasedBy) — a re-derived triple that never
|
|
2063
|
+
// equals the stored bare actor_key — and this hook would wrongly treat its own claim
|
|
2064
|
+
// as foreign (the #291 seam, relocated to the release path; see performLocalRelease's
|
|
2065
|
+
// doc comment in src/cli/assignment-provider.ts).
|
|
2066
|
+
const releasedBy = { runtime: detectRuntime(process.env), session_id: actor, host: os.hostname(), human: null };
|
|
2067
|
+
const artifactRoot = flowAgentsArtifactRoot(root);
|
|
2068
|
+
provider.performLocalRelease(artifactRoot, slug, releasedBy, {
|
|
2069
|
+
reason: 'stop-hook non-terminal release',
|
|
2070
|
+
tolerateNoActiveClaim: true,
|
|
2071
|
+
actorKey: actor,
|
|
2072
|
+
});
|
|
2073
|
+
} catch (err) {
|
|
2074
|
+
process.stderr.write(`[Hook] Goal Fit: stop-hook local-file release failed for "${safeOneLine(slug, 80)}": ${safeOneLine(err && err.message || err, 160)}\n`);
|
|
2075
|
+
}
|
|
2076
|
+
}
|
|
2077
|
+
} else {
|
|
2078
|
+
// AC3: github (or any non-local-file kind) — render-don't-execute. Never call
|
|
2079
|
+
// render-release, never invoke gh. Disclose the pending intent honestly.
|
|
2080
|
+
providerReleasePending = true;
|
|
2081
|
+
providerReleaseNextCommand = `npm run workflow -- assignment-provider status --provider github --subject-id "${slug}" ... # then: assignment-provider render-release --provider github --subject-id "${slug}" ... (emits the gh argv to run)`;
|
|
2082
|
+
}
|
|
2083
|
+
|
|
2084
|
+
// AC4: refresh handoff.json — merge, never fully overwrite.
|
|
2085
|
+
refreshHandoffAfterRelease(artifactDir, slug, state, providerReleasePending, providerReleaseNextCommand);
|
|
2086
|
+
} catch (err) {
|
|
2087
|
+
// Fail-open (AC7): any unexpected failure anywhere above (JSON parse error, fs error,
|
|
2088
|
+
// etc.) must never crash the Stop hook or affect its exit code / goal-fit's output.
|
|
2089
|
+
try {
|
|
2090
|
+
process.stderr.write(`[Hook] Goal Fit: stop-hook release logic error (fail-open): ${safeOneLine(err && err.message || err, 160)}\n`);
|
|
2091
|
+
} catch { /* best-effort diagnostic only */ }
|
|
2092
|
+
}
|
|
2093
|
+
}
|
|
2094
|
+
|
|
1847
2095
|
async function run(rawInput) {
|
|
1848
2096
|
const input = parseJson(rawInput);
|
|
1849
2097
|
const root = findRepoRoot(input.cwd || process.cwd());
|
|
1850
2098
|
const mode = resolveGoalFitMode();
|
|
1851
2099
|
if (mode === 'off') return rawInput;
|
|
1852
2100
|
const result = await analyze(root);
|
|
2101
|
+
// #292 Wave 2: additive side effect only — never changes analyze()'s warnings/blocking
|
|
2102
|
+
// contract or this function's return value. Reuses analyze()'s already-resolved
|
|
2103
|
+
// latestArtifactDir rather than re-deriving a second "what is the active session" path.
|
|
2104
|
+
// Runs regardless of whether goal-fit itself has any warnings this turn (AC1: the
|
|
2105
|
+
// liveness release must always be attempted on a non-terminal Stop, independent of
|
|
2106
|
+
// whether goal-fit found anything to warn about).
|
|
2107
|
+
releaseOnNonTerminalStop(root, result.latestArtifactDir || null);
|
|
1853
2108
|
if (result.warnings.length === 0) {
|
|
1854
2109
|
clearBlockStreak(root);
|
|
1855
2110
|
return rawInput;
|
|
@@ -1939,4 +2194,4 @@ if (require.main === module) {
|
|
|
1939
2194
|
});
|
|
1940
2195
|
}
|
|
1941
2196
|
|
|
1942
|
-
module.exports = { analyze, run, resolveGoalFitMode, uncheckedInSection, findRepoRoot, sidecarGuidance, safeOneLine, captureCrossReference, bundleEnforcement, loadActiveFlowStep, readCommandLog, resolveTrustedCommand, declaredManifestTarget, verifyCommandLogChain, CHAIN_GENESIS_VERIFY, hasLaunderingOperator };
|
|
2197
|
+
module.exports = { analyze, run, resolveGoalFitMode, uncheckedInSection, findRepoRoot, sidecarGuidance, safeOneLine, captureCrossReference, bundleEnforcement, loadActiveFlowStep, readCommandLog, resolveTrustedCommand, declaredManifestTarget, verifyCommandLogChain, CHAIN_GENESIS_VERIFY, hasLaunderingOperator, releaseOnNonTerminalStop };
|
|
@@ -463,3 +463,114 @@ the workflow to fake the post-merge no-op path; this is closed by required code-
|
|
|
463
463
|
review on `.github/workflows/trust-reconcile.yml` (#225, not yet server-side enforced),
|
|
464
464
|
the same residual class as every other self-asserted CI input this ADR already carries.
|
|
465
465
|
Step 1 (fresh verify) is unaffected by event scoping either way and always runs.
|
|
466
|
+
|
|
467
|
+
## Addendum (2026-07-04, part 5): per-session delivery paths — concurrent deliveries stop contending (#379)
|
|
468
|
+
|
|
469
|
+
Owner-approved follow-up closing a structural defect the fail-closed gate's first real
|
|
470
|
+
deliveries surfaced repeatedly: the delivery transport used a SINGLE shared path
|
|
471
|
+
(`delivery/trust.bundle` + `delivery/trust.checkpoint.json`), so every sealed delivery
|
|
472
|
+
force-committed to the SAME two files. Any two concurrent deliveries therefore
|
|
473
|
+
merge-conflict by construction — three seal collisions inside 24h (#330, #358, #378) each
|
|
474
|
+
needed manual conflict resolution. Worse than the conflict itself: **GitHub schedules NO
|
|
475
|
+
`pull_request` workflows for a conflicting (DIRTY) PR** — zero checks, no error — so the
|
|
476
|
+
required Trust Reconcile check silently never re-runs. The symptom reads as "CI vanished,"
|
|
477
|
+
not "conflict." This collision class scales with delivery frequency; ADR 0021's own "work
|
|
478
|
+
area" vocabulary predicted it, and #335 named the `resolveDeliveryCandidates()` seam as the
|
|
479
|
+
fix site.
|
|
480
|
+
|
|
481
|
+
**Decision (structural): per-session delivery paths.** `publishDelivery()`
|
|
482
|
+
(`src/cli/workflow-sidecar.ts`) writes to `delivery/<slug>/trust.bundle` (+ checkpoint
|
|
483
|
+
companions), where `<slug>` is the session artifact dir's basename, instead of the shared
|
|
484
|
+
flat path. Concurrent deliveries write to DISTINCT files and cannot contend: two deliveries
|
|
485
|
+
add different `delivery/<slug>/` dirs (add/add of different paths — not a conflict), and both
|
|
486
|
+
deleting the same inherited flat/legacy file is a delete/delete (auto-merges — not a
|
|
487
|
+
conflict). The per-session dir NAME is only a collision-avoidance handle; it carries no
|
|
488
|
+
trust weight.
|
|
489
|
+
|
|
490
|
+
**Reconciler side: ownership-aware discovery, prefer-newest, not first-match.**
|
|
491
|
+
`resolveDeliveryCandidates()` now returns the flat path (FIRST, for full back-compat)
|
|
492
|
+
followed by every `delivery/<slug>/<filename>` (sorted). `discoverBundle()` no longer returns
|
|
493
|
+
the first-existing candidate; it collects every candidate that attests THIS change
|
|
494
|
+
(ancestor-or-equal, the SAME `bundleAttestsThisChange()` binding Addendum part 2 defined) and
|
|
495
|
+
selects the one attesting the **NEWEST** (descendant-most) commit. This prefer-newest rule is
|
|
496
|
+
load-bearing in a **merge-commit** repo (this repo's own history has merge commits, e.g.
|
|
497
|
+
release-please merges): an inherited FLAT bundle's `commit_sha` can be a REAL ancestor of HEAD
|
|
498
|
+
— it was committed on the trunk's linear history before this branch point — so it legitimately
|
|
499
|
+
"owns" the change too, and a naive first-fresh-wins would select that STALE inherited bundle
|
|
500
|
+
purely because it sorts first, reconciling the PREVIOUS delivery's claims against THIS change's
|
|
501
|
+
CI. Prefer-newest makes the fresh per-session bundle win on recency, not on being deleted
|
|
502
|
+
first — which in turn is what lets the cleanup policy leave the flat path in place (below)
|
|
503
|
+
without corrupting selection. (Addendum parts 2/4's squash-merge reasoning still holds for
|
|
504
|
+
squash repos; prefer-newest is the strict generalization that also covers merge-commit repos,
|
|
505
|
+
where "inherited ⇒ non-ancestor" is NOT guaranteed.) `extractBundleCommitSha()` now resolves a
|
|
506
|
+
bundle's sibling checkpoint from the bundle's OWN directory (`path.dirname(bundlePath)`), not a
|
|
507
|
+
global scan — a global scan would pair a per-session bundle with the wrong session's (or the
|
|
508
|
+
flat) checkpoint and read the wrong commit binding. For the flat layout this is byte-identical
|
|
509
|
+
to the prior behavior; for per-session it is the only correct pairing.
|
|
510
|
+
|
|
511
|
+
**Back-compat (retained, deprecation-noted).** The flat `delivery/trust.bundle` path stays
|
|
512
|
+
fully supported on the READ side: an already-committed flat bundle from before this change,
|
|
513
|
+
or an external adopter that has not migrated, still resolves and reconciles exactly as
|
|
514
|
+
before (regression-locked by the negatives suite's flat-owner-coexist case and the
|
|
515
|
+
unchanged `test_publish_delivery.sh` TEST 3/4 flat-fixture cases). Only the WRITE side moved
|
|
516
|
+
to per-session — writing to both flat and per-session would re-introduce the very contention
|
|
517
|
+
this closes, so the flat path is write-deprecated, read-supported.
|
|
518
|
+
|
|
519
|
+
**Cleanup policy: supersede-on-publish, per-session dirs only (delivery/ stays bounded).** A
|
|
520
|
+
publishing session prunes every inherited **per-session** seal dir except its own, then
|
|
521
|
+
commits only `delivery/<slug>/`. Per-session dirs are the growth vector (one per delivery) and
|
|
522
|
+
are UNIQUELY named, so pruning one can never conflict with a concurrent PR: two branches
|
|
523
|
+
deleting the same inherited dir is a delete/delete (auto-merges), and each delivery adds its
|
|
524
|
+
own distinct dir. Leaving an inherited per-session dir would be harmless anyway (prefer-newest
|
|
525
|
+
ignores it) — pruning is purely to stop unbounded accumulation. The alternative,
|
|
526
|
+
retain-as-history, was rejected as unbounded growth of permanently-superseded dirs with no
|
|
527
|
+
reader. Pruning is best-effort: a prune failure is logged, never fatal to the delivery.
|
|
528
|
+
|
|
529
|
+
**The shared flat path is deliberately NOT pruned per-delivery.** An earlier iteration of this
|
|
530
|
+
addendum also pruned the flat `delivery/trust.bundle` legacy seals on every publish (to
|
|
531
|
+
"migrate off" the shared path). That was reverted: during the migration window a concurrent PR
|
|
532
|
+
may still seal to the flat path (this was written while PR #370 had an open flat-path seal),
|
|
533
|
+
and a per-delivery deletion of that file is a **modify/delete conflict** against such a PR → a
|
|
534
|
+
DIRTY PR → precisely the no-CI failure mode this whole change exists to remove. Because the
|
|
535
|
+
flat path is a single fixed location (not a growth vector) and prefer-newest selection makes a
|
|
536
|
+
lingering flat bundle harmless, retaining it costs nothing. Removing the flat legacy seals is a
|
|
537
|
+
one-time cleanup for a **dedicated** PR once no open PR still seals to the flat path — not
|
|
538
|
+
something safely bundled into every delivery.
|
|
539
|
+
|
|
540
|
+
**The SILENT failure mode (documented, not "solved").** No repo-side code can make GitHub
|
|
541
|
+
run `pull_request` workflows on a conflicted PR — that is platform behavior. Per-session
|
|
542
|
+
paths remove the STRUCTURAL cause (the shared-path conflict) for agent-vs-agent delivery
|
|
543
|
+
contention, which is the overwhelming majority of the incidents. The residual — a PR that
|
|
544
|
+
goes DIRTY for some OTHER reason (a genuine same-file edit conflict with `main`) still gets
|
|
545
|
+
no CI silently — is addressed by making the failure mode LOUD where we can: the deliver
|
|
546
|
+
skill now documents the DIRTY→no-CI symptom and its diagnosis (`gh pr view --json
|
|
547
|
+
mergeStateStatus`), and `discoverBundle()` emits a grep-stable `#379: examined N delivery
|
|
548
|
+
candidate(s) … none attests this change …` concurrency hint so the next agent can tell a
|
|
549
|
+
per-session collision apart from a plain stale/absent bundle. #335's detectability half
|
|
550
|
+
(treating `mergeStateStatus=DIRTY` as a first-class steering/doctor input) remains open and
|
|
551
|
+
is explicitly NOT claimed closed here.
|
|
552
|
+
|
|
553
|
+
**Security: the forgery surface moved with the write path.** `scripts/hooks/config-protection.js`
|
|
554
|
+
(and its `context/` mirror) protected only the flat `delivery/trust.bundle` /
|
|
555
|
+
`delivery/trust.checkpoint.json` from direct agent Write/Edit/cp/redirect. Its three
|
|
556
|
+
delivery regexes now carry an optional `(?:[^/]+\/)?` segment so `delivery/<slug>/trust.*`
|
|
557
|
+
is equally blocked — otherwise moving the write path would have opened a hand-forgery hole
|
|
558
|
+
one directory down. Regression-locked by `test_gate_lockdown.sh` AC1.26b/c/d. The
|
|
559
|
+
`delivery/*` gitignore already covers per-session dirs (they are force-added deliberately per
|
|
560
|
+
delivery exactly like the flat path).
|
|
561
|
+
|
|
562
|
+
**Residuals (honest):** (1) the DIRTY→no-CI platform behavior for non-per-session conflicts
|
|
563
|
+
is documented, not eliminated (above). (2) The per-session dir name is derived from the
|
|
564
|
+
local session slug; a colliding slug across two sessions would re-share a path — acceptable
|
|
565
|
+
because ownership is still decided by commit ancestry (a stale same-named sibling is ignored,
|
|
566
|
+
not trusted), and slugs are session-unique in practice. (3) Selection correctness does NOT
|
|
567
|
+
depend on merge strategy: prefer-newest resolves the owning candidate for both squash-merge
|
|
568
|
+
(inherited seals are non-ancestors, trivially not-owning) and merge-commit (inherited seals
|
|
569
|
+
can be ancestors, but attest an OLDER commit than this session's, so they lose on recency)
|
|
570
|
+
histories — the merge-commit case was the concrete defect that forced prefer-newest and is
|
|
571
|
+
regression-locked by `test_trust_reconcile_negatives.sh` §8d against a real git repo. (4) The
|
|
572
|
+
flat legacy seals still on `main` are retained by design (see cleanup policy above) and remain
|
|
573
|
+
until a dedicated one-time cleanup PR; they are harmless (prefer-newest) but do accumulate as a
|
|
574
|
+
single fixed path, not a growth vector. This addendum changes WHERE seals live and HOW the
|
|
575
|
+
owning one is chosen; it does not touch Step 1 (fresh verify), the DECLARED exemption path, or
|
|
576
|
+
any fail-closed verdict — all of which are regression-locked unchanged.
|
package/evals/ci/run-baseline.sh
CHANGED
|
@@ -24,6 +24,7 @@ CHECKS=(
|
|
|
24
24
|
"Actor identity resolver integration|bash evals/integration/test_actor_identity.sh"
|
|
25
25
|
"Assignment provider local-file integration|bash evals/integration/test_assignment_provider_local_file.sh"
|
|
26
26
|
"Assignment provider github integration|bash evals/integration/test_assignment_provider_github.sh"
|
|
27
|
+
"Stop hook release-with-handoff integration|bash evals/integration/test_stop_hook_release.sh"
|
|
27
28
|
"Pull work assignment join integration|bash evals/integration/test_pull_work_assignment_join.sh"
|
|
28
29
|
"Ensure-session ownership guard integration|bash evals/integration/test_ensure_session_ownership_guard.sh"
|
|
29
30
|
"Current.json per-actor integration|bash evals/integration/test_current_json_per_actor.sh"
|
|
@@ -79,6 +80,7 @@ LANE_WORKFLOW_CONTRACTS=(
|
|
|
79
80
|
"Actor identity resolver integration"
|
|
80
81
|
"Assignment provider local-file integration"
|
|
81
82
|
"Assignment provider github integration"
|
|
83
|
+
"Stop hook release-with-handoff integration"
|
|
82
84
|
"Pull work assignment join integration"
|
|
83
85
|
"Ensure-session ownership guard integration"
|
|
84
86
|
"Current.json per-actor integration"
|
|
@@ -298,10 +298,11 @@ else
|
|
|
298
298
|
_fail "advance-state --status delivered exited $SEAL_EXIT (signing must not break the seal)"
|
|
299
299
|
fi
|
|
300
300
|
|
|
301
|
-
|
|
302
|
-
|
|
301
|
+
# #379: publishDelivery writes to the per-session path delivery/<slug>/trust.bundle.
|
|
302
|
+
if [[ -f "$REPO_ROOT2/delivery/$SLUG2/trust.bundle" ]]; then
|
|
303
|
+
_pass "publish-delivery published into the explicit scratch --repo-root ($REPO_ROOT2/delivery/$SLUG2/trust.bundle, #379 per-session), not process.cwd()"
|
|
303
304
|
else
|
|
304
|
-
_fail "publish-delivery did not write to the explicit scratch --repo-root ($REPO_ROOT2/delivery/trust.bundle) — check the --repo-root plumbing in advanceState"
|
|
305
|
+
_fail "publish-delivery did not write to the explicit scratch --repo-root ($REPO_ROOT2/delivery/$SLUG2/trust.bundle) — check the --repo-root plumbing in advanceState"
|
|
305
306
|
fi
|
|
306
307
|
|
|
307
308
|
if [[ -f "$SESSION_DIR2/trust.checkpoint.json" ]]; then
|
|
@@ -706,6 +706,42 @@ else
|
|
|
706
706
|
_fail "AC1.26: redirect > delivery/trust.bundle NOT blocked (exit=$prot_exit)"
|
|
707
707
|
fi
|
|
708
708
|
|
|
709
|
+
echo ""
|
|
710
|
+
echo "--- AC1.26b: #379 per-session Write to delivery/<slug>/trust.bundle BLOCKED ---"
|
|
711
|
+
set +e
|
|
712
|
+
prot_out=$(echo '{"tool_name":"Write","tool_input":{"path":"/repo/delivery/i379-delivery-paths/trust.bundle"}}' | node "$PROT" 2>&1)
|
|
713
|
+
prot_exit=$?
|
|
714
|
+
set -e
|
|
715
|
+
if [ "$prot_exit" -eq 2 ] && echo "$prot_out" | grep -q "BLOCKED"; then
|
|
716
|
+
_pass "AC1.26b: Write to delivery/<slug>/trust.bundle blocked (exit 2) — #379 forgery surface moved with the write path"
|
|
717
|
+
else
|
|
718
|
+
_fail "AC1.26b: Write to delivery/<slug>/trust.bundle NOT blocked (exit=$prot_exit, out=$prot_out)"
|
|
719
|
+
fi
|
|
720
|
+
|
|
721
|
+
echo ""
|
|
722
|
+
echo "--- AC1.26c: #379 per-session cp to delivery/<slug>/trust.checkpoint.json BLOCKED ---"
|
|
723
|
+
set +e
|
|
724
|
+
prot_out=$(echo '{"tool_name":"Bash","tool_input":{"command":"cp forged.json delivery/some-session/trust.checkpoint.json"}}' | node "$PROT" 2>&1)
|
|
725
|
+
prot_exit=$?
|
|
726
|
+
set -e
|
|
727
|
+
if [ "$prot_exit" -eq 2 ] && echo "$prot_out" | grep -q "BLOCKED"; then
|
|
728
|
+
_pass "AC1.26c: cp to delivery/<slug>/trust.checkpoint.json blocked (exit 2)"
|
|
729
|
+
else
|
|
730
|
+
_fail "AC1.26c: cp to delivery/<slug>/trust.checkpoint.json NOT blocked (exit=$prot_exit, out=$prot_out)"
|
|
731
|
+
fi
|
|
732
|
+
|
|
733
|
+
echo ""
|
|
734
|
+
echo "--- AC1.26d: #379 per-session redirect > delivery/<slug>/trust.bundle BLOCKED ---"
|
|
735
|
+
set +e
|
|
736
|
+
prot_out=$(echo '{"tool_name":"Bash","tool_input":{"command":"echo {} > delivery/some-session/trust.bundle"}}' | node "$PROT" 2>&1)
|
|
737
|
+
prot_exit=$?
|
|
738
|
+
set -e
|
|
739
|
+
if [ "$prot_exit" -eq 2 ] && echo "$prot_out" | grep -q "BLOCKED"; then
|
|
740
|
+
_pass "AC1.26d: redirect > delivery/<slug>/trust.bundle blocked (exit 2)"
|
|
741
|
+
else
|
|
742
|
+
_fail "AC1.26d: redirect > delivery/<slug>/trust.bundle NOT blocked (exit=$prot_exit)"
|
|
743
|
+
fi
|
|
744
|
+
|
|
709
745
|
echo ""
|
|
710
746
|
echo "--- AC1.27: cp x src/foo.ts ALLOWED (no over-block on normal copy) ---"
|
|
711
747
|
set +e
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# test_model_routing_escalation.sh — #376
|
|
3
|
+
# Deliver-loop eval: a cheap-tier delegate's output FAILS a gate, the fix is
|
|
4
|
+
# re-dispatched one tier HIGHER (escalation), and the escalation + every
|
|
5
|
+
# per-delegation role/model decision are recorded on the session artifact in a
|
|
6
|
+
# shape a downstream economics record (#349) can consume.
|
|
7
|
+
#
|
|
8
|
+
# R2/AC2: fail -> escalate -> pass, escalation recorded with the tier it climbed FROM.
|
|
9
|
+
# R3/AC3: a verify/review delegation's role tier is >= the worker tier it checks.
|
|
10
|
+
# R4/AC4: per-delegation role/model land on agents/<id>/events.jsonl, additive.
|
|
11
|
+
set -uo pipefail
|
|
12
|
+
|
|
13
|
+
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
14
|
+
source "$ROOT/evals/lib/node.sh"
|
|
15
|
+
|
|
16
|
+
WRITER="workflow-sidecar"
|
|
17
|
+
VALIDATOR="validate-workflow-artifacts"
|
|
18
|
+
TMPDIR_EVAL="$(mktemp -d)"
|
|
19
|
+
errors=0
|
|
20
|
+
cleanup() { rm -rf "$TMPDIR_EVAL"; }
|
|
21
|
+
trap cleanup EXIT
|
|
22
|
+
|
|
23
|
+
_pass() { echo " ✓ $1"; }
|
|
24
|
+
_fail() { echo " ✗ $1"; errors=$((errors + 1)); }
|
|
25
|
+
|
|
26
|
+
AR="$TMPDIR_EVAL/repo/.kontourai/flow-agents"
|
|
27
|
+
SLUG="routing-escalation"
|
|
28
|
+
SDIR="$AR/$SLUG"
|
|
29
|
+
mkdir -p "$AR"
|
|
30
|
+
|
|
31
|
+
ev() { flow_agents_node "$WRITER" record-agent-event --artifact-root "$AR" "$@"; }
|
|
32
|
+
|
|
33
|
+
# ── session ───────────────────────────────────────────────────────────────────
|
|
34
|
+
if flow_agents_node "$WRITER" ensure-session \
|
|
35
|
+
--artifact-root "$AR" --task-slug "$SLUG" \
|
|
36
|
+
--title "Routing escalation" \
|
|
37
|
+
--summary "Cheap tier fails a gate; fix escalates one tier higher." \
|
|
38
|
+
--criterion "Escalation recorded" \
|
|
39
|
+
--timestamp "2026-07-04T00:00:00Z" >"$TMPDIR_EVAL/ensure.out" 2>&1; then
|
|
40
|
+
_pass "ensure-session created the deliver-loop session"
|
|
41
|
+
else
|
|
42
|
+
_fail "ensure-session failed: $(cat "$TMPDIR_EVAL/ensure.out")"
|
|
43
|
+
fi
|
|
44
|
+
|
|
45
|
+
# ── deliver loop: delegate cheap -> gate FAIL -> escalate one tier up -> PASS ──
|
|
46
|
+
# 1. Dispatch a fully-specified mechanical slice at the cheap tier (R4 record).
|
|
47
|
+
ev --agent-id tool-worker-1 --kind delegation --status active \
|
|
48
|
+
--role delegate-mechanical --model "claude-haiku-4-5@anthropic" \
|
|
49
|
+
--summary "mechanical slice: routine config edit" \
|
|
50
|
+
--timestamp "2026-07-04T00:00:01Z" >/dev/null 2>&1 \
|
|
51
|
+
&& _pass "recorded mechanical delegation (delegate-mechanical)" \
|
|
52
|
+
|| _fail "failed to record mechanical delegation"
|
|
53
|
+
|
|
54
|
+
# 2. Verify gate FAILS on that cheap delegate's output.
|
|
55
|
+
ev --agent-id tool-verifier --kind evidence --status fail \
|
|
56
|
+
--role delegate-implementation --model "claude-sonnet-5@anthropic" \
|
|
57
|
+
--summary "verify gate FAILED on tool-worker-1 output" \
|
|
58
|
+
--timestamp "2026-07-04T00:00:02Z" >/dev/null 2>&1 \
|
|
59
|
+
&& _pass "recorded verify gate FAILURE on cheap-tier output" \
|
|
60
|
+
|| _fail "failed to record gate failure"
|
|
61
|
+
|
|
62
|
+
# 3. Escalate: re-dispatch the FIX one tier higher, recording the tier climbed FROM.
|
|
63
|
+
ev --agent-id tool-worker-2 --kind escalation --status active \
|
|
64
|
+
--role delegate-implementation --model "claude-sonnet-5@anthropic" \
|
|
65
|
+
--escalated-from delegate-mechanical \
|
|
66
|
+
--summary "fix re-dispatched one tier higher after gate failure" \
|
|
67
|
+
--timestamp "2026-07-04T00:00:03Z" >/dev/null 2>&1 \
|
|
68
|
+
&& _pass "recorded escalation (delegate-mechanical -> delegate-implementation)" \
|
|
69
|
+
|| _fail "failed to record escalation"
|
|
70
|
+
|
|
71
|
+
# 4. Re-verify PASSES on the escalated fix. Goodhart guard: the verifier tier
|
|
72
|
+
# (delegate-implementation) is >= the worker tier it checks (delegate-implementation).
|
|
73
|
+
ev --agent-id tool-verifier --kind evidence --status pass \
|
|
74
|
+
--role delegate-implementation --model "claude-sonnet-5@anthropic" \
|
|
75
|
+
--summary "verify gate PASSED on escalated fix" \
|
|
76
|
+
--timestamp "2026-07-04T00:00:04Z" >/dev/null 2>&1 \
|
|
77
|
+
&& _pass "recorded verify PASS after escalation" \
|
|
78
|
+
|| _fail "failed to record verify pass"
|
|
79
|
+
|
|
80
|
+
# ── assertions over the recorded session artifact ─────────────────────────────
|
|
81
|
+
node - "$SDIR" <<'NODE'
|
|
82
|
+
const fs = require("fs");
|
|
83
|
+
const path = require("path");
|
|
84
|
+
const sdir = process.argv[2];
|
|
85
|
+
const TIER = { "delegate-mechanical": 1, "delegate-implementation": 2, "delegate-design": 3, "orchestrator": 4 };
|
|
86
|
+
|
|
87
|
+
function readEvents(agent) {
|
|
88
|
+
const f = path.join(sdir, "agents", agent, "events.jsonl");
|
|
89
|
+
if (!fs.existsSync(f)) return [];
|
|
90
|
+
return fs.readFileSync(f, "utf8").trim().split("\n").filter(Boolean).map((l) => JSON.parse(l));
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
let bad = 0;
|
|
94
|
+
const say = (ok, msg) => { console.log(` ${ok ? "✓" : "✗"} ${msg}`); if (!ok) bad++; };
|
|
95
|
+
|
|
96
|
+
// R4/AC4: every delegation/escalation event carries a consumable role + model.
|
|
97
|
+
const w1 = readEvents("tool-worker-1");
|
|
98
|
+
const del = w1.find((e) => e.kind === "delegation");
|
|
99
|
+
say(del && del.role === "delegate-mechanical" && del.model === "claude-haiku-4-5@anthropic",
|
|
100
|
+
"R4: mechanical delegation event carries role + model (economics-consumable)");
|
|
101
|
+
|
|
102
|
+
// R4 additive/back-compat: a plain event carries no routing keys.
|
|
103
|
+
const plainRoot = path.join(sdir, "state.json");
|
|
104
|
+
say(fs.existsSync(plainRoot), "session state.json exists (additive change did not break session shape)");
|
|
105
|
+
|
|
106
|
+
// R2/AC2: escalation event recorded, one tier higher, with the tier climbed FROM.
|
|
107
|
+
const esc = readEvents("tool-worker-2").find((e) => e.kind === "escalation");
|
|
108
|
+
say(!!esc, "R2: escalation event recorded");
|
|
109
|
+
say(esc && esc.escalated_from === "delegate-mechanical", "R2: escalation records escalated_from = delegate-mechanical");
|
|
110
|
+
say(esc && TIER[esc.role] === TIER[esc.escalated_from] + 1,
|
|
111
|
+
"R2: escalation climbed exactly one tier up the ladder (mechanical -> implementation)");
|
|
112
|
+
|
|
113
|
+
// R3/AC3: the verifier tier is >= the worker tier it checks (Goodhart guard).
|
|
114
|
+
const verifs = readEvents("tool-verifier").filter((e) => e.role);
|
|
115
|
+
const workerTier = TIER["delegate-implementation"]; // tier of the escalated fix under verification
|
|
116
|
+
const allGuarded = verifs.every((v) => TIER[v.role] >= workerTier);
|
|
117
|
+
say(verifs.length >= 1 && allGuarded, "R3: verify role tier >= worker tier (never a cheaper checker)");
|
|
118
|
+
|
|
119
|
+
// AC4: demonstrate the #349-consumable reduction — price per role across the run.
|
|
120
|
+
const allEvents = [];
|
|
121
|
+
for (const agent of fs.readdirSync(path.join(sdir, "agents"))) {
|
|
122
|
+
allEvents.push(...readEvents(agent));
|
|
123
|
+
}
|
|
124
|
+
const byRole = {};
|
|
125
|
+
for (const e of allEvents) if (e.role) byRole[e.role] = (byRole[e.role] || 0) + 1;
|
|
126
|
+
say(Object.keys(byRole).length >= 2 && byRole["delegate-mechanical"] >= 1 && byRole["delegate-implementation"] >= 1,
|
|
127
|
+
`AC4: routing records reduce to a per-role economics shape: ${JSON.stringify(byRole)}`);
|
|
128
|
+
|
|
129
|
+
process.exit(bad === 0 ? 0 : 1);
|
|
130
|
+
NODE
|
|
131
|
+
if [[ $? -eq 0 ]]; then _pass "session artifact routing/escalation/Goodhart assertions passed"; else _fail "session artifact assertions failed"; fi
|
|
132
|
+
|
|
133
|
+
# ── additive shape must still validate ────────────────────────────────────────
|
|
134
|
+
if flow_agents_node "$VALIDATOR" "$SDIR" >"$TMPDIR_EVAL/validate.out" 2>&1; then
|
|
135
|
+
_pass "validate-workflow-artifacts accepts the additive routing fields"
|
|
136
|
+
else
|
|
137
|
+
_fail "validate-workflow-artifacts rejected routing fields: $(cat "$TMPDIR_EVAL/validate.out")"
|
|
138
|
+
fi
|
|
139
|
+
|
|
140
|
+
if [[ "$errors" -eq 0 ]]; then
|
|
141
|
+
echo "Model routing escalation (deliver-loop) integration passed."
|
|
142
|
+
exit 0
|
|
143
|
+
fi
|
|
144
|
+
echo "Model routing escalation integration failed: $errors issue(s)."
|
|
145
|
+
exit 1
|