arkaos 4.41.0 → 4.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THE-ARKAOS-GUIDE.md +1 -1
- package/VERSION +1 -1
- package/arka/skills/checkpoint/SKILL.md +10 -3
- package/arka/skills/flow/SKILL.md +4 -0
- package/config/claude-agents/eduardo-copy.md +13 -2
- package/config/claude-agents/francisca-tech.md +12 -2
- package/config/claude-agents/marta-cqo.md +68 -10
- package/config/constitution.yaml +4 -1
- package/core/evals/record_cli.py +184 -24
- package/core/evals/runner_cli.py +5 -2
- package/core/evals/verdict_labels.py +19 -6
- package/core/governance/aggregate_guard.py +672 -0
- package/core/governance/qg_verdict.py +56 -2
- package/core/governance/redo_counter.py +13 -8
- package/core/governance/reviewer_ledger.py +94 -12
- package/core/hooks/session_end.py +16 -0
- package/departments/quality/SKILL.md +38 -18
- package/harness/codex/AGENTS.md +1 -1
- package/harness/copilot/copilot-instructions.md +1 -1
- package/harness/cursor/rules/arkaos.mdc +2 -2
- package/harness/gemini/GEMINI.md +1 -1
- package/harness/opencode/AGENTS.md +1 -1
- package/harness/opencode/agents/arka-architect-gabriel.md +1 -1
- package/harness/opencode/agents/arka-brand-director-valentina.md +1 -1
- package/harness/opencode/agents/arka-cfo-helena.md +1 -1
- package/harness/opencode/agents/arka-chief-of-staff-afonso.md +1 -1
- package/harness/opencode/agents/arka-community-strategist-beatriz.md +1 -1
- package/harness/opencode/agents/arka-content-strategist-rafael.md +1 -1
- package/harness/opencode/agents/arka-conversion-strategist-ines.md +1 -1
- package/harness/opencode/agents/arka-coo-sofia.md +1 -1
- package/harness/opencode/agents/arka-copy-director-eduardo.md +1 -1
- package/harness/opencode/agents/arka-cqo-marta.md +1 -1
- package/harness/opencode/agents/arka-cto-marco.md +1 -1
- package/harness/opencode/agents/arka-design-ops-lead-iris.md +1 -1
- package/harness/opencode/agents/arka-ecom-director-ricardo.md +1 -1
- package/harness/opencode/agents/arka-knowledge-director-clara.md +1 -1
- package/harness/opencode/agents/arka-leadership-director-rodrigo.md +1 -1
- package/harness/opencode/agents/arka-marketing-director-luna.md +1 -1
- package/harness/opencode/agents/arka-ops-lead-daniel.md +1 -1
- package/harness/opencode/agents/arka-pm-director-carolina.md +1 -1
- package/harness/opencode/agents/arka-revops-lead-vicente.md +1 -1
- package/harness/opencode/agents/arka-saas-strategist-tiago.md +1 -1
- package/harness/opencode/agents/arka-sales-director-miguel.md +1 -1
- package/harness/opencode/agents/arka-strategy-director-tomas.md +1 -1
- package/harness/opencode/agents/arka-tech-director-francisca.md +1 -1
- package/harness/opencode/agents/arka-tech-lead-paulo.md +1 -1
- package/harness/opencode/agents/arka-video-producer-simao.md +1 -1
- package/harness/zed/.rules +1 -1
- package/knowledge/skills-manifest.json +1 -1
- package/package.json +1 -1
- package/pyproject.toml +1 -1
package/THE-ARKAOS-GUIDE.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# The ArkaOS Guide
|
|
2
2
|
|
|
3
|
-
> v4.
|
|
3
|
+
> v4.43.0 — 89 agents, 17 departments, 332 skills, 297 commands, 18 ADRs.
|
|
4
4
|
> One file, everything you need to start. Generated by `scripts/guide_gen.py` — never hand-edited.
|
|
5
5
|
|
|
6
6
|
## What it is
|
package/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
4.
|
|
1
|
+
4.43.0
|
|
@@ -53,7 +53,12 @@ Any work item whose execution would block the user for **> 30 seconds**
|
|
|
53
53
|
Per the brainstorm (Marta + Tomas):
|
|
54
54
|
|
|
55
55
|
- **Technical conflicts** between reviewers (e.g. Eduardo + Francisca
|
|
56
|
-
disagree on a code-style call) → Marta resolves
|
|
56
|
+
disagree on a code-style call) → Marta resolves ON THE RECORD:
|
|
57
|
+
evidence decides (reproduce the claim, cite the reproduction), and
|
|
58
|
+
the resolution lands in her verdict notes. A blocker backed by
|
|
59
|
+
evidence is fixed or REFUTED with a substantive reason — never
|
|
60
|
+
dropped silently (constitution `inter-agent-checkpoints` enforcement
|
|
61
|
+
+ `evidence-flow`; the aggregate guard enforces the bar).
|
|
57
62
|
- **Strategic / taste / business-knowledge conflicts** → Marta surfaces
|
|
58
63
|
the disagreement at the next checkpoint: *"Eduardo + Francisca
|
|
59
64
|
discordam em X, eu inclino para A — qual o teu input?"*. The user
|
|
@@ -74,8 +79,10 @@ Tens contexto a acrescentar antes de eu arrancar? (Silêncio = procedo.)
|
|
|
74
79
|
follows by emitting checkpoint markers between agent calls.
|
|
75
80
|
- Does not block the user. The user is free to stay silent (proceed)
|
|
76
81
|
or to send any message (parsed by `parse_user_injection`).
|
|
77
|
-
- Does not bypass the mandatory
|
|
78
|
-
|
|
82
|
+
- Does not bypass the mandatory 4-gate evidence flow (the 13-phase
|
|
83
|
+
flow it referenced was replaced in v4.1.0 — constitution
|
|
84
|
+
`evidence-flow`). Checkpoints are emitted **inside** Gate 3
|
|
85
|
+
execution, between sub-dispatches; Gate 4 still runs.
|
|
79
86
|
|
|
80
87
|
## Cross-references
|
|
81
88
|
|
|
@@ -142,6 +142,10 @@ resumes at the right gate.
|
|
|
142
142
|
Time and token cost are not acceptable answers to any of the three.
|
|
143
143
|
- Quality Gate REJECTED loops back at most twice; a third REJECTED
|
|
144
144
|
escalates to the operator with the full verdict.
|
|
145
|
+
- The gate-closing report reproduces each reviewer verdict VERBATIM
|
|
146
|
+
under `### <Reviewer> — verbatim`, with the ledger artifact path
|
|
147
|
+
(`~/.arkaos/quality-gate/<session>/`) beside it. Summarising a
|
|
148
|
+
reviewer in your own words is relay, not report.
|
|
145
149
|
- Close with an honest summary: what changed, where, how it was
|
|
146
150
|
verified (real commands + results), what remains open.
|
|
147
151
|
|
|
@@ -55,10 +55,21 @@ DOCUMENTATION-scope prose it is advisory.
|
|
|
55
55
|
Return a `QGVerdict` JSON object (schema: `QG_VERDICT_JSON_SCHEMA` in
|
|
56
56
|
`core.governance.qg_verdict`): `verdict`, `evidence_report` summary,
|
|
57
57
|
`blockers` [{check, detail, file, verdict}] with exact location and
|
|
58
|
-
correction — `
|
|
58
|
+
correction — `check` names the evidence check or rubric area (the
|
|
59
|
+
aggregate's coverage matching keys on it; never leave it empty),
|
|
60
|
+
`verdict` is claim-level: CONFIRMED (you verified the
|
|
59
61
|
error on the page/line), PLAUSIBLE (credible, unverified), REFUTED
|
|
60
62
|
(disproven; recorded, never counts toward rejection) —,
|
|
61
|
-
`reviewer: "copy-director-eduardo"`, `model_used`, `
|
|
63
|
+
`reviewer: "copy-director-eduardo"`, `model_used`, `evidence_digest`
|
|
64
|
+
(the `report_digest` of the report you interpreted — mandatory since
|
|
65
|
+
PR-B4; an artifact without it cannot support an APPROVED aggregate),
|
|
66
|
+
`notes`.
|
|
67
|
+
|
|
68
|
+
Emit the JSON inside a ```arka-qgverdict fence in your FINAL message —
|
|
69
|
+
the fence is what the hook-boundary ledger captures verbatim. Never
|
|
70
|
+
write triple backticks inside a JSON string — one inside notes cut the
|
|
71
|
+
extractor mid-string (francisca-tech-17); the balanced-JSON cut now
|
|
72
|
+
recovers most such cases, and none of them is worth relying on.
|
|
62
73
|
|
|
63
74
|
Model tier: single source is constitution `quality_gate.model_policy` —
|
|
64
75
|
Quality Gate reviewers run on the best model available (frontier tier,
|
|
@@ -54,8 +54,18 @@ you did not attempt to reproduce is PLAUSIBLE at best, never CONFIRMED.
|
|
|
54
54
|
Return a `QGVerdict` JSON object (schema: `QG_VERDICT_JSON_SCHEMA` in
|
|
55
55
|
`core.governance.qg_verdict`): `verdict`, `evidence_report` summary,
|
|
56
56
|
`blockers` [{check, detail, file, verdict}] numbered B1./B2. with line
|
|
57
|
-
references and fix suggestions
|
|
58
|
-
|
|
57
|
+
references and fix suggestions — `check` names the evidence check or
|
|
58
|
+
rubric area (the aggregate's coverage matching keys on it; never
|
|
59
|
+
leave it empty) —, `reviewer: "tech-director-francisca"`,
|
|
60
|
+
`model_used`, `evidence_digest` (the `report_digest` of the report
|
|
61
|
+
you interpreted — mandatory since PR-B4; an artifact without it
|
|
62
|
+
cannot support an APPROVED aggregate), `notes`.
|
|
63
|
+
|
|
64
|
+
Emit the JSON inside a ```arka-qgverdict fence in your FINAL message —
|
|
65
|
+
the fence is what the hook-boundary ledger captures verbatim. Never
|
|
66
|
+
write triple backticks inside a JSON string — one inside notes cut the
|
|
67
|
+
extractor mid-string (francisca-tech-17); the balanced-JSON cut now
|
|
68
|
+
recovers most such cases, and none of them is worth relying on.
|
|
59
69
|
|
|
60
70
|
Model tier: single source is constitution `quality_gate.model_policy` —
|
|
61
71
|
Quality Gate reviewers run on the best model available (frontier tier,
|
|
@@ -26,9 +26,20 @@ evidence report, never from model size.
|
|
|
26
26
|
1. Run the engine first — no verdict without a report:
|
|
27
27
|
`~/.arkaos/bin/arka-py -m core.governance.evidence_checks <project_dir> [--changed-files ...] [--test-command '...'] --json`
|
|
28
28
|
2. Dispatch Eduardo (spellcheck + changed copy) and Francisca
|
|
29
|
-
(lint/typecheck/tests/coverage/security-grep) with the report and
|
|
30
|
-
|
|
31
|
-
`core.governance.qg_verdict
|
|
29
|
+
(lint/typecheck/tests/coverage/security-grep) with the report and,
|
|
30
|
+
in the prompt, the QGVerdict field names (`QG_VERDICT_JSON_SCHEMA`
|
|
31
|
+
in `core.governance.qg_verdict` is that contract — the Agent tool
|
|
32
|
+
has no structured-output parameter). Dispatch shape (PR-B4): the
|
|
33
|
+
prompt names the FULL field set the reviewer returns — `verdict`,
|
|
34
|
+
`evidence_report` {overall, checks_ran, checks_failed,
|
|
35
|
+
checks_skipped}, `blockers` [{`check` (the evidence check name;
|
|
36
|
+
coverage matching keys on it), `detail`, `file`, `verdict`
|
|
37
|
+
CONFIRMED/PLAUSIBLE/REFUTED}], `reviewer`, `model_used`,
|
|
38
|
+
`evidence_digest` (= the report's `report_digest`), `notes`. A
|
|
39
|
+
dispatch that invents its own field names fail-softs the artifact
|
|
40
|
+
(16 schema errors on one B2 round); a reviewer artifact without
|
|
41
|
+
`evidence_digest` cannot support an APPROVED aggregate — the guard
|
|
42
|
+
refuses it and the reviewer must be re-dispatched.
|
|
32
43
|
3. Aggregate at CLAIM level (Constitution 2.0): every reviewer blocker
|
|
33
44
|
carries `verdict` CONFIRMED / PLAUSIBLE / REFUTED. Only CONFIRMED and
|
|
34
45
|
PLAUSIBLE blockers count toward rejection; REFUTED are recorded for
|
|
@@ -39,15 +50,25 @@ evidence report, never from model size.
|
|
|
39
50
|
- overall == "pass" → APPROVED only if zero CONFIRMED/PLAUSIBLE blockers.
|
|
40
51
|
- overall == "insufficient-evidence" → APPROVED only with explicit
|
|
41
52
|
justification in notes; otherwise REJECTED.
|
|
42
|
-
4. Record the
|
|
43
|
-
`evidence_overall` — it raises on APPROVED-over-fail by design.
|
|
44
|
-
5. Record the eval label (evals ADR 2026-07-09) as your FINAL act — the
|
|
53
|
+
4. Record the eval label (evals ADR 2026-07-09) as your FINAL act — the
|
|
45
54
|
corpus only grows if the verdict-issuer writes it, and dispatch
|
|
46
55
|
through this agent bypasses the department SKILL's step 6: write your
|
|
47
56
|
final QGVerdict JSON to a temp file and run
|
|
48
57
|
`~/.arkaos/bin/arka-py -m core.evals.record_cli --file <f> --kind qg
|
|
49
|
-
--department <dept> --deliverable "<title>"`.
|
|
50
|
-
|
|
58
|
+
--session-id <session> --department <dept> --deliverable "<title>"`.
|
|
59
|
+
It fails LOUDLY for three distinct reasons, each with its own
|
|
60
|
+
remedy: invalid JSON (fix the JSON and re-run), a missing
|
|
61
|
+
--session-id (pass the session id — the anti-self-approval guard
|
|
62
|
+
reads that session's reviewer ledger), or a guard refusal (the
|
|
63
|
+
ledger cannot support your aggregate: quorum, a missing or
|
|
64
|
+
mismatched `evidence_digest` without a justified `digest_carries`
|
|
65
|
+
entry, a session already stamped as ended (digest and session
|
|
66
|
+
reasons refuse only an APPROVED aggregate — a REJECTED one records
|
|
67
|
+
with warnings), blocker coverage, or an
|
|
68
|
+
APPROVED verdict standing over a rejecting reviewer — read the
|
|
69
|
+
stderr reasons and fix the REVIEW, not the JSON; if the reason
|
|
70
|
+
names AGGREGATE.json or the session id, fix that instead). Never
|
|
71
|
+
skip.
|
|
51
72
|
Every review feeds `~/.arkaos/telemetry/qg-verdicts.jsonl`, redo
|
|
52
73
|
verdicts included (a REJECTED→APPROVED pair is two labels).
|
|
53
74
|
|
|
@@ -56,11 +77,25 @@ evidence report, never from model size.
|
|
|
56
77
|
Return a `QGVerdict` JSON object: `verdict` (APPROVED|REJECTED),
|
|
57
78
|
`evidence_report` {overall, checks_ran, checks_failed, checks_skipped},
|
|
58
79
|
`blockers` [{check, detail, file, verdict}], `reviewer: "cqo-marta"`,
|
|
59
|
-
`model_used`, `notes
|
|
80
|
+
`model_used`, `notes`, `evidence_digest` (the `report_digest` of the
|
|
81
|
+
report you aggregated — mandatory since PR-B4) and, when you carry an
|
|
82
|
+
earlier review over a report change, `digest_carries`
|
|
83
|
+
[{reviewer, evidence_digest, reason}] naming the digest THAT reviewer
|
|
84
|
+
actually reviewed and why the review still stands (>= 40 chars).
|
|
85
|
+
Binary — there is no "approved with caveats".
|
|
86
|
+
|
|
87
|
+
Emit the final JSON inside a ```arka-qgverdict fence in your FINAL
|
|
88
|
+
message — the fence is what the hook-boundary ledger captures, and an
|
|
89
|
+
aggregate that exists only as prose is a relay (the B1 gate closed
|
|
90
|
+
with this fence present by ad-hoc instruction; it is contract now).
|
|
91
|
+
Never write triple backticks inside a JSON string — one inside notes
|
|
92
|
+
cut the extractor mid-string (francisca-tech-17); the balanced-JSON
|
|
93
|
+
cut now recovers most such cases, and none of them is worth relying
|
|
94
|
+
on.
|
|
60
95
|
|
|
61
96
|
Filled example (the shape you return, not a schema):
|
|
62
97
|
|
|
63
|
-
```
|
|
98
|
+
```arka-qgverdict
|
|
64
99
|
{"verdict": "REJECTED",
|
|
65
100
|
"evidence_report": {"overall": "pass", "checks_ran": ["lint","tests"],
|
|
66
101
|
"checks_failed": [], "checks_skipped": ["coverage"]},
|
|
@@ -69,9 +104,32 @@ Filled example (the shape you return, not a schema):
|
|
|
69
104
|
"detail": "AttributeError on malformed record — docstring claims 'never raises'; reproduced via check_x('bad')",
|
|
70
105
|
"file": "core/governance/x.py:138", "verdict": "CONFIRMED"}],
|
|
71
106
|
"reviewer": "cqo-marta", "model_used": "opus",
|
|
107
|
+
"evidence_digest": "3f2a3f2a3f2a3f2a3f2a3f2a3f2a3f2a3f2a3f2a3f2a3f2a3f2a3f2a3f2a3f2a",
|
|
72
108
|
"notes": "Engine pass but 1 CONFIRMED blocker, reproduced by my own hand."}
|
|
73
109
|
```
|
|
74
110
|
|
|
111
|
+
## Reporting (verbatim, never relay)
|
|
112
|
+
|
|
113
|
+
The gate-closing report reproduces every reviewer verdict VERBATIM
|
|
114
|
+
under `### <Reviewer> — verbatim`, each with its ledger artifact path
|
|
115
|
+
beside it. Summarising a reviewer in your own words is relay, not
|
|
116
|
+
report — a relay inside a gate is a single point of distortion, and
|
|
117
|
+
it is how a corpus reached 80 aggregator-authored records with zero
|
|
118
|
+
reviewer-signed ones.
|
|
119
|
+
|
|
120
|
+
## Conflict Handling (no silent resolution)
|
|
121
|
+
|
|
122
|
+
A reviewer blocker BACKED BY EVIDENCE is never resolved silently: it
|
|
123
|
+
is fixed (and the fix verified by execution), or REFUTED on the
|
|
124
|
+
record with a substantive reason (>= 40 chars — the guard enforces
|
|
125
|
+
the bar), or it blocks. Disagreement between reviewers is settled by
|
|
126
|
+
evidence, not by rank: reproduce the claim, cite the reproduction.
|
|
127
|
+
Only a blocker with no evidence behind it (no repro, no citation) may
|
|
128
|
+
be dropped, and even that drop is recorded in `notes`, never omitted.
|
|
129
|
+
A CONFIRMED blocker is never merely noted: the guard reads
|
|
130
|
+
`blockers`, not `notes` — it is fixed, or REFUTED in `blockers` with
|
|
131
|
+
its reason, or it blocks.
|
|
132
|
+
|
|
75
133
|
## Signature Rules (anti-sycophancy)
|
|
76
134
|
|
|
77
135
|
- Open with "Quality Gate Verdict:" and close with "Final:".
|
package/config/constitution.yaml
CHANGED
|
@@ -248,7 +248,7 @@ enforcement_levels:
|
|
|
248
248
|
|
|
249
249
|
- id: inter-agent-checkpoints
|
|
250
250
|
rule: "Long-running multi-agent dispatches (estimated >30s) are fragmented into sub-dispatches of 2-3min each. Between sub-dispatches, the orchestrator emits a proactive [arka:checkpoint] prompt announcing the next sub-dispatch and inviting user context injection. The user can inject (continue with addition) or stay silent (proceed)."
|
|
251
|
-
enforcement: "Orchestrator wraps multi-step work in checkpointed dispatches; reviewer conflicts escalated to user when strategic (taste/business knowledge), resolved
|
|
251
|
+
enforcement: "Orchestrator wraps multi-step work in checkpointed dispatches; reviewer conflicts escalated to user when strategic (taste/business knowledge), resolved on the record when purely technical (evidence cited in the verdict notes — never silently; the earlier self-contradiction was resolved 2026-07-30 in favour of visibility); QG deliberations are visible, not just verdict."
|
|
252
252
|
|
|
253
253
|
- id: hybrid-learning
|
|
254
254
|
rule: "ArkaOS learns from user corrections via hybrid mechanism: implicit auto-detection with confidence scoring for typical corrections (default), explicit Marta-led confirmation for high-leverage rules (top-level rule candidates) or rules that contradict existing memory. Marta is the owner of the learning loop. Memory rules carry a confidence field that climbs as the rule is applied without correction."
|
|
@@ -492,3 +492,6 @@ amendments:
|
|
|
492
492
|
- version: "4.12.0"
|
|
493
493
|
date: "2026-07-09"
|
|
494
494
|
changes: "Interaction Reform PR2 (operator-approved plan, 2026-07-09): added gate-judges MUST rule — plan-judge at Gate 2 and output-judge at Gate 4 for MEDIUM/HIGH work, structured JudgeVerdict (core.governance.judge), frontier tier, arkaos-not-yes-man standard applied to agents, user_challenge surfacing for technically wrong requests; verdicts recorded to judge-verdicts.jsonl."
|
|
495
|
+
- version: "4.43.0"
|
|
496
|
+
date: "2026-07-30"
|
|
497
|
+
changes: "PR-B4 (operator-approved repair-campaign plan): inter-agent-checkpoints enforcement text amended — technical reviewer conflicts change from 'resolved silently' to 'resolved on the record' (evidence cited in verdict notes), resolving the self-contradiction with the same enforcement's 'QG deliberations are visible' clause in favour of visibility. Rule text, level and scope unchanged."
|
package/core/evals/record_cli.py
CHANGED
|
@@ -1,11 +1,35 @@
|
|
|
1
1
|
"""Record a QGVerdict or gate JudgeVerdict as an eval label.
|
|
2
2
|
|
|
3
|
-
Reads the
|
|
4
|
-
|
|
5
|
-
``--kind judge`` → judge-verdicts.jsonl). Invoked by
|
|
6
|
-
right after a
|
|
7
|
-
|
|
8
|
-
|
|
3
|
+
Reads the verdict JSON from stdin (or --file) and appends it to the
|
|
4
|
+
matching label corpus (``--kind qg``/``--kind reviewer`` →
|
|
5
|
+
qg-verdicts.jsonl, ``--kind judge`` → judge-verdicts.jsonl). Invoked by
|
|
6
|
+
the orchestrator right after a verdict lands (see the Quality Gate and
|
|
7
|
+
flow skill instructions), closing the "labels gratuitos" loop from the
|
|
8
|
+
evals ADR.
|
|
9
|
+
|
|
10
|
+
``--kind qg`` is the AGGREGATE path and runs the anti-self-approval
|
|
11
|
+
guard (PR-B3, hardened in PR-B4): ``--session-id`` is mandatory and
|
|
12
|
+
validated, the session's reviewer ledger must hold at least two
|
|
13
|
+
hook-captured reviewer verdicts, the digest chain must hold (the
|
|
14
|
+
aggregate and every counted reviewer artifact carry evidence_digest;
|
|
15
|
+
a reviewer's digest that predates the final report needs an explicit
|
|
16
|
+
``digest_carries`` justification), the session must not be stamped
|
|
17
|
+
ended, no CONFIRMED reviewer blocker may disappear silently, and an
|
|
18
|
+
APPROVED aggregate may not stand over a rejecting reviewer.
|
|
19
|
+
Dispatch-shape refusals are verdict-aware: they refuse an APPROVED
|
|
20
|
+
aggregate and demote to warnings on a REJECTED one, so a rejection
|
|
21
|
+
label survives a bad delta. The label records ONLY once
|
|
22
|
+
AGGREGATE.json is verifiably on disk — an accepted-but-unwritten
|
|
23
|
+
aggregate is a refusal, not a success.
|
|
24
|
+
|
|
25
|
+
``--kind reviewer`` records an individual reviewer's verdict as a
|
|
26
|
+
label and cross-references the ledger by ``verdict_digest`` — it never
|
|
27
|
+
writes ledger records (only the hook capture may; anything the
|
|
28
|
+
orchestrator could write itself would let it fabricate its own
|
|
29
|
+
quorum), and it refuses aggregator identities: the aggregator's
|
|
30
|
+
verdict enters through the guarded ``--kind qg`` path, never as a
|
|
31
|
+
"reviewer" label. For reviewer identities it remains an unguarded
|
|
32
|
+
label path — the cross-reference is provenance, not admission.
|
|
9
33
|
|
|
10
34
|
Unlike the underlying writer (telemetry contract: never raises), this
|
|
11
35
|
explicit CLI fails LOUDLY on invalid verdict JSON — a malformed label
|
|
@@ -18,6 +42,7 @@ import argparse
|
|
|
18
42
|
import json
|
|
19
43
|
import sys
|
|
20
44
|
from pathlib import Path
|
|
45
|
+
from typing import TYPE_CHECKING
|
|
21
46
|
|
|
22
47
|
from pydantic import ValidationError
|
|
23
48
|
|
|
@@ -25,51 +50,186 @@ from core.evals.verdict_labels import record_judge_label, record_verdict_label
|
|
|
25
50
|
from core.governance.judge import JudgeVerdict
|
|
26
51
|
from core.governance.qg_verdict import QGVerdict
|
|
27
52
|
|
|
53
|
+
if TYPE_CHECKING: # the runtime import stays lazy (guard pulls pydantic)
|
|
54
|
+
from core.governance.aggregate_guard import GuardResult
|
|
28
55
|
|
|
29
|
-
|
|
56
|
+
|
|
57
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
30
58
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
31
59
|
parser.add_argument("--file", help="verdict JSON file (default: stdin)")
|
|
32
60
|
parser.add_argument(
|
|
33
61
|
"--kind",
|
|
34
|
-
choices=("qg", "judge"),
|
|
62
|
+
choices=("qg", "reviewer", "judge"),
|
|
35
63
|
default="qg",
|
|
36
|
-
help=
|
|
64
|
+
help=(
|
|
65
|
+
"qg = aggregate QGVerdict, guarded (default); reviewer = "
|
|
66
|
+
"individual reviewer QGVerdict; judge = gate JudgeVerdict"
|
|
67
|
+
),
|
|
37
68
|
)
|
|
38
69
|
parser.add_argument("--deliverable", default="")
|
|
39
70
|
parser.add_argument("--department", default="")
|
|
40
71
|
parser.add_argument("--eval-task-id", default="")
|
|
41
72
|
parser.add_argument("--session-id", default="")
|
|
42
|
-
|
|
73
|
+
parser.add_argument("--round", default="", dest="round_label")
|
|
74
|
+
parser.add_argument("--head", default="")
|
|
75
|
+
return parser
|
|
43
76
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
77
|
+
|
|
78
|
+
def main(argv: list[str] | None = None) -> int:
|
|
79
|
+
args = _build_parser().parse_args(argv)
|
|
80
|
+
raw = _read_raw(args)
|
|
81
|
+
if raw is None:
|
|
82
|
+
return 1
|
|
49
83
|
if args.kind == "judge":
|
|
50
84
|
return _record_judge(raw, args)
|
|
51
|
-
|
|
85
|
+
verdict = _parse_qg(raw)
|
|
86
|
+
if verdict is None:
|
|
87
|
+
return 1
|
|
88
|
+
if args.kind == "reviewer":
|
|
89
|
+
return _record_reviewer(verdict, args)
|
|
90
|
+
return _record_aggregate(verdict, args)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _read_raw(args: argparse.Namespace) -> str | None:
|
|
94
|
+
"""Verdict text, or None after a LOUD error (never a traceback)."""
|
|
95
|
+
if not args.file:
|
|
96
|
+
return sys.stdin.read()
|
|
97
|
+
try:
|
|
98
|
+
return Path(args.file).read_text(encoding="utf-8")
|
|
99
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
100
|
+
print(f"error: cannot read --file — {exc}", file=sys.stderr)
|
|
101
|
+
return None
|
|
52
102
|
|
|
53
103
|
|
|
54
|
-
def
|
|
104
|
+
def _parse_qg(raw: str) -> QGVerdict | None:
|
|
55
105
|
try:
|
|
56
|
-
|
|
106
|
+
return QGVerdict.model_validate(json.loads(raw))
|
|
57
107
|
except (json.JSONDecodeError, ValidationError) as exc:
|
|
58
108
|
print(f"error: invalid QGVerdict JSON — {exc}", file=sys.stderr)
|
|
109
|
+
return None
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _refuse(reasons: list[str]) -> int:
|
|
113
|
+
for reason in reasons:
|
|
114
|
+
print(f"refused: {reason}", file=sys.stderr)
|
|
115
|
+
print(
|
|
116
|
+
"error: aggregate refused — nothing recorded (anti-self-"
|
|
117
|
+
"approval guard, PR-B3)",
|
|
118
|
+
file=sys.stderr,
|
|
119
|
+
)
|
|
120
|
+
return 1
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _drive_redo(verdict: QGVerdict, session_id: str) -> str:
|
|
124
|
+
from core.governance import redo_counter
|
|
125
|
+
|
|
126
|
+
if verdict.verdict == "REJECTED":
|
|
127
|
+
return redo_counter.record_rejected(session_id).to_message()
|
|
128
|
+
redo_counter.reset(session_id)
|
|
129
|
+
return ""
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _record_aggregate(verdict: QGVerdict, args: argparse.Namespace) -> int:
|
|
133
|
+
from core.governance.aggregate_guard import check_aggregate, write_aggregate
|
|
134
|
+
|
|
135
|
+
if not args.session_id:
|
|
136
|
+
print(
|
|
137
|
+
"error: --kind qg records an AGGREGATE and requires "
|
|
138
|
+
"--session-id (the guard reads that session's reviewer ledger)",
|
|
139
|
+
file=sys.stderr,
|
|
140
|
+
)
|
|
141
|
+
return 1
|
|
142
|
+
dumped = verdict.model_dump()
|
|
143
|
+
result = check_aggregate(dumped, args.session_id)
|
|
144
|
+
if not result.ok:
|
|
145
|
+
return _refuse(result.reasons)
|
|
146
|
+
aggregate_path = write_aggregate(args.session_id, dumped, result)
|
|
147
|
+
if aggregate_path is None:
|
|
148
|
+
# An accepted-but-unwritten aggregate recorded anyway would be
|
|
149
|
+
# the incident shape again, one filesystem error away.
|
|
150
|
+
return _refuse([
|
|
151
|
+
"AGGREGATE.json could not be written and verified on disk "
|
|
152
|
+
"— a label without its artifact is the incident shape"
|
|
153
|
+
])
|
|
154
|
+
_record_label(verdict, args)
|
|
155
|
+
print(json.dumps(_accepted_payload(verdict, args, aggregate_path, result)))
|
|
156
|
+
return 0
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _accepted_payload(
|
|
160
|
+
verdict: QGVerdict,
|
|
161
|
+
args: argparse.Namespace,
|
|
162
|
+
aggregate_path: Path,
|
|
163
|
+
result: GuardResult,
|
|
164
|
+
) -> dict:
|
|
165
|
+
return {
|
|
166
|
+
"recorded": True,
|
|
167
|
+
"verdict": verdict.verdict,
|
|
168
|
+
"eval_task_id": args.eval_task_id,
|
|
169
|
+
"aggregate": str(aggregate_path),
|
|
170
|
+
"reviewers": result.reviewers,
|
|
171
|
+
"warnings": result.warnings,
|
|
172
|
+
"redo": _drive_redo(verdict, args.session_id),
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _record_reviewer(verdict: QGVerdict, args: argparse.Namespace) -> int:
|
|
177
|
+
from core.governance.reviewer_ledger import AGGREGATOR_IDS
|
|
178
|
+
|
|
179
|
+
if verdict.reviewer in AGGREGATOR_IDS:
|
|
180
|
+
print(
|
|
181
|
+
f"error: {verdict.reviewer!r} is an aggregator identity — "
|
|
182
|
+
"an aggregator's verdict is recorded via --kind qg, which "
|
|
183
|
+
"is guarded; recording it as a 'reviewer' label would repeat "
|
|
184
|
+
"the 2026-07-20 incident under a different flag",
|
|
185
|
+
file=sys.stderr,
|
|
186
|
+
)
|
|
59
187
|
return 1
|
|
188
|
+
_record_label(verdict, args)
|
|
189
|
+
print(json.dumps({
|
|
190
|
+
"recorded": True,
|
|
191
|
+
"verdict": verdict.verdict,
|
|
192
|
+
"reviewer": verdict.reviewer,
|
|
193
|
+
"ledger_match": _ledger_match(verdict, args.session_id),
|
|
194
|
+
}))
|
|
195
|
+
return 0
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _ledger_match(verdict: QGVerdict, session_id: str) -> str:
|
|
199
|
+
"""Name of the hook-captured record carrying this exact verdict."""
|
|
200
|
+
if not session_id:
|
|
201
|
+
return ""
|
|
202
|
+
try:
|
|
203
|
+
from core.governance.aggregate_guard import _session_records
|
|
204
|
+
from core.governance.qg_digest import verdict_digest
|
|
205
|
+
|
|
206
|
+
target = verdict_digest(verdict.model_dump())
|
|
207
|
+
for name, record in _session_records(session_id):
|
|
208
|
+
their = record.get("verdict")
|
|
209
|
+
if not isinstance(their, dict):
|
|
210
|
+
continue
|
|
211
|
+
try: # digest the NORMALIZED dump — raw dicts differ on defaults
|
|
212
|
+
normalized = QGVerdict.model_validate(their).model_dump()
|
|
213
|
+
except ValidationError:
|
|
214
|
+
continue
|
|
215
|
+
if verdict_digest(normalized) == target:
|
|
216
|
+
return name
|
|
217
|
+
except Exception: # cross-reference is provenance, never a blocker
|
|
218
|
+
return ""
|
|
219
|
+
return ""
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _record_label(verdict: QGVerdict, args: argparse.Namespace) -> None:
|
|
60
223
|
record_verdict_label(
|
|
61
224
|
verdict,
|
|
62
225
|
deliverable=args.deliverable,
|
|
63
226
|
department=args.department,
|
|
64
227
|
eval_task_id=args.eval_task_id,
|
|
65
228
|
session_id=args.session_id,
|
|
229
|
+
kind=args.kind,
|
|
230
|
+
round_label=args.round_label,
|
|
231
|
+
head=args.head,
|
|
66
232
|
)
|
|
67
|
-
print(json.dumps({
|
|
68
|
-
"recorded": True,
|
|
69
|
-
"verdict": verdict.verdict,
|
|
70
|
-
"eval_task_id": args.eval_task_id,
|
|
71
|
-
}))
|
|
72
|
-
return 0
|
|
73
233
|
|
|
74
234
|
|
|
75
235
|
def _record_judge(raw: str, args: argparse.Namespace) -> int:
|
package/core/evals/runner_cli.py
CHANGED
|
@@ -65,8 +65,11 @@ def _dispatch_prompt(task: EvalTask) -> str:
|
|
|
65
65
|
f"O deliverable será julgado pelo Quality Gate contra estas "
|
|
66
66
|
f"propriedades verificáveis:\n{properties}{rubric}\n\n"
|
|
67
67
|
f"Após o verdict, regista o label com:\n"
|
|
68
|
-
f" arka-py -m core.evals.record_cli --
|
|
69
|
-
f"--
|
|
68
|
+
f" arka-py -m core.evals.record_cli --kind qg "
|
|
69
|
+
f"--session-id <session> --eval-task-id {task.id} "
|
|
70
|
+
f"--department {task.department} < verdict.json\n"
|
|
71
|
+
f"(--session-id é obrigatório no caminho do agregado — o guard "
|
|
72
|
+
f"anti-self-approval lê o ledger de reviewers dessa sessão)"
|
|
70
73
|
)
|
|
71
74
|
|
|
72
75
|
|
|
@@ -17,7 +17,7 @@ from __future__ import annotations
|
|
|
17
17
|
|
|
18
18
|
import json
|
|
19
19
|
import os
|
|
20
|
-
from datetime import
|
|
20
|
+
from datetime import UTC, datetime
|
|
21
21
|
from pathlib import Path
|
|
22
22
|
from typing import Any
|
|
23
23
|
|
|
@@ -49,20 +49,33 @@ def record_verdict_label(
|
|
|
49
49
|
department: str = "",
|
|
50
50
|
eval_task_id: str = "",
|
|
51
51
|
session_id: str = "",
|
|
52
|
+
kind: str = "",
|
|
53
|
+
round_label: str = "",
|
|
54
|
+
head: str = "",
|
|
52
55
|
) -> None:
|
|
53
|
-
"""Append one labeled QG example. Never raises (telemetry contract).
|
|
56
|
+
"""Append one labeled QG example. Never raises (telemetry contract).
|
|
57
|
+
|
|
58
|
+
``kind``/``round_label``/``head`` live on the envelope, not the
|
|
59
|
+
schema: QGVerdict drops unknown fields, so without them the corpus
|
|
60
|
+
could not distinguish rounds of one PR (QG r13 register).
|
|
61
|
+
``round_label`` persists under the ``round`` key — the parameter
|
|
62
|
+
avoids shadowing the builtin.
|
|
63
|
+
"""
|
|
54
64
|
try:
|
|
55
65
|
entry: dict[str, Any] = {
|
|
56
|
-
"ts": datetime.now(
|
|
66
|
+
"ts": datetime.now(UTC).isoformat(),
|
|
57
67
|
"deliverable": str(deliverable or ""),
|
|
58
68
|
"department": str(department or ""),
|
|
59
69
|
"eval_task_id": str(eval_task_id or ""),
|
|
60
70
|
"session_id": str(session_id or ""),
|
|
71
|
+
"kind": str(kind or ""),
|
|
72
|
+
"round": str(round_label or ""),
|
|
73
|
+
"head": str(head or ""),
|
|
61
74
|
**verdict.model_dump(),
|
|
62
75
|
}
|
|
63
76
|
with _locked_append(_labels_path()) as fh:
|
|
64
77
|
fh.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
|
65
|
-
except Exception:
|
|
78
|
+
except Exception:
|
|
66
79
|
return
|
|
67
80
|
|
|
68
81
|
|
|
@@ -75,7 +88,7 @@ def record_judge_label(
|
|
|
75
88
|
"""Append one labeled gate-judge example. Never raises."""
|
|
76
89
|
try:
|
|
77
90
|
entry: dict[str, Any] = {
|
|
78
|
-
"ts": datetime.now(
|
|
91
|
+
"ts": datetime.now(UTC).isoformat(),
|
|
79
92
|
"deliverable": str(deliverable or ""),
|
|
80
93
|
"department": str(department or ""),
|
|
81
94
|
"session_id": str(session_id or ""),
|
|
@@ -83,7 +96,7 @@ def record_judge_label(
|
|
|
83
96
|
}
|
|
84
97
|
with _locked_append(_judge_labels_path()) as fh:
|
|
85
98
|
fh.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
|
86
|
-
except Exception:
|
|
99
|
+
except Exception:
|
|
87
100
|
return
|
|
88
101
|
|
|
89
102
|
|