@heretek-ai/epistemic-swarm 0.7.11 → 0.7.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/research-cache/skills/research_cache/hasher.py +56 -14
- package/prompts/agent_brainstormer.md +14 -0
- package/prompts/agent_darkharvest.md +20 -1
- package/runner/__pycache__/__init__.cpython-311.pyc +0 -0
- package/runner/__pycache__/auctioneer.cpython-311.pyc +0 -0
- package/runner/__pycache__/auditor_engine.cpython-311.pyc +0 -0
- package/runner/__pycache__/claim_store.cpython-311.pyc +0 -0
- package/runner/__pycache__/claim_witness.cpython-311.pyc +0 -0
- package/runner/__pycache__/darkharvest_claims.cpython-311.pyc +0 -0
- package/runner/__pycache__/living_dossiers.cpython-311.pyc +0 -0
- package/runner/__pycache__/mcp_protocol.cpython-311.pyc +0 -0
- package/runner/__pycache__/mcp_server.cpython-311.pyc +0 -0
- package/runner/__pycache__/path_safety.cpython-311.pyc +0 -0
- package/runner/__pycache__/pcrb.cpython-311.pyc +0 -0
- package/runner/__pycache__/pcrb_verify.cpython-311.pyc +0 -0
- package/runner/__pycache__/refinement.cpython-311.pyc +0 -0
- package/runner/__pycache__/research_swarm.cpython-311.pyc +0 -0
- package/runner/__pycache__/retrieval_log.cpython-311.pyc +0 -0
- package/runner/__pycache__/state_machine.cpython-311.pyc +0 -0
- package/runner/auditor_engine.py +267 -80
- package/runner/darkharvest_claims.py +326 -0
- package/runner/refinement.py +74 -0
- package/runner/research_swarm.py +119 -24
- package/runner/retrieval_log.py +90 -0
- package/runner/state_machine.py +35 -0
- package/runner/tests/__pycache__/test_auction_order.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_backends.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_bet1_spike.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_claim_store.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_claim_witness.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_claude_plugin.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_darkharvest_audit.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_domain_packs.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_dossier_contract.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_factory.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_fleet_seam.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_grounding.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_living_dossiers.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_manifest_concurrency.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_mcp_server.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_opencode_ux.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_opencode_v2_registrars.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_pcrb.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_refinement.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_swarm.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_swarm_config.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_sweep_regressions.cpython-311.pyc +0 -0
- package/runner/tests/__pycache__/test_webcache.cpython-311.pyc +0 -0
- package/runner/tests/test_darkharvest_audit.py +290 -0
- package/runner/tests/test_grounding.py +183 -0
- package/scripts/__pycache__/bet1_advisory_spike.cpython-311.pyc +0 -0
- package/scripts/__pycache__/build_adapters.cpython-311.pyc +0 -0
- package/scripts/__pycache__/divergence_experiment.cpython-311.pyc +0 -0
- package/skills/epistemic_search/scripts/__pycache__/search.cpython-311.pyc +0 -0
- package/skills/epistemic_search/scripts/search.py +16 -0
- package/skills/epistemic_search/scripts/webcache.py +11 -0
- package/skills/research_cache/__pycache__/__init__.cpython-311.pyc +0 -0
- package/skills/research_cache/__pycache__/hasher.cpython-311.pyc +0 -0
- package/skills/research_cache/hasher.py +56 -14
- package/skills/swarm_config/__pycache__/__init__.cpython-311.pyc +0 -0
- package/skills/swarm_config/__pycache__/configure.cpython-311.pyc +0 -0
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Darkharvest claim normalization, license cross-check, and repo validation.
|
|
3
|
+
|
|
4
|
+
Heretek-AI/IUMBTEMS#5: darkharvest dossiers emit `candidate_repositories[]` with
|
|
5
|
+
their own `source_hash` witnesses, but the auditor only reads
|
|
6
|
+
`affirmative_claims` / `falsification_claims` — so `total_claims_audited` was
|
|
7
|
+
always 0 and the dialectic was structurally inert. Separately, alpha and beta
|
|
8
|
+
can disagree about a repo's license (observed: `MIT` vs `NOASSERTION` for the
|
|
9
|
+
same repo, with alpha recommending `depend-or-vendor`) and nothing cross-checked
|
|
10
|
+
them, because the audit produced no claims to compare.
|
|
11
|
+
|
|
12
|
+
This module supplies the three missing pieces:
|
|
13
|
+
- `normalize_dossier_claims` — fold darkharvest's shape into the standard claim
|
|
14
|
+
schema so one contract flows to the auditor, PCRB, and ledger.
|
|
15
|
+
- `cross_check_repos` — alpha/beta license disagreement is a blocking finding,
|
|
16
|
+
resolved conservatively.
|
|
17
|
+
- `RepoValidator` — ground-truth (existence / license / archived) with fail-open
|
|
18
|
+
offline behavior.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
import re
|
|
24
|
+
import urllib.error
|
|
25
|
+
import urllib.request
|
|
26
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
27
|
+
|
|
28
|
+
# Top-level dossier keys that *look* like audit output but are agent-authored.
|
|
29
|
+
# Renamed to `self_reported_*` on load so no consumer mistakes them for verified
|
|
30
|
+
# counts (see #5 layer 3).
|
|
31
|
+
AUDIT_SHAPED_KEYS = frozenset(
|
|
32
|
+
{
|
|
33
|
+
"epistemic_audit",
|
|
34
|
+
"confidence",
|
|
35
|
+
"gaps",
|
|
36
|
+
"verified_claims",
|
|
37
|
+
"unverified_claims",
|
|
38
|
+
"verified_sources",
|
|
39
|
+
"total_sources",
|
|
40
|
+
"total_claims",
|
|
41
|
+
"total_candidates",
|
|
42
|
+
"claims_total",
|
|
43
|
+
"claims_verified",
|
|
44
|
+
"claims_unverified",
|
|
45
|
+
"claims_falsified",
|
|
46
|
+
"claims_unverifiable",
|
|
47
|
+
"rejected_claims",
|
|
48
|
+
"mean_epistemic_score",
|
|
49
|
+
"primary_sources",
|
|
50
|
+
"primary_benchmarks",
|
|
51
|
+
"technical_specs",
|
|
52
|
+
"peer_reviewed",
|
|
53
|
+
}
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
# Licenses that may be `depend`ed on or `vendor`ed. Everything else (incl.
|
|
57
|
+
# unknown / NOASSERTION) resolves to clean-room-rebuild-only.
|
|
58
|
+
PERMISSIVE_LICENSES = frozenset({"MIT", "APACHE-2.0", "BSD-3-CLAUSE", "ISC"})
|
|
59
|
+
|
|
60
|
+
UNKNOWN_LICENSES = frozenset({"", "UNKNOWN", "NOASSERTION", "NONE", "OTHER"})
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _norm_license(value: Any) -> str:
|
|
64
|
+
return str(value or "").strip().upper()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _is_permissive(value: Any) -> bool:
|
|
68
|
+
return _norm_license(value) in PERMISSIVE_LICENSES
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def sanitize_self_reported(dossier: Dict[str, Any]) -> Tuple[Dict[str, Any], List[str]]:
|
|
72
|
+
"""Rename agent-authored audit-shaped keys; return (dossier, renamed_keys).
|
|
73
|
+
|
|
74
|
+
Never deletes — the values are preserved under `self_reported_*` so a human
|
|
75
|
+
can still see what the agent claimed, but nothing counts them.
|
|
76
|
+
"""
|
|
77
|
+
renamed: List[str] = []
|
|
78
|
+
if not isinstance(dossier, dict):
|
|
79
|
+
return dossier, renamed
|
|
80
|
+
for key in list(dossier.keys()):
|
|
81
|
+
if key in AUDIT_SHAPED_KEYS:
|
|
82
|
+
dossier[f"self_reported_{key}"] = dossier.pop(key)
|
|
83
|
+
renamed.append(key)
|
|
84
|
+
return dossier, renamed
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _repo_key(candidate: Dict[str, Any]) -> Optional[str]:
|
|
88
|
+
"""Stable key for a candidate repo: `owner/repo`, lowercased."""
|
|
89
|
+
raw = str(candidate.get("name") or candidate.get("repo") or "").strip()
|
|
90
|
+
if not raw:
|
|
91
|
+
url = str(candidate.get("url") or "").strip()
|
|
92
|
+
match = re.search(r"github\.com/([^/]+/[^/#?]+)", url)
|
|
93
|
+
if match:
|
|
94
|
+
raw = match.group(1)
|
|
95
|
+
raw = raw.removesuffix(".git").strip("/")
|
|
96
|
+
return raw.lower() if raw else None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _verdict_claims(role: str, candidate: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
100
|
+
"""One standard claim per candidate verdict (verifiable when a quote exists)."""
|
|
101
|
+
name = candidate.get("name") or candidate.get("url") or "unknown-repo"
|
|
102
|
+
source_hash = candidate.get("source_hash")
|
|
103
|
+
source_url = candidate.get("url")
|
|
104
|
+
quote = candidate.get("verbatim_quote")
|
|
105
|
+
out: List[Dict[str, Any]] = []
|
|
106
|
+
for i, verdict in enumerate(candidate.get("verdicts") or [], start=1):
|
|
107
|
+
if not isinstance(verdict, dict):
|
|
108
|
+
continue
|
|
109
|
+
rid = candidate.get("repo_id") or name
|
|
110
|
+
out.append(
|
|
111
|
+
{
|
|
112
|
+
"claim_id": f"{rid}-{role}-V{i}",
|
|
113
|
+
"tag": "VERIFIED",
|
|
114
|
+
"statement": (
|
|
115
|
+
f"{name}: {verdict.get('feature', '?')} → "
|
|
116
|
+
f"{verdict.get('verdict', '?')} ({verdict.get('reason', '')})"
|
|
117
|
+
),
|
|
118
|
+
"source_hash": source_hash,
|
|
119
|
+
"source_url": source_url,
|
|
120
|
+
# Prefer a verdict-specific quote; fall back to the candidate's.
|
|
121
|
+
"verbatim_quote": verdict.get("verbatim_quote") or quote,
|
|
122
|
+
}
|
|
123
|
+
)
|
|
124
|
+
return out
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def normalize_dossier_claims(
|
|
128
|
+
dossier: Dict[str, Any], role: str
|
|
129
|
+
) -> List[Dict[str, Any]]:
|
|
130
|
+
"""Standard-schema claims for a dossier.
|
|
131
|
+
|
|
132
|
+
Uses the canonical keys when present; otherwise (darkharvest) derives them
|
|
133
|
+
from `candidate_repositories[]` so the auditor has something to verify.
|
|
134
|
+
"""
|
|
135
|
+
if not isinstance(dossier, dict):
|
|
136
|
+
return []
|
|
137
|
+
standard = dossier.get("affirmative_claims") or dossier.get("falsification_claims")
|
|
138
|
+
if isinstance(standard, list) and standard:
|
|
139
|
+
return [c for c in standard if isinstance(c, dict)]
|
|
140
|
+
|
|
141
|
+
out: List[Dict[str, Any]] = []
|
|
142
|
+
for candidate in dossier.get("candidate_repositories") or []:
|
|
143
|
+
if isinstance(candidate, dict):
|
|
144
|
+
out.extend(_verdict_claims(role, candidate))
|
|
145
|
+
return out
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _resolved_policy(alpha: Dict[str, Any], beta: Dict[str, Any]) -> str:
|
|
149
|
+
"""Conservative harvest policy when the two dossiers disagree."""
|
|
150
|
+
for side in (alpha, beta):
|
|
151
|
+
policy = str(side.get("harvest_policy") or "").lower()
|
|
152
|
+
risk = str(side.get("license_risk") or "").upper()
|
|
153
|
+
if policy == "clean-room-rebuild-only" or risk in (
|
|
154
|
+
"COPYLEFT_WARNING",
|
|
155
|
+
"PROHIBITIVE",
|
|
156
|
+
):
|
|
157
|
+
return "clean-room-rebuild-only"
|
|
158
|
+
for side in (alpha, beta):
|
|
159
|
+
if not _is_permissive(side.get("license")):
|
|
160
|
+
return "clean-room-rebuild-only"
|
|
161
|
+
return "depend-or-vendor"
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
class RepoValidator:
|
|
165
|
+
"""Ground-truth repo checks via the GitHub API.
|
|
166
|
+
|
|
167
|
+
Fail-open: any network/timeout error yields `exists=None` and blocks nothing,
|
|
168
|
+
so an offline or rate-limited run degrades rather than aborting.
|
|
169
|
+
"""
|
|
170
|
+
|
|
171
|
+
def __init__(
|
|
172
|
+
self,
|
|
173
|
+
token: Optional[str] = None,
|
|
174
|
+
timeout: float = 5.0,
|
|
175
|
+
enabled: bool = True,
|
|
176
|
+
):
|
|
177
|
+
self.token = token if token is not None else os.environ.get("GITHUB_TOKEN")
|
|
178
|
+
self.timeout = timeout
|
|
179
|
+
self.enabled = enabled
|
|
180
|
+
self._cache: Dict[str, Dict[str, Any]] = {}
|
|
181
|
+
|
|
182
|
+
def validate(self, owner_repo: str) -> Dict[str, Any]:
|
|
183
|
+
key = owner_repo.strip().lower()
|
|
184
|
+
if key in self._cache:
|
|
185
|
+
return self._cache[key]
|
|
186
|
+
result = {"exists": None, "license": None, "archived": None}
|
|
187
|
+
if self.enabled and key:
|
|
188
|
+
result = self._fetch(owner_repo)
|
|
189
|
+
self._cache[key] = result
|
|
190
|
+
return result
|
|
191
|
+
|
|
192
|
+
def _fetch(self, owner_repo: str) -> Dict[str, Any]:
|
|
193
|
+
request = urllib.request.Request(
|
|
194
|
+
f"https://api.github.com/repos/{owner_repo}",
|
|
195
|
+
headers={
|
|
196
|
+
"Accept": "application/vnd.github+json",
|
|
197
|
+
"User-Agent": "iumbtems-darkharvest-audit",
|
|
198
|
+
**({"Authorization": f"Bearer {self.token}"} if self.token else {}),
|
|
199
|
+
},
|
|
200
|
+
)
|
|
201
|
+
try:
|
|
202
|
+
with urllib.request.urlopen(request, timeout=self.timeout) as resp:
|
|
203
|
+
data = json.loads(resp.read().decode("utf-8", errors="replace"))
|
|
204
|
+
license_field = data.get("license") or {}
|
|
205
|
+
return {
|
|
206
|
+
"exists": True,
|
|
207
|
+
"license": license_field.get("spdx_id"),
|
|
208
|
+
"archived": bool(data.get("archived")),
|
|
209
|
+
}
|
|
210
|
+
except urllib.error.HTTPError as e:
|
|
211
|
+
if e.code == 404:
|
|
212
|
+
return {"exists": False, "license": None, "archived": None}
|
|
213
|
+
return {"exists": None, "license": None, "archived": None}
|
|
214
|
+
except Exception:
|
|
215
|
+
return {"exists": None, "license": None, "archived": None}
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def cross_check_repos(
|
|
219
|
+
alpha_dossier: Dict[str, Any],
|
|
220
|
+
beta_dossier: Dict[str, Any],
|
|
221
|
+
validator: Optional[RepoValidator] = None,
|
|
222
|
+
) -> List[Dict[str, Any]]:
|
|
223
|
+
"""Blocking findings for license disagreement and ground-truth failures.
|
|
224
|
+
|
|
225
|
+
A dialectic that produces a `depend-or-vendor` verdict on an unstated license
|
|
226
|
+
must not reach a human as `divergence: 0`. Each finding carries the
|
|
227
|
+
conservative resolution.
|
|
228
|
+
"""
|
|
229
|
+
alpha = {
|
|
230
|
+
key: c
|
|
231
|
+
for c in (alpha_dossier.get("candidate_repositories") or [])
|
|
232
|
+
if isinstance(c, dict) and (key := _repo_key(c))
|
|
233
|
+
}
|
|
234
|
+
beta = {
|
|
235
|
+
key: c
|
|
236
|
+
for c in (beta_dossier.get("candidate_repositories") or [])
|
|
237
|
+
if isinstance(c, dict) and (key := _repo_key(c))
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
findings: List[Dict[str, Any]] = []
|
|
241
|
+
for key in sorted(set(alpha) | set(beta)):
|
|
242
|
+
a = alpha.get(key)
|
|
243
|
+
b = beta.get(key)
|
|
244
|
+
if a and b:
|
|
245
|
+
alic, blic = (
|
|
246
|
+
_norm_license(a.get("license")),
|
|
247
|
+
_norm_license(b.get("license")),
|
|
248
|
+
)
|
|
249
|
+
if alic != blic:
|
|
250
|
+
findings.append(
|
|
251
|
+
{
|
|
252
|
+
"repo": key,
|
|
253
|
+
"kind": "LICENSE_DISAGREEMENT",
|
|
254
|
+
"severity": "BLOCKING",
|
|
255
|
+
"alpha": {
|
|
256
|
+
"license": a.get("license"),
|
|
257
|
+
"risk": a.get("license_risk"),
|
|
258
|
+
},
|
|
259
|
+
"beta": {
|
|
260
|
+
"license": b.get("license"),
|
|
261
|
+
"risk": b.get("license_risk"),
|
|
262
|
+
},
|
|
263
|
+
"resolution": _resolved_policy(a, b),
|
|
264
|
+
}
|
|
265
|
+
)
|
|
266
|
+
elif str(a.get("harvest_policy") or "") != str(
|
|
267
|
+
b.get("harvest_policy") or ""
|
|
268
|
+
):
|
|
269
|
+
findings.append(
|
|
270
|
+
{
|
|
271
|
+
"repo": key,
|
|
272
|
+
"kind": "POLICY_DISAGREEMENT",
|
|
273
|
+
"severity": "BLOCKING",
|
|
274
|
+
"alpha": {"policy": a.get("harvest_policy")},
|
|
275
|
+
"beta": {"policy": b.get("harvest_policy")},
|
|
276
|
+
"resolution": _resolved_policy(a, b),
|
|
277
|
+
}
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
# Ground truth (fail-open when the API is unreachable).
|
|
281
|
+
if validator is not None:
|
|
282
|
+
for side_name, side in (("alpha", a), ("beta", b)):
|
|
283
|
+
if not side:
|
|
284
|
+
continue
|
|
285
|
+
truth = validator.validate(key)
|
|
286
|
+
if truth.get("exists") is False:
|
|
287
|
+
findings.append(
|
|
288
|
+
{
|
|
289
|
+
"repo": key,
|
|
290
|
+
"kind": "REPO_UNRESOLVABLE",
|
|
291
|
+
"severity": "BLOCKING",
|
|
292
|
+
"source": side_name,
|
|
293
|
+
"detail": "GitHub API returned 404",
|
|
294
|
+
"resolution": "reject-citation",
|
|
295
|
+
}
|
|
296
|
+
)
|
|
297
|
+
break
|
|
298
|
+
if truth.get("archived") is True:
|
|
299
|
+
findings.append(
|
|
300
|
+
{
|
|
301
|
+
"repo": key,
|
|
302
|
+
"kind": "REPO_ARCHIVED",
|
|
303
|
+
"severity": "WARNING",
|
|
304
|
+
"detail": "repository is archived",
|
|
305
|
+
}
|
|
306
|
+
)
|
|
307
|
+
break
|
|
308
|
+
truth_license = _norm_license(truth.get("license"))
|
|
309
|
+
if (
|
|
310
|
+
truth.get("exists")
|
|
311
|
+
and truth_license
|
|
312
|
+
and not _is_permissive(truth.get("license"))
|
|
313
|
+
and _is_permissive(side.get("license"))
|
|
314
|
+
):
|
|
315
|
+
findings.append(
|
|
316
|
+
{
|
|
317
|
+
"repo": key,
|
|
318
|
+
"kind": "LICENSE_CONTRADICTS_GROUND_TRUTH",
|
|
319
|
+
"severity": "BLOCKING",
|
|
320
|
+
"source": side_name,
|
|
321
|
+
"detail": f"dossier says {side.get('license')}, GitHub says {truth.get('license')}",
|
|
322
|
+
"resolution": "clean-room-rebuild-only",
|
|
323
|
+
}
|
|
324
|
+
)
|
|
325
|
+
break
|
|
326
|
+
return findings
|
package/runner/refinement.py
CHANGED
|
@@ -390,3 +390,77 @@ def compute_epistemic_score_from_claims(
|
|
|
390
390
|
"raw_score": raw_score,
|
|
391
391
|
}
|
|
392
392
|
return epistemic_score, breakdown
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
# Brainstorm's deliverable is hypotheses, not quote-verified facts. Scoring it
|
|
396
|
+
# with the research formula (which counts only verified claims positively)
|
|
397
|
+
# guarantees WARNING_LOW_GROUNDING on a well-formed ideation run.
|
|
398
|
+
BRAINSTORM_MODE_THRESHOLD = 0.5
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def compute_brainstorm_score_from_claims(
|
|
402
|
+
claims: Iterable[ClaimWitness],
|
|
403
|
+
constitution: Optional[Constitution] = None,
|
|
404
|
+
) -> Tuple[float, Dict[str, Any]]:
|
|
405
|
+
"""E(D) variant for brainstorm mode: credit *well-formed* speculation.
|
|
406
|
+
|
|
407
|
+
A hypothesis scores when it carries a falsification criterion; an inference
|
|
408
|
+
when it names its parent claims; negative knowledge earns the usual bonus;
|
|
409
|
+
verified workspace facts still count. Only rejected (quote-unverified)
|
|
410
|
+
VERIFIED claims are penalized. This keeps brainstorm honest about grounding
|
|
411
|
+
without failing a mode whose output is speculative by design.
|
|
412
|
+
"""
|
|
413
|
+
constitution = constitution or LEGACY_CONSTITUTION
|
|
414
|
+
counts = {
|
|
415
|
+
"verified": 0,
|
|
416
|
+
"rejected": 0,
|
|
417
|
+
"inferred": 0,
|
|
418
|
+
"hypotheses": 0,
|
|
419
|
+
"neg_knowledge": 0,
|
|
420
|
+
}
|
|
421
|
+
well_formed = 0
|
|
422
|
+
for c in claims:
|
|
423
|
+
if c.status == STATUS_REJECTED or c.tag == "UNVERIFIED_REJECTED":
|
|
424
|
+
counts["rejected"] += 1
|
|
425
|
+
elif c.tag == TAG_VERIFIED:
|
|
426
|
+
counts["verified"] += 1
|
|
427
|
+
elif c.tag == TAG_INFERRED:
|
|
428
|
+
counts["inferred"] += 1
|
|
429
|
+
if c.parent_claims:
|
|
430
|
+
well_formed += 1
|
|
431
|
+
elif c.tag == TAG_HYPOTHESIS:
|
|
432
|
+
counts["hypotheses"] += 1
|
|
433
|
+
if (c.falsification or "").strip():
|
|
434
|
+
well_formed += 1
|
|
435
|
+
elif c.tag == TAG_NEGATIVE_KNOWLEDGE:
|
|
436
|
+
counts["neg_knowledge"] += 1
|
|
437
|
+
|
|
438
|
+
total_assertions = max(
|
|
439
|
+
1,
|
|
440
|
+
counts["verified"]
|
|
441
|
+
+ counts["inferred"]
|
|
442
|
+
+ counts["hypotheses"]
|
|
443
|
+
+ counts["rejected"]
|
|
444
|
+
+ counts["neg_knowledge"],
|
|
445
|
+
)
|
|
446
|
+
raw_score = (
|
|
447
|
+
counts["verified"]
|
|
448
|
+
+ well_formed
|
|
449
|
+
+ constitution.neg_bonus * counts["neg_knowledge"]
|
|
450
|
+
- constitution.reject_penalty * counts["rejected"]
|
|
451
|
+
) / total_assertions
|
|
452
|
+
epistemic_score = max(0.0, min(1.0, round(raw_score, 3)))
|
|
453
|
+
|
|
454
|
+
breakdown = {
|
|
455
|
+
"total_claims_audited": counts["verified"] + counts["rejected"],
|
|
456
|
+
"verified_passed": counts["verified"],
|
|
457
|
+
"unverified_rejected": counts["rejected"],
|
|
458
|
+
"negative_knowledge_count": counts["neg_knowledge"],
|
|
459
|
+
"inferred_count": counts["inferred"],
|
|
460
|
+
"hypothesis_count": counts["hypotheses"],
|
|
461
|
+
"well_formed_count": well_formed,
|
|
462
|
+
"verified_weight": float(counts["verified"]),
|
|
463
|
+
"total_assertions": total_assertions,
|
|
464
|
+
"raw_score": raw_score,
|
|
465
|
+
}
|
|
466
|
+
return epistemic_score, breakdown
|
package/runner/research_swarm.py
CHANGED
|
@@ -28,6 +28,8 @@ from runner.state_machine import (
|
|
|
28
28
|
_atomic_write_json,
|
|
29
29
|
)
|
|
30
30
|
from runner.auditor_engine import EpistemicAuditorEngine
|
|
31
|
+
from runner.retrieval_log import current_offset as _retrieval_offset
|
|
32
|
+
from runner.retrieval_log import summarize as _retrieval_summary
|
|
31
33
|
from skills.research_cache.hasher import SourceHasher
|
|
32
34
|
from skills.swarm_config.configure import load_config
|
|
33
35
|
|
|
@@ -268,7 +270,17 @@ class SwarmRunner:
|
|
|
268
270
|
self.state_machine = ResearchStateMachine(
|
|
269
271
|
base_dir=self.base_dir, mode=self.mode
|
|
270
272
|
)
|
|
271
|
-
|
|
273
|
+
from runner.auditor_engine import _repo_validation_enabled
|
|
274
|
+
from runner.darkharvest_claims import RepoValidator
|
|
275
|
+
|
|
276
|
+
self.auditor = EpistemicAuditorEngine(
|
|
277
|
+
base_dir=self.base_dir,
|
|
278
|
+
# Ground-truth repo checks are for real runs; mock dossiers are
|
|
279
|
+
# synthetic, so the unit suite must stay offline.
|
|
280
|
+
repo_validator=RepoValidator(
|
|
281
|
+
enabled=(not self.mock_mode) and _repo_validation_enabled()
|
|
282
|
+
),
|
|
283
|
+
)
|
|
272
284
|
self.hasher = SourceHasher(base_dir=self.base_dir)
|
|
273
285
|
self.prompts_dir = PROJECT_ROOT / "prompts"
|
|
274
286
|
|
|
@@ -421,7 +433,13 @@ class SwarmRunner:
|
|
|
421
433
|
# this the worker operated on the launcher's PWD (observed live:
|
|
422
434
|
# runner evidence dir 5da7ec/sunny-otter, worker cwd
|
|
423
435
|
# /home/john/Projects/STC). Keep PWD consistent with cwd.
|
|
424
|
-
|
|
436
|
+
# IUMBTEMS_RESEARCH_DIR points agent-invoked skill scripts
|
|
437
|
+
# (search.py/hasher.py) at this workspace for retrieval telemetry.
|
|
438
|
+
env={
|
|
439
|
+
**os.environ,
|
|
440
|
+
"PWD": agent_cwd,
|
|
441
|
+
"IUMBTEMS_RESEARCH_DIR": str(self.base_dir),
|
|
442
|
+
},
|
|
425
443
|
# stdin MUST be DEVNULL: `opencode run` reads piped stdin to
|
|
426
444
|
# EOF before starting, and the MCP server's inherited stdin
|
|
427
445
|
# pipe is held open by the harness — every agent hung forever
|
|
@@ -580,14 +598,28 @@ Output ONLY valid JSON representing the scope decomposition conforming to prompt
|
|
|
580
598
|
return SwarmRunner._parse_json_block(text, require_key="scope_id")
|
|
581
599
|
|
|
582
600
|
@staticmethod
|
|
583
|
-
def _load_dossier_file(
|
|
584
|
-
|
|
601
|
+
def _load_dossier_file(
|
|
602
|
+
path: Path, expected_scope: Optional[str] = None
|
|
603
|
+
) -> Optional[Dict[str, Any]]:
|
|
604
|
+
"""Load a path as a dossier dict for `expected_scope`, or None.
|
|
605
|
+
|
|
606
|
+
A file under the research dir is untrusted input: require a `scope_id`
|
|
607
|
+
that matches the scope being loaded, so a stray or stub artifact cannot
|
|
608
|
+
satisfy completion for a different scope.
|
|
609
|
+
"""
|
|
585
610
|
try:
|
|
586
611
|
with open(path, "r", encoding="utf-8") as f:
|
|
587
612
|
data = json.load(f)
|
|
588
613
|
except (OSError, ValueError):
|
|
589
614
|
return None
|
|
590
|
-
|
|
615
|
+
if not isinstance(data, dict):
|
|
616
|
+
return None
|
|
617
|
+
scope_id = data.get("scope_id")
|
|
618
|
+
if not scope_id:
|
|
619
|
+
return None
|
|
620
|
+
if expected_scope is not None and scope_id != expected_scope:
|
|
621
|
+
return None
|
|
622
|
+
return data
|
|
591
623
|
|
|
592
624
|
def _dossier_candidates(
|
|
593
625
|
self, dossier_path: Path, aliases: Sequence[str]
|
|
@@ -639,7 +671,7 @@ Output ONLY valid JSON representing the scope decomposition conforming to prompt
|
|
|
639
671
|
agent's stdout, then raises a diagnostic error naming where it looked.
|
|
640
672
|
"""
|
|
641
673
|
for candidate in self._dossier_candidates(dossier_path, aliases):
|
|
642
|
-
data = self._load_dossier_file(candidate)
|
|
674
|
+
data = self._load_dossier_file(candidate, scope_id)
|
|
643
675
|
if data is None:
|
|
644
676
|
continue
|
|
645
677
|
if candidate != dossier_path:
|
|
@@ -1128,7 +1160,9 @@ Output ONLY valid JSON representing the scope decomposition conforming to prompt
|
|
|
1128
1160
|
from runner.refinement import load_domain_pack
|
|
1129
1161
|
|
|
1130
1162
|
constitution = load_domain_pack(self.domain_pack)
|
|
1131
|
-
audit_report = self.auditor.audit_scope(
|
|
1163
|
+
audit_report = self.auditor.audit_scope(
|
|
1164
|
+
scope_id, constitution=constitution, mode=self.mode
|
|
1165
|
+
)
|
|
1132
1166
|
summary = audit_report["summary"]
|
|
1133
1167
|
print(
|
|
1134
1168
|
f" [Audit Result] Score: {summary['epistemic_score']}/1.0 | Divergence: {summary['divergence_score']} | Verified: {summary['verified_passed']} | Rejected: {summary['unverified_rejected']}"
|
|
@@ -1210,6 +1244,11 @@ Output ONLY valid JSON representing the scope decomposition conforming to prompt
|
|
|
1210
1244
|
)
|
|
1211
1245
|
print("=" * 70)
|
|
1212
1246
|
|
|
1247
|
+
# Retrieval telemetry baseline: count only events from this run.
|
|
1248
|
+
self._retrieval_offset = _retrieval_offset(self.base_dir)
|
|
1249
|
+
print(f" {self._retrieval_banner()}")
|
|
1250
|
+
print("=" * 70)
|
|
1251
|
+
|
|
1213
1252
|
# 1. Orchestrate
|
|
1214
1253
|
self.orchestrate_objective(objective, frontier_file)
|
|
1215
1254
|
|
|
@@ -1223,17 +1262,51 @@ Output ONLY valid JSON representing the scope decomposition conforming to prompt
|
|
|
1223
1262
|
)
|
|
1224
1263
|
report_path = self._compile_master_synthesis(objective)
|
|
1225
1264
|
self.state_machine.update_session_status(SessionStatus.COMPLETED)
|
|
1265
|
+
self._write_orphan_report()
|
|
1226
1266
|
|
|
1227
1267
|
duration = (datetime.now(timezone.utc) - start_time).total_seconds()
|
|
1268
|
+
print(f" {self._retrieval_banner()}")
|
|
1228
1269
|
print(f"\n🎉 Swarm run completed in {duration:.1f}s. Report: {report_path}")
|
|
1229
1270
|
|
|
1271
|
+
def _write_orphan_report(self) -> None:
|
|
1272
|
+
"""Record scope dirs absent from every manifest (mark-only, never delete)."""
|
|
1273
|
+
try:
|
|
1274
|
+
orphans = self.state_machine.find_orphan_scopes()
|
|
1275
|
+
except Exception: # noqa: BLE001 - hygiene must not fail the run
|
|
1276
|
+
return
|
|
1277
|
+
if not orphans:
|
|
1278
|
+
return
|
|
1279
|
+
path = self.base_dir / "orphans.json"
|
|
1280
|
+
try:
|
|
1281
|
+
path.write_text(json.dumps(orphans, indent=2), encoding="utf-8")
|
|
1282
|
+
except OSError:
|
|
1283
|
+
return
|
|
1284
|
+
print(
|
|
1285
|
+
f" ⚠️ {len(orphans)} orphaned scope dir(s) not referenced by any "
|
|
1286
|
+
f"manifest -> {path.name} (not deleted)"
|
|
1287
|
+
)
|
|
1288
|
+
|
|
1289
|
+
def _retrieval_banner(self) -> str:
|
|
1290
|
+
"""One-line retrieval summary for the current run (issue #3).
|
|
1291
|
+
|
|
1292
|
+
`N queries, M results, K cached` makes an ungrounded run obvious at a
|
|
1293
|
+
glance instead of inferring it from an empty `sources/`.
|
|
1294
|
+
"""
|
|
1295
|
+
since = getattr(self, "_retrieval_offset", 0)
|
|
1296
|
+
stats = _retrieval_summary(self.base_dir, since=since)
|
|
1297
|
+
return (
|
|
1298
|
+
f"retrieval: {stats['queries']} queries, {stats['results']} results, "
|
|
1299
|
+
f"{stats['cached']} cached"
|
|
1300
|
+
)
|
|
1301
|
+
|
|
1230
1302
|
def _collect_scope_totals(
|
|
1231
1303
|
self, manifest: Dict[str, Any], synthesis_lines: List[str]
|
|
1232
|
-
) -> Tuple[int, int, List[float]]:
|
|
1304
|
+
) -> Tuple[int, int, List[float], List[str]]:
|
|
1233
1305
|
"""Fold per-scope audit/synthesis artifacts into running totals."""
|
|
1234
1306
|
total_verified = 0
|
|
1235
1307
|
total_rejected = 0
|
|
1236
1308
|
all_divergences: List[float] = []
|
|
1309
|
+
verdicts: List[str] = []
|
|
1237
1310
|
for scope in manifest["scopes"]:
|
|
1238
1311
|
sid = scope["scope_id"]
|
|
1239
1312
|
scope_dir = self.state_machine.get_scope_dir(sid)
|
|
@@ -1246,11 +1319,31 @@ Output ONLY valid JSON representing the scope decomposition conforming to prompt
|
|
|
1246
1319
|
total_verified += summary["verified_passed"]
|
|
1247
1320
|
total_rejected += summary["unverified_rejected"]
|
|
1248
1321
|
all_divergences.append(summary["divergence_score"])
|
|
1322
|
+
verdicts.append(summary.get("verdict", ""))
|
|
1249
1323
|
|
|
1250
1324
|
if synth_file.exists():
|
|
1251
1325
|
synthesis_lines.append(synth_file.read_text(encoding="utf-8"))
|
|
1252
1326
|
synthesis_lines.append("\n---\n")
|
|
1253
|
-
return total_verified, total_rejected, all_divergences
|
|
1327
|
+
return total_verified, total_rejected, all_divergences, verdicts
|
|
1328
|
+
|
|
1329
|
+
def _grounding_sentence(self, verified: int, rejected: int, warnings: int) -> str:
|
|
1330
|
+
"""Executive summary that matches the audit — never claims certainty it lacks."""
|
|
1331
|
+
if verified == 0 and warnings:
|
|
1332
|
+
return (
|
|
1333
|
+
"⚠️ **WARNING_LOW_GROUNDING**: this run verified 0 primary citations "
|
|
1334
|
+
f"({rejected} claim(s) rejected). Treat the findings below as unverified "
|
|
1335
|
+
"hypotheses, not evidence."
|
|
1336
|
+
)
|
|
1337
|
+
if warnings:
|
|
1338
|
+
return (
|
|
1339
|
+
f"⚠️ {warnings} scope(s) reported WARNING_LOW_GROUNDING. "
|
|
1340
|
+
f"{verified} primary citation(s) verified; {rejected} rejected."
|
|
1341
|
+
)
|
|
1342
|
+
return (
|
|
1343
|
+
f"Compiled by the IUMBTEMS dialectic harness ({self.mode} mode); "
|
|
1344
|
+
f"{verified} primary citation(s) verified against the content-addressed "
|
|
1345
|
+
"source cache."
|
|
1346
|
+
)
|
|
1254
1347
|
|
|
1255
1348
|
def _write_report(self, filename: str, lines: List[str]) -> Path:
|
|
1256
1349
|
path = self.base_dir / filename
|
|
@@ -1269,28 +1362,30 @@ Output ONLY valid JSON representing the scope decomposition conforming to prompt
|
|
|
1269
1362
|
}
|
|
1270
1363
|
title = mode_titles.get(self.mode, "Master Epistemic Research Report")
|
|
1271
1364
|
|
|
1365
|
+
# Collect per-scope sections first so the executive summary can reflect
|
|
1366
|
+
# the actual audit verdict instead of asserting certainty we don't have.
|
|
1367
|
+
scope_sections: List[str] = []
|
|
1368
|
+
total_verified, total_rejected, all_divergences, verdicts = (
|
|
1369
|
+
self._collect_scope_totals(manifest, scope_sections)
|
|
1370
|
+
)
|
|
1371
|
+
avg_div = round(sum(all_divergences) / max(1, len(all_divergences)), 2)
|
|
1372
|
+
warnings = sum(1 for v in verdicts if v == "WARNING_LOW_GROUNDING")
|
|
1373
|
+
|
|
1272
1374
|
synthesis_lines = [
|
|
1273
1375
|
f"# {title}: {objective}\n",
|
|
1274
1376
|
f"**Session ID**: `{manifest['session_id']}` | **Mode**: `{self.mode.upper()}` | **Engine**: `{self.engine}` | **Generated**: `{manifest['updated_at']}`\n",
|
|
1275
1377
|
"## Executive Summary",
|
|
1276
|
-
|
|
1378
|
+
self._grounding_sentence(total_verified, total_rejected, warnings) + "\n",
|
|
1379
|
+
f"{self._retrieval_banner()}\n",
|
|
1277
1380
|
"## Scope Findings & Dialectic Balance Sheets\n",
|
|
1381
|
+
*scope_sections,
|
|
1382
|
+
"\n## Swarm Epistemic Audit Totals\n",
|
|
1383
|
+
f"- **Total Verified Primary Citations**: `{total_verified}`",
|
|
1384
|
+
f"- **Total Unverified Claims Purged**: `{total_rejected}`",
|
|
1385
|
+
f"- **Mean Swarm Divergence Score**: `{avg_div}`",
|
|
1386
|
+
f"- **Scopes Flagged WARNING_LOW_GROUNDING**: `{warnings}`",
|
|
1278
1387
|
]
|
|
1279
1388
|
|
|
1280
|
-
total_verified, total_rejected, all_divergences = self._collect_scope_totals(
|
|
1281
|
-
manifest, synthesis_lines
|
|
1282
|
-
)
|
|
1283
|
-
|
|
1284
|
-
avg_div = round(sum(all_divergences) / max(1, len(all_divergences)), 2)
|
|
1285
|
-
synthesis_lines.append("\n## Swarm Epistemic Audit Totals\n")
|
|
1286
|
-
synthesis_lines.append(
|
|
1287
|
-
f"- **Total Verified Primary Citations**: `{total_verified}`"
|
|
1288
|
-
)
|
|
1289
|
-
synthesis_lines.append(
|
|
1290
|
-
f"- **Total Unverified Claims Purged**: `{total_rejected}`"
|
|
1291
|
-
)
|
|
1292
|
-
synthesis_lines.append(f"- **Mean Swarm Divergence Score**: `{avg_div}`")
|
|
1293
|
-
|
|
1294
1389
|
final_path = self._write_report("final_synthesis.md", synthesis_lines)
|
|
1295
1390
|
|
|
1296
1391
|
# Also write specialized report files for audit/scout/brainstorm/darkharvest.
|