@dzhechkov/p-replicator 1.12.0 → 1.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dz-manifest.json +225 -61
- package/CHANGELOG.md +148 -1
- package/LICENSE +21 -0
- package/MULTIPLATFORM_ROADMAP.md +1 -1
- package/README/eng/01_quickstart.md +2 -2
- package/README/eng/02_user_guide.md +1 -1
- package/README/eng/03_admin_guide.md +2 -2
- package/README/eng/05_architecture.md +1 -1
- package/README/eng/README.md +2 -1
- package/README/ru/01_quickstart.md +2 -2
- package/README/ru/02_user_guide.md +1 -1
- package/README/ru/03_admin_guide.md +2 -2
- package/README/ru/05_architecture.md +1 -1
- package/README/ru/README.md +2 -1
- package/README/ru/html/index.html +8 -8
- package/README.md +132 -9
- package/bin/cli.js +0 -0
- package/package.json +10 -11
- package/sbom.json +470 -60
- package/scripts/check-pipeline-gaps.sh +0 -0
- package/src/commands/init.js +1 -1
- package/src/rule-components.json +5 -1
- package/src/utils.js +32 -3
- package/templates/.claude/agents/product-discoverer.md +38 -0
- package/templates/.claude/agents/replicate-coordinator.md +11 -1
- package/templates/.claude/commands/feature.md +29 -5
- package/templates/.claude/commands/go.md +6 -8
- package/templates/.claude/commands/harvest.md +5 -7
- package/templates/.claude/commands/replicate.md +169 -44
- package/templates/.claude/commands/start.md +28 -7
- package/templates/.claude/hooks/capture-source-path.cjs +795 -0
- package/templates/.claude/hooks/check-canon.cjs +493 -0
- package/templates/.claude/hooks/check-embed-contract.cjs +374 -0
- package/templates/.claude/hooks/check-external-deps.cjs +288 -0
- package/templates/.claude/hooks/check-file-ownership.cjs +424 -0
- package/templates/.claude/hooks/check-handoff-manifest.cjs +367 -0
- package/templates/.claude/hooks/check-job-contract.cjs +501 -0
- package/templates/.claude/hooks/check-look-origin.cjs +240 -0
- package/templates/.claude/hooks/check-look-trace.cjs +385 -0
- package/templates/.claude/hooks/check-metric-source.cjs +296 -0
- package/templates/.claude/hooks/check-model-cost.cjs +470 -0
- package/templates/.claude/hooks/check-ports.cjs +27 -6
- package/templates/.claude/hooks/check-source-version.cjs +312 -0
- package/templates/.claude/hooks/check-swarm-receipts.cjs +197 -0
- package/templates/.claude/hooks/check-webhook-contract.cjs +535 -0
- package/templates/.claude/hooks/statusline.cjs +2 -2
- package/templates/.claude/rules/embeddable-widget.md +73 -0
- package/templates/.claude/rules/feature-lifecycle.md +5 -6
- package/templates/.claude/rules/incoming-webhooks.md +99 -0
- package/templates/.claude/rules/long-running-job.md +73 -0
- package/templates/.claude/rules/model-call-cost.md +85 -0
- package/templates/.claude/rules/replicate-pipeline.md +121 -52
- package/templates/.claude/skills/brutal-honesty-review/SKILL.md +9 -0
- package/templates/.claude/skills/cc-toolkit-generator-enhanced/SKILL.md +4 -0
- package/templates/.claude/skills/cc-toolkit-generator-enhanced/modules/03-generate-p0.md +46 -1
- package/templates/.claude/skills/cc-toolkit-generator-enhanced/modules/04-generate-p1.md +7 -1
- package/templates/.claude/skills/cc-toolkit-generator-enhanced/modules/06-package-deliver.md +20 -2
- package/templates/.claude/skills/cc-toolkit-generator-enhanced/references/claude-md-strategy.md +7 -0
- package/templates/.claude/skills/cc-toolkit-generator-enhanced/references/templates/automation-commands.md +17 -0
- package/templates/.claude/skills/cc-toolkit-generator-enhanced/references/templates/feature-lifecycle-ent.md +43 -5
- package/templates/.claude/skills/cc-toolkit-generator-enhanced/references/templates/feature-lifecycle.md +43 -7
- package/templates/.claude/skills/cc-toolkit-generator-enhanced/references/templates/start-command.md +19 -1
- package/templates/.claude/skills/cc-toolkit-generator-enhanced/references/templates/swarm-file-evidence.md +151 -0
- package/templates/.claude/skills/goap-research-ed25519/SKILL.md +37 -22
- package/templates/.claude/skills/goap-research-ed25519/references/negative-results.md +94 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/check_report_evidence.py +368 -4
- package/templates/.claude/skills/goap-research-ed25519/scripts/ed25519_verifier.py +122 -5
- package/templates/.claude/skills/goap-research-ed25519/scripts/evidence_fetch.py +33 -16
- package/templates/.claude/skills/goap-research-ed25519/scripts/quote_provenance.py +342 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/test_ed25519_verifier.py +60 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/test_evidence_provenance.py +139 -6
- package/templates/.claude/skills/goap-research-ed25519/scripts/test_quote_provenance.py +274 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/test_suite_completeness.py +2 -1
- package/templates/.claude/skills/knowledge-extractor/SKILL.md +4 -0
- package/templates/.claude/skills/pipeline-forge/SKILL.md +18 -23
- package/templates/.claude/skills/pipeline-forge/examples/replicate-analysis.md +7 -2
- package/templates/.claude/skills/pipeline-forge/references/patterns-catalog.md +19 -1
- package/templates/.claude/skills/pipeline-forge/references/self-extracted-patterns.md +17 -6
- package/templates/.claude/skills/pipeline-forge/references/skill-anatomy.md +0 -1
- package/templates/.claude/skills/reverse-engineering-unicorn/modules/025-cjm-prototype.md +21 -1
- package/templates/.claude/skills/sparc-prd-mini/SKILL.md +173 -725
- package/tests/snapshot/baseline.json +60 -38
- package/tests/unit/capture-source-path.test.js +492 -0
- package/tests/unit/check-canon.test.js +403 -0
- package/tests/unit/check-embed-contract.test.js +422 -0
- package/tests/unit/check-external-deps.test.js +363 -0
- package/tests/unit/check-file-ownership.test.js +388 -0
- package/tests/unit/check-handoff-manifest.test.js +410 -0
- package/tests/unit/check-job-contract.test.js +514 -0
- package/tests/unit/check-look-origin.test.js +180 -0
- package/tests/unit/check-look-trace.test.js +420 -0
- package/tests/unit/check-metric-source.test.js +325 -0
- package/tests/unit/check-model-cost.test.js +425 -0
- package/tests/unit/check-ports.test.js +46 -2
- package/tests/unit/check-source-version.test.js +344 -0
- package/tests/unit/check-swarm-receipts.test.js +231 -0
- package/tests/unit/check-webhook-contract.test.js +536 -0
- package/tests/unit/db-port-rule.test.js +8 -2
- package/tests/unit/detection-ladder-registry.test.js +2 -2
- package/tests/unit/generator-swarm-contract.test.js +287 -0
- package/tests/unit/guard-honest-input-meta.test.js +64 -0
- package/tests/unit/honest-failure-rules.test.js +91 -9
- package/tests/unit/look-phase-contract.test.js +231 -0
- package/tests/unit/negative-conclusion-gate.test.js +300 -0
- package/tests/unit/quote-provenance.test.js +122 -0
- package/tests/unit/utils.test.js +40 -3
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Deterministic quote provenance over bounded, witness-captured excerpts.
|
|
3
|
+
|
|
4
|
+
The normalization corridor is CLOSED and shared by capture and verification:
|
|
5
|
+
Unicode NFC; HTML entity decoding for HTML bodies; whitespace-run collapse;
|
|
6
|
+
and only these typography folds: «»„“” -> ``"`` and —– -> ``-``. Case and
|
|
7
|
+
words are never changed, and word-overlap is never a verbatim check.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
import html
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
import tempfile
|
|
18
|
+
import unicodedata
|
|
19
|
+
from dataclasses import asdict, dataclass
|
|
20
|
+
from email.message import Message
|
|
21
|
+
from typing import Any, Dict, Mapping, Optional, Tuple
|
|
22
|
+
|
|
23
|
+
ACQUISITION_METHODS = ("raw-fetch", "tool-summary", "search-listing", "manual")
|
|
24
|
+
METHOD_UNKNOWN = "method-unknown"
|
|
25
|
+
VERBATIM_VERDICTS = (
|
|
26
|
+
"verbatim-confirmed",
|
|
27
|
+
"not-in-excerpt",
|
|
28
|
+
"no-excerpt",
|
|
29
|
+
"method-ineligible",
|
|
30
|
+
"hash-mismatch",
|
|
31
|
+
METHOD_UNKNOWN,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
EXCERPT_RADIUS_CHARS = 500
|
|
35
|
+
EXCERPT_MAX_CHARS = 2000
|
|
36
|
+
_TYPOGRAPHY_FOLD = str.maketrans({
|
|
37
|
+
"«": '"', "»": '"', "„": '"', "“": '"', "”": '"',
|
|
38
|
+
"—": "-", "–": "-",
|
|
39
|
+
})
|
|
40
|
+
_HTML_ENTITY_RE = re.compile(r"&(?:#\d+|#x[0-9a-fA-F]+|[A-Za-z][A-Za-z0-9]+);")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class ExcerptCaptureRefused(ValueError):
|
|
44
|
+
"""A named author-time refusal before an excerpt can be persisted."""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class ExcerptStoreError(ValueError):
|
|
48
|
+
"""The excerpt store exists but cannot be interpreted safely."""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass(frozen=True)
|
|
52
|
+
class QuoteRecord:
|
|
53
|
+
quote: str
|
|
54
|
+
acquisition: str
|
|
55
|
+
source_url: str
|
|
56
|
+
sha256_body: str
|
|
57
|
+
locator: Optional[str] = None
|
|
58
|
+
excerpt_id: Optional[str] = None
|
|
59
|
+
|
|
60
|
+
def __post_init__(self) -> None:
|
|
61
|
+
if not isinstance(self.quote, str) or not self.quote.strip():
|
|
62
|
+
raise ValueError("quote must be a non-blank string")
|
|
63
|
+
if self.acquisition not in ACQUISITION_METHODS:
|
|
64
|
+
raise ValueError(
|
|
65
|
+
f"acquisition {self.acquisition!r} is outside the closed set {ACQUISITION_METHODS}"
|
|
66
|
+
)
|
|
67
|
+
if not isinstance(self.source_url, str) or not self.source_url.strip():
|
|
68
|
+
raise ValueError("source_url must be a non-blank string")
|
|
69
|
+
if not isinstance(self.sha256_body, str) or not re.fullmatch(r"[0-9a-f]{64}", self.sha256_body):
|
|
70
|
+
raise ValueError("sha256_body must be 64 lowercase hexadecimal characters")
|
|
71
|
+
|
|
72
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
73
|
+
return asdict(self)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def read_acquisition(value_or_none: Any) -> str:
|
|
77
|
+
"""Read-side compatibility: absent or unknown stays unknown and grants nothing."""
|
|
78
|
+
return value_or_none if value_or_none in ACQUISITION_METHODS else METHOD_UNKNOWN
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _content_type_parts(content_type: Any) -> Tuple[Optional[str], Optional[str]]:
|
|
82
|
+
if not isinstance(content_type, str) or not content_type.strip():
|
|
83
|
+
return None, None
|
|
84
|
+
message = Message()
|
|
85
|
+
message["content-type"] = content_type
|
|
86
|
+
mime = message.get_content_type().lower()
|
|
87
|
+
charset = message.get_content_charset() or "utf-8"
|
|
88
|
+
return mime, charset
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _is_decodable_mime(mime: Optional[str]) -> bool:
|
|
92
|
+
if mime is None:
|
|
93
|
+
return False
|
|
94
|
+
return (
|
|
95
|
+
mime.startswith("text/")
|
|
96
|
+
or mime in {
|
|
97
|
+
"application/json", "application/ld+json", "application/xml",
|
|
98
|
+
"application/xhtml+xml", "application/javascript",
|
|
99
|
+
}
|
|
100
|
+
or mime.endswith("+json")
|
|
101
|
+
or mime.endswith("+xml")
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _decode_body(body: Any, content_type: Any) -> Tuple[str, str, str]:
|
|
106
|
+
mime, charset = _content_type_parts(content_type)
|
|
107
|
+
if not _is_decodable_mime(mime):
|
|
108
|
+
raise ExcerptCaptureRefused(
|
|
109
|
+
f"non-text body with content_type {content_type!r} cannot produce verbatim evidence"
|
|
110
|
+
)
|
|
111
|
+
if not isinstance(body, (bytes, bytearray, memoryview)):
|
|
112
|
+
raise ExcerptCaptureRefused("source body must be bytes captured by the fetch path")
|
|
113
|
+
try:
|
|
114
|
+
return bytes(body).decode(charset or "utf-8", errors="strict"), mime or "", charset or "utf-8"
|
|
115
|
+
except (LookupError, UnicodeDecodeError) as exc:
|
|
116
|
+
raise ExcerptCaptureRefused(
|
|
117
|
+
f"undecodable source body under declared charset {charset!r}: {exc}"
|
|
118
|
+
) from None
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _html_unescape_with_offsets(text: str) -> Tuple[str, list]:
|
|
122
|
+
output = []
|
|
123
|
+
offsets = []
|
|
124
|
+
cursor = 0
|
|
125
|
+
for match in _HTML_ENTITY_RE.finditer(text):
|
|
126
|
+
for index in range(cursor, match.start()):
|
|
127
|
+
output.append(text[index])
|
|
128
|
+
offsets.append(index)
|
|
129
|
+
decoded = html.unescape(match.group(0))
|
|
130
|
+
for char in decoded:
|
|
131
|
+
output.append(char)
|
|
132
|
+
offsets.append(match.start())
|
|
133
|
+
cursor = match.end()
|
|
134
|
+
for index in range(cursor, len(text)):
|
|
135
|
+
output.append(text[index])
|
|
136
|
+
offsets.append(index)
|
|
137
|
+
return "".join(output), offsets
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _normalize_with_offsets(text: str, *, is_html: bool) -> Tuple[str, list]:
|
|
141
|
+
if is_html:
|
|
142
|
+
current, offsets = _html_unescape_with_offsets(text)
|
|
143
|
+
else:
|
|
144
|
+
current, offsets = text, list(range(len(text)))
|
|
145
|
+
|
|
146
|
+
nfc_chars = []
|
|
147
|
+
nfc_offsets = []
|
|
148
|
+
index = 0
|
|
149
|
+
while index < len(current):
|
|
150
|
+
end = index + 1
|
|
151
|
+
while end < len(current) and unicodedata.combining(current[end]):
|
|
152
|
+
end += 1
|
|
153
|
+
normalized = unicodedata.normalize("NFC", current[index:end])
|
|
154
|
+
for char in normalized:
|
|
155
|
+
nfc_chars.append(char.translate(_TYPOGRAPHY_FOLD))
|
|
156
|
+
nfc_offsets.append(offsets[index])
|
|
157
|
+
index = end
|
|
158
|
+
|
|
159
|
+
collapsed = []
|
|
160
|
+
collapsed_offsets = []
|
|
161
|
+
in_whitespace = False
|
|
162
|
+
for char, original_offset in zip(nfc_chars, nfc_offsets):
|
|
163
|
+
if char.isspace():
|
|
164
|
+
if not in_whitespace:
|
|
165
|
+
collapsed.append(" ")
|
|
166
|
+
collapsed_offsets.append(original_offset)
|
|
167
|
+
in_whitespace = True
|
|
168
|
+
else:
|
|
169
|
+
collapsed.append(char)
|
|
170
|
+
collapsed_offsets.append(original_offset)
|
|
171
|
+
in_whitespace = False
|
|
172
|
+
return "".join(collapsed), collapsed_offsets
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def normalize_text(text: str, content_type: Optional[str] = "text/plain; charset=utf-8") -> str:
|
|
176
|
+
"""Apply the closed normalization corridor; never lowercase or paraphrase."""
|
|
177
|
+
if not isinstance(text, str):
|
|
178
|
+
raise TypeError(f"text must be str, got {type(text).__name__}")
|
|
179
|
+
mime, _ = _content_type_parts(content_type)
|
|
180
|
+
normalized, _ = _normalize_with_offsets(text, is_html=bool(mime and "html" in mime))
|
|
181
|
+
return normalized
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _record_value(record: Any, name: str, default: Any = None) -> Any:
|
|
185
|
+
if isinstance(record, Mapping):
|
|
186
|
+
return record.get(name, default)
|
|
187
|
+
return getattr(record, name, default)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def capture_excerpt(body: bytes, quote: str, record: Any) -> Dict[str, Any]:
|
|
191
|
+
"""Cut a bounded excerpt only after authentic FetchRecord/body hash agreement."""
|
|
192
|
+
if not getattr(record, "is_authentic", None) or not record.is_authentic():
|
|
193
|
+
raise ExcerptCaptureRefused(
|
|
194
|
+
"excerpt capture requires an authentic FetchRecord from evidence_fetch.fetch_source()"
|
|
195
|
+
)
|
|
196
|
+
if not isinstance(quote, str) or not quote.strip():
|
|
197
|
+
raise ExcerptCaptureRefused("quote must be a non-blank string")
|
|
198
|
+
raw = bytes(body) if isinstance(body, (bytes, bytearray, memoryview)) else body
|
|
199
|
+
if not isinstance(raw, bytes):
|
|
200
|
+
raise ExcerptCaptureRefused("source body must be bytes captured by the fetch path")
|
|
201
|
+
body_hash = hashlib.sha256(raw).hexdigest()
|
|
202
|
+
if body_hash != getattr(record, "sha256_body", None):
|
|
203
|
+
raise ExcerptCaptureRefused(
|
|
204
|
+
"hash-mismatch: source bytes do not match the FetchRecord.sha256_body witness"
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
text, mime, encoding = _decode_body(raw, getattr(record, "content_type", None))
|
|
208
|
+
normalized_body, offsets = _normalize_with_offsets(text, is_html="html" in mime)
|
|
209
|
+
normalized_quote = normalize_text(quote, getattr(record, "content_type", None))
|
|
210
|
+
if len(quote) > EXCERPT_MAX_CHARS or len(normalized_quote) > EXCERPT_MAX_CHARS:
|
|
211
|
+
raise ExcerptCaptureRefused(
|
|
212
|
+
f"excerpt too narrow: quote exceeds EXCERPT_MAX_CHARS={EXCERPT_MAX_CHARS}"
|
|
213
|
+
)
|
|
214
|
+
found = normalized_body.find(normalized_quote)
|
|
215
|
+
if found < 0:
|
|
216
|
+
raise ExcerptCaptureRefused(f"quote absent from captured source bytes: {quote!r}")
|
|
217
|
+
|
|
218
|
+
raw_start = offsets[found]
|
|
219
|
+
raw_end = offsets[found + len(normalized_quote) - 1] + 1
|
|
220
|
+
start = max(0, raw_start - EXCERPT_RADIUS_CHARS)
|
|
221
|
+
end = min(len(text), raw_end + EXCERPT_RADIUS_CHARS)
|
|
222
|
+
if end - start > EXCERPT_MAX_CHARS:
|
|
223
|
+
surplus = (end - start) - EXCERPT_MAX_CHARS
|
|
224
|
+
trim_left = min(surplus // 2, raw_start - start)
|
|
225
|
+
start += trim_left
|
|
226
|
+
surplus -= trim_left
|
|
227
|
+
end -= min(surplus, end - raw_end)
|
|
228
|
+
if end - start > EXCERPT_MAX_CHARS or start > raw_start or end < raw_end:
|
|
229
|
+
raise ExcerptCaptureRefused(
|
|
230
|
+
f"excerpt too narrow: quote cannot fit inside EXCERPT_MAX_CHARS={EXCERPT_MAX_CHARS}"
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
excerpt = text[start:end]
|
|
234
|
+
excerpt_hash = hashlib.sha256(excerpt.encode("utf-8")).hexdigest()
|
|
235
|
+
quote_hash = hashlib.sha256(quote.encode("utf-8")).hexdigest()
|
|
236
|
+
excerpt_id = f"{body_hash[:16]}-{quote_hash[:8]}.json"
|
|
237
|
+
try:
|
|
238
|
+
excerpt_offset = len(text[:start].encode(encoding, errors="strict"))
|
|
239
|
+
except (LookupError, UnicodeEncodeError) as exc:
|
|
240
|
+
raise ExcerptCaptureRefused(
|
|
241
|
+
f"cannot reproduce excerpt byte offset under declared charset {encoding!r}: {exc}"
|
|
242
|
+
) from None
|
|
243
|
+
return {
|
|
244
|
+
"excerpt_id": excerpt_id,
|
|
245
|
+
"sha256_body": body_hash,
|
|
246
|
+
"fetched_at": getattr(record, "fetched_at", None),
|
|
247
|
+
"source_url": getattr(record, "final_url", None),
|
|
248
|
+
"excerpt": excerpt,
|
|
249
|
+
"excerpt_offset": excerpt_offset,
|
|
250
|
+
"radius_chars": EXCERPT_RADIUS_CHARS,
|
|
251
|
+
"encoding": encoding,
|
|
252
|
+
"content_type": getattr(record, "content_type", None),
|
|
253
|
+
"sha256_excerpt": excerpt_hash,
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _atomic_write(path: str, text: str) -> None:
|
|
258
|
+
fd, temporary = tempfile.mkstemp(prefix=".quote-provenance-", dir=os.path.dirname(path), text=True)
|
|
259
|
+
try:
|
|
260
|
+
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
|
|
261
|
+
handle.write(text)
|
|
262
|
+
handle.flush()
|
|
263
|
+
os.fsync(handle.fileno())
|
|
264
|
+
os.replace(temporary, path)
|
|
265
|
+
except Exception:
|
|
266
|
+
try:
|
|
267
|
+
os.unlink(temporary)
|
|
268
|
+
except OSError:
|
|
269
|
+
pass
|
|
270
|
+
raise
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def write_excerpt(directory: str, excerpt_rec: Mapping[str, Any]) -> str:
|
|
274
|
+
"""Atomically write one excerpt and make its fresh store self-gitignoring."""
|
|
275
|
+
if not isinstance(excerpt_rec, Mapping):
|
|
276
|
+
raise ExcerptStoreError("excerpt record must be a mapping")
|
|
277
|
+
excerpt_id = excerpt_rec.get("excerpt_id")
|
|
278
|
+
if not isinstance(excerpt_id, str) or not excerpt_id or os.path.basename(excerpt_id) != excerpt_id:
|
|
279
|
+
raise ExcerptStoreError("excerpt_id must be a non-empty basename")
|
|
280
|
+
os.makedirs(directory, exist_ok=True)
|
|
281
|
+
ignore_path = os.path.join(directory, ".gitignore")
|
|
282
|
+
if not os.path.exists(ignore_path):
|
|
283
|
+
_atomic_write(ignore_path, "*\n")
|
|
284
|
+
target = os.path.join(directory, excerpt_id)
|
|
285
|
+
try:
|
|
286
|
+
payload = json.dumps(dict(excerpt_rec), ensure_ascii=False, sort_keys=True, indent=2) + "\n"
|
|
287
|
+
_atomic_write(target, payload)
|
|
288
|
+
except (OSError, TypeError, ValueError) as exc:
|
|
289
|
+
raise ExcerptStoreError(f"cannot write excerpt {excerpt_id!r}: {exc}") from None
|
|
290
|
+
return target
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def load_excerpt(directory: str, excerpt_id: Any) -> Optional[Dict[str, Any]]:
|
|
294
|
+
"""Load one store leaf. Missing material is distinct from an unreadable store."""
|
|
295
|
+
if not isinstance(excerpt_id, str) or not excerpt_id:
|
|
296
|
+
return None
|
|
297
|
+
if os.path.basename(excerpt_id) != excerpt_id:
|
|
298
|
+
raise ExcerptStoreError(f"unsafe excerpt_id {excerpt_id!r}")
|
|
299
|
+
path = os.path.join(directory, excerpt_id)
|
|
300
|
+
try:
|
|
301
|
+
with open(path, "r", encoding="utf-8") as handle:
|
|
302
|
+
data = json.load(handle)
|
|
303
|
+
except FileNotFoundError:
|
|
304
|
+
return None
|
|
305
|
+
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
306
|
+
raise ExcerptStoreError(f"cannot read excerpt {excerpt_id!r}: {exc}") from None
|
|
307
|
+
if not isinstance(data, dict):
|
|
308
|
+
raise ExcerptStoreError(f"excerpt {excerpt_id!r} is not a JSON object")
|
|
309
|
+
return data
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def verify_verbatim(span: str, excerpt_rec: Any, fact: Any) -> str:
|
|
313
|
+
"""Return one closed verdict; only an exact normalized leaf match can confirm."""
|
|
314
|
+
acquisition = read_acquisition(_record_value(fact, "acquisition"))
|
|
315
|
+
if acquisition == METHOD_UNKNOWN:
|
|
316
|
+
return METHOD_UNKNOWN
|
|
317
|
+
if acquisition == "tool-summary":
|
|
318
|
+
return "method-ineligible"
|
|
319
|
+
if isinstance(excerpt_rec, (list, tuple)):
|
|
320
|
+
return "not-in-excerpt" # a JOIN of leaves is never a source leaf
|
|
321
|
+
if not isinstance(excerpt_rec, Mapping) or not isinstance(excerpt_rec.get("excerpt"), str):
|
|
322
|
+
return "no-excerpt"
|
|
323
|
+
mime, _ = _content_type_parts(excerpt_rec.get("content_type"))
|
|
324
|
+
if not _is_decodable_mime(mime):
|
|
325
|
+
return "no-excerpt"
|
|
326
|
+
excerpt = excerpt_rec["excerpt"]
|
|
327
|
+
expected_excerpt_hash = excerpt_rec.get("sha256_excerpt")
|
|
328
|
+
actual_excerpt_hash = hashlib.sha256(excerpt.encode("utf-8")).hexdigest()
|
|
329
|
+
if not isinstance(expected_excerpt_hash, str) or expected_excerpt_hash != actual_excerpt_hash:
|
|
330
|
+
return "hash-mismatch"
|
|
331
|
+
fact_body_hash = _record_value(fact, "sha256_body")
|
|
332
|
+
if not isinstance(fact_body_hash, str) or fact_body_hash != excerpt_rec.get("sha256_body"):
|
|
333
|
+
return "hash-mismatch"
|
|
334
|
+
fact_url = _record_value(fact, "source_url")
|
|
335
|
+
if fact_url and excerpt_rec.get("source_url") and fact_url != excerpt_rec.get("source_url"):
|
|
336
|
+
return "hash-mismatch"
|
|
337
|
+
try:
|
|
338
|
+
normalized_span = normalize_text(span, excerpt_rec.get("content_type"))
|
|
339
|
+
normalized_excerpt = normalize_text(excerpt, excerpt_rec.get("content_type"))
|
|
340
|
+
except (TypeError, ValueError):
|
|
341
|
+
return "no-excerpt"
|
|
342
|
+
return "verbatim-confirmed" if normalized_span in normalized_excerpt else "not-in-excerpt"
|
|
@@ -14,6 +14,7 @@ test_*.py`, so a module that stops being collected fails BY NAME.
|
|
|
14
14
|
"""
|
|
15
15
|
|
|
16
16
|
import copy
|
|
17
|
+
import hashlib
|
|
17
18
|
import sys
|
|
18
19
|
|
|
19
20
|
# Import NOTHING local before this line: a stray __pycache__ inside this vendored
|
|
@@ -23,6 +24,8 @@ sys.dont_write_bytecode = True
|
|
|
23
24
|
import unittest
|
|
24
25
|
|
|
25
26
|
import ed25519_verifier as ev
|
|
27
|
+
import evidence_fetch as ef
|
|
28
|
+
import quote_provenance as qp
|
|
26
29
|
|
|
27
30
|
|
|
28
31
|
def _pop():
|
|
@@ -162,6 +165,63 @@ class Ed25519VerifierSecurityTests(unittest.TestCase):
|
|
|
162
165
|
self.assertFalse(reordered_ok)
|
|
163
166
|
self.assertIn("Invalid", reordered_error)
|
|
164
167
|
|
|
168
|
+
def test_schema_v4_signs_quote_provenance_and_tamper_fails(self):
|
|
169
|
+
signer = ev.Ed25519Verifier(auto_generate_keypair=True)
|
|
170
|
+
body = b"The source explicitly recommends gradual refeeding."
|
|
171
|
+
record = ef.FetchRecord(
|
|
172
|
+
url="https://example.test/source",
|
|
173
|
+
final_url="https://example.test/source",
|
|
174
|
+
status=200,
|
|
175
|
+
sha256_body=hashlib.sha256(body).hexdigest(),
|
|
176
|
+
bytes_len=len(body),
|
|
177
|
+
fetched_at="2026-09-02T00:00:00Z",
|
|
178
|
+
content_type="text/plain; charset=utf-8",
|
|
179
|
+
witness=ef._FETCH_WITNESS,
|
|
180
|
+
)
|
|
181
|
+
quote = qp.QuoteRecord(
|
|
182
|
+
quote="The source explicitly recommends gradual refeeding",
|
|
183
|
+
acquisition="raw-fetch",
|
|
184
|
+
source_url=record.final_url,
|
|
185
|
+
sha256_body=record.sha256_body,
|
|
186
|
+
locator="paragraph 1",
|
|
187
|
+
excerpt_id="quote.json",
|
|
188
|
+
)
|
|
189
|
+
fact = signer.create_fetched_fact(
|
|
190
|
+
claim=quote.quote,
|
|
191
|
+
fetch_record=record,
|
|
192
|
+
issuer="researcher",
|
|
193
|
+
study_population=_pop(),
|
|
194
|
+
quote=quote,
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
result = signer.verify_fact(fact)
|
|
198
|
+
self.assertTrue(result.verified, result.error)
|
|
199
|
+
self.assertEqual(ev.fact_schema_version(fact), ev.QUOTE_ATTESTED_MIN_SCHEMA)
|
|
200
|
+
self.assertIn("quote", result.signed_fields)
|
|
201
|
+
self.assertIn("acquisition", result.signed_fields)
|
|
202
|
+
self.assertIn("sha256_body", result.signed_fields)
|
|
203
|
+
|
|
204
|
+
tampered = copy.deepcopy(fact)
|
|
205
|
+
tampered.quote = "The source recommends immediate supplementation"
|
|
206
|
+
self.assertFalse(signer.verify_fact(tampered).verified)
|
|
207
|
+
|
|
208
|
+
def test_non_fetch_factory_cannot_claim_raw_fetch_acquisition(self):
|
|
209
|
+
signer = ev.Ed25519Verifier(auto_generate_keypair=True)
|
|
210
|
+
quote = qp.QuoteRecord(
|
|
211
|
+
quote="A raw quote",
|
|
212
|
+
acquisition="raw-fetch",
|
|
213
|
+
source_url="https://example.test/source",
|
|
214
|
+
sha256_body="a" * 64,
|
|
215
|
+
)
|
|
216
|
+
with self.assertRaisesRegex(ValueError, "requires create_fetched_fact"):
|
|
217
|
+
signer.create_listing_fact(
|
|
218
|
+
claim=quote.quote,
|
|
219
|
+
source_url=quote.source_url,
|
|
220
|
+
reason="listing only",
|
|
221
|
+
study_population=_pop(),
|
|
222
|
+
quote=quote,
|
|
223
|
+
)
|
|
224
|
+
|
|
165
225
|
|
|
166
226
|
if __name__ == "__main__":
|
|
167
227
|
unittest.main()
|
|
@@ -30,6 +30,7 @@ import ed25519_verifier as ev
|
|
|
30
30
|
import evidence_fetch as ef
|
|
31
31
|
import source_tiers as st
|
|
32
32
|
import check_report_evidence as gate
|
|
33
|
+
import quote_provenance as qp
|
|
33
34
|
|
|
34
35
|
|
|
35
36
|
def _verifier():
|
|
@@ -307,15 +308,63 @@ class FetchTests(unittest.TestCase):
|
|
|
307
308
|
def log_message(self, *args): # silence
|
|
308
309
|
pass
|
|
309
310
|
|
|
310
|
-
cls.
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
311
|
+
cls._original_build_opener = None
|
|
312
|
+
try:
|
|
313
|
+
cls.server = HTTPServer(("127.0.0.1", 0), Handler)
|
|
314
|
+
except PermissionError:
|
|
315
|
+
# Managed coding sandboxes can forbid even loopback sockets. Preserve
|
|
316
|
+
# every existing fetch assertion with a response-level fallback; on a
|
|
317
|
+
# normal host the real HTTPServer path above remains the path exercised.
|
|
318
|
+
cls.server = None
|
|
319
|
+
cls.port = 80
|
|
320
|
+
cls._original_build_opener = ef.urlrequest.build_opener
|
|
321
|
+
|
|
322
|
+
class Response:
|
|
323
|
+
def __init__(self, body, url, status=200, content_type="text/plain"):
|
|
324
|
+
self.body = body
|
|
325
|
+
self.url = url
|
|
326
|
+
self.status = status
|
|
327
|
+
self.headers = {"Content-Type": content_type}
|
|
328
|
+
|
|
329
|
+
def __enter__(self):
|
|
330
|
+
return self
|
|
331
|
+
|
|
332
|
+
def __exit__(self, *unused):
|
|
333
|
+
return False
|
|
334
|
+
|
|
335
|
+
def read(self, limit):
|
|
336
|
+
return self.body[:limit]
|
|
337
|
+
|
|
338
|
+
def getcode(self):
|
|
339
|
+
return self.status
|
|
340
|
+
|
|
341
|
+
def geturl(self):
|
|
342
|
+
return self.url
|
|
343
|
+
|
|
344
|
+
class Opener:
|
|
345
|
+
def open(self, request, timeout=None):
|
|
346
|
+
url = request.full_url
|
|
347
|
+
if url.endswith("/ok"):
|
|
348
|
+
return Response(b"hello evidence", url)
|
|
349
|
+
if url.endswith("/redirect"):
|
|
350
|
+
return Response(b"hello evidence", url[:-len("redirect")] + "ok")
|
|
351
|
+
if url.endswith("/big"):
|
|
352
|
+
return Response(b"x" * 4096, url)
|
|
353
|
+
raise ef.urlerror.HTTPError(url, 404, "Not Found", {}, None)
|
|
354
|
+
|
|
355
|
+
ef.urlrequest.build_opener = lambda *unused: Opener()
|
|
356
|
+
else:
|
|
357
|
+
cls.port = cls.server.server_address[1]
|
|
358
|
+
cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
|
|
359
|
+
cls.thread.start()
|
|
314
360
|
|
|
315
361
|
@classmethod
|
|
316
362
|
def tearDownClass(cls):
|
|
317
|
-
cls.server
|
|
318
|
-
|
|
363
|
+
if cls.server is not None:
|
|
364
|
+
cls.server.shutdown()
|
|
365
|
+
cls.server.server_close()
|
|
366
|
+
if cls._original_build_opener is not None:
|
|
367
|
+
ef.urlrequest.build_opener = cls._original_build_opener
|
|
319
368
|
|
|
320
369
|
def _url(self, path):
|
|
321
370
|
return f"http://127.0.0.1:{self.port}{path}"
|
|
@@ -328,6 +377,15 @@ class FetchTests(unittest.TestCase):
|
|
|
328
377
|
self.assertEqual(record.sha256_body, hashlib.sha256(b"hello evidence").hexdigest())
|
|
329
378
|
self.assertTrue(record.fetched_at.endswith("Z"))
|
|
330
379
|
|
|
380
|
+
def test_body_returning_seam_preserves_the_compatibility_record(self):
|
|
381
|
+
from unittest import mock
|
|
382
|
+
fixed_time = "2026-09-02T00:00:00Z"
|
|
383
|
+
with mock.patch.object(ef, "_now_iso", return_value=fixed_time):
|
|
384
|
+
seam_record, body = ef.fetch_source_returning_body(self._url("/ok"), _allow_private=True)
|
|
385
|
+
wrapper_record = ef.fetch_source(self._url("/ok"), _allow_private=True)
|
|
386
|
+
self.assertEqual(body, b"hello evidence")
|
|
387
|
+
self.assertEqual(seam_record.to_dict(), wrapper_record.to_dict())
|
|
388
|
+
|
|
331
389
|
def test_redirect_is_followed_and_final_url_recorded(self):
|
|
332
390
|
record = ef.fetch_source(self._url("/redirect"), _allow_private=True)
|
|
333
391
|
self.assertIsInstance(record, ef.FetchRecord)
|
|
@@ -447,6 +505,81 @@ class ReportGateTests(unittest.TestCase):
|
|
|
447
505
|
self.assertEqual(gate.main(["--report", report_path, "--facts", os.path.join(tmp, "nope.json")]), 2)
|
|
448
506
|
|
|
449
507
|
|
|
508
|
+
class QuoteProvenanceGateTests(unittest.TestCase):
|
|
509
|
+
"""ADR quote-provenance Confirmation: source bytes, never author status, decide."""
|
|
510
|
+
|
|
511
|
+
quote = "The source explicitly recommends gradual refeeding"
|
|
512
|
+
source_url = "https://example.test/source"
|
|
513
|
+
|
|
514
|
+
def _fixture(self, directory, excerpt_text, **fact_overrides):
|
|
515
|
+
body_hash = ev.hashlib.sha256(excerpt_text.encode("utf-8")).hexdigest()
|
|
516
|
+
excerpt_id = "quote.json"
|
|
517
|
+
excerpt = {
|
|
518
|
+
"sha256_body": body_hash,
|
|
519
|
+
"fetched_at": "2026-09-02T00:00:00Z",
|
|
520
|
+
"source_url": self.source_url,
|
|
521
|
+
"excerpt": excerpt_text,
|
|
522
|
+
"excerpt_offset": 0,
|
|
523
|
+
"radius_chars": 500,
|
|
524
|
+
"encoding": "utf-8",
|
|
525
|
+
"content_type": "text/plain; charset=utf-8",
|
|
526
|
+
"sha256_excerpt": ev.hashlib.sha256(excerpt_text.encode("utf-8")).hexdigest(),
|
|
527
|
+
}
|
|
528
|
+
with open(os.path.join(directory, excerpt_id), "w", encoding="utf-8") as handle:
|
|
529
|
+
json.dump(excerpt, handle)
|
|
530
|
+
fact = {
|
|
531
|
+
"claim": self.quote,
|
|
532
|
+
"source_url": self.source_url,
|
|
533
|
+
"evidence_class": "FETCH_VERIFIED",
|
|
534
|
+
"schema_version": 4,
|
|
535
|
+
"quote": self.quote,
|
|
536
|
+
"acquisition": "raw-fetch",
|
|
537
|
+
"sha256_body": body_hash,
|
|
538
|
+
"locator": "paragraph 1",
|
|
539
|
+
"excerpt_id": excerpt_id,
|
|
540
|
+
}
|
|
541
|
+
fact.update(fact_overrides)
|
|
542
|
+
return fact
|
|
543
|
+
|
|
544
|
+
def test_report_quote_absent_from_source_body_is_named_violation(self):
|
|
545
|
+
import tempfile
|
|
546
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
547
|
+
fact = self._fixture(tmp, "The source discusses measured recovery outcomes.")
|
|
548
|
+
findings, _ = gate.evaluate_quotes(f'The report says "{self.quote}".', [fact], tmp)
|
|
549
|
+
self.assertEqual([finding.kind for finding in findings], ["QUOTE_NOT_IN_SOURCE"])
|
|
550
|
+
self.assertIn(self.quote, findings[0].detail)
|
|
551
|
+
|
|
552
|
+
def test_report_quote_present_in_source_body_passes(self):
|
|
553
|
+
import tempfile
|
|
554
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
555
|
+
fact = self._fixture(tmp, f"Methods. {self.quote}. Results.")
|
|
556
|
+
findings, counts = gate.evaluate_quotes(f'The report says "{self.quote}".', [fact], tmp)
|
|
557
|
+
self.assertEqual(findings, [])
|
|
558
|
+
self.assertEqual(counts["quote-verbatim-confirmed"], 1)
|
|
559
|
+
|
|
560
|
+
def test_author_supplied_verbatim_status_is_ignored_and_downgraded(self):
|
|
561
|
+
import tempfile
|
|
562
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
563
|
+
fact = self._fixture(
|
|
564
|
+
tmp,
|
|
565
|
+
"The source discusses measured recovery outcomes.",
|
|
566
|
+
verbatim_status="verbatim",
|
|
567
|
+
)
|
|
568
|
+
findings, _ = gate.evaluate_quotes(f'The report says "{self.quote}".', [fact], tmp)
|
|
569
|
+
self.assertEqual([finding.kind for finding in findings], ["QUOTE_NOT_IN_SOURCE"])
|
|
570
|
+
self.assertIn("not-in-excerpt", findings[0].detail)
|
|
571
|
+
|
|
572
|
+
def test_unreadable_excerpt_store_never_reads_as_clean(self):
|
|
573
|
+
import tempfile
|
|
574
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
575
|
+
fact = self._fixture(tmp, self.quote)
|
|
576
|
+
with open(os.path.join(tmp, fact["excerpt_id"]), "w", encoding="utf-8") as handle:
|
|
577
|
+
handle.write("not json")
|
|
578
|
+
findings, counts = gate.evaluate_quotes(f'The report says "{self.quote}".', [fact], tmp)
|
|
579
|
+
self.assertEqual([finding.kind for finding in findings], ["QUOTES_UNCHECKED"])
|
|
580
|
+
self.assertEqual(counts["quote-unchecked"], 1)
|
|
581
|
+
|
|
582
|
+
|
|
450
583
|
|
|
451
584
|
|
|
452
585
|
class HardeningTests(unittest.TestCase):
|