@petercjl/taobao-ai-ops 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -22
- package/package.json +2 -7
- package/requirements.txt +1 -0
- package/skills/taobao-ai-operations/SKILL.md +8 -1
- package/src/cli.mjs +16 -6
- package/src/components.mjs +73 -0
- package/src/core.mjs +5 -1
- package/src/runtime.mjs +18 -16
- package/src/skill-manager.mjs +4 -4
- package/src/update.mjs +5 -5
- package/src/workflow.mjs +3 -3
- package/suite-manifest.json +186 -10
- package/skills/taobao-search-product-form/SKILL.md +0 -74
- package/skills/taobao-search-product-form/adapters/codex.json +0 -139
- package/skills/taobao-search-product-form/adapters/sealseek.json +0 -24
- package/skills/taobao-search-product-form/agents/openai.yaml +0 -3
- package/skills/taobao-search-product-form/capabilities.json +0 -148
- package/skills/taobao-search-product-form/references/SCHEMA.md +0 -3
- package/skills/taobao-search-product-form/references/index.md +0 -10
- package/skills/taobao-search-product-form/references/log.md +0 -35
- package/skills/taobao-search-product-form/references/queries/evaluate-white-label-forms.md +0 -3
- package/skills/taobao-search-product-form/references/queries/identify-advertisers.md +0 -5
- package/skills/taobao-search-product-form/references/queries/identify-geography.md +0 -5
- package/skills/taobao-search-product-form/references/queries/identify-price-bands.md +0 -3
- package/skills/taobao-search-product-form/references/queries/identify-product-forms.md +0 -3
- package/skills/taobao-search-product-form/references/queries/identify-title-roots.md +0 -5
- package/skills/taobao-search-product-form/references/report-output.md +0 -23
- package/skills/taobao-search-product-form/references/source-manifest.md +0 -37
- package/skills/taobao-search-product-form/references/title-lexicon.json +0 -55
- package/skills/taobao-search-product-form/references/topics/advertiser-analysis-method.md +0 -43
- package/skills/taobao-search-product-form/references/topics/geography-analysis-method.md +0 -23
- package/skills/taobao-search-product-form/references/topics/price-band-analysis-method.md +0 -27
- package/skills/taobao-search-product-form/references/topics/title-root-analysis-method.md +0 -37
- package/skills/taobao-search-product-form/references/topics/visual-form-method.md +0 -31
- package/skills/taobao-search-product-form/references/topics/white-label-evidence.md +0 -38
- package/skills/taobao-search-product-form/references/workflow-ledger.md +0 -28
- package/skills/taobao-search-product-form/report_runtime/VERSION +0 -1
- package/skills/taobao-search-product-form/report_runtime/renderer.py +0 -485
- package/skills/taobao-search-product-form/report_runtime/templates/workbench.html +0 -67
- package/skills/taobao-search-product-form/report_runtime/validator.py +0 -117
- package/skills/taobao-search-product-form/scripts/analyze.py +0 -119
- package/skills/taobao-search-product-form/scripts/analyze_advertisers.py +0 -352
- package/skills/taobao-search-product-form/scripts/analyze_geography.py +0 -316
- package/skills/taobao-search-product-form/scripts/analyze_price_bands.py +0 -257
- package/skills/taobao-search-product-form/scripts/analyze_title_roots.py +0 -327
- package/skills/taobao-search-product-form/scripts/assemble_opportunity_evidence.py +0 -322
- package/skills/taobao-search-product-form/scripts/build_report_viewmodel.py +0 -822
- package/skills/taobao-search-product-form/scripts/build_visual_review_viewmodel.py +0 -177
- package/skills/taobao-search-product-form/scripts/clean_search_export.mjs +0 -61
- package/skills/taobao-search-product-form/scripts/cluster_visual_forms.py +0 -193
- package/skills/taobao-search-product-form/scripts/compile_style_prototypes.py +0 -115
- package/skills/taobao-search-product-form/scripts/compile_visual_review.py +0 -60
- package/skills/taobao-search-product-form/scripts/compile_visual_selection.py +0 -148
- package/skills/taobao-search-product-form/scripts/prepare.py +0 -180
- package/skills/taobao-search-product-form/scripts/render_report.py +0 -31
- package/skills/taobao-search-product-form/scripts/validate_opportunity_cards.py +0 -84
- package/skills/taobao-search-product-form/scripts/validate_report_viewmodel.py +0 -152
- package/skills/taobao-search-product-form/scripts/visual_form_pipeline.py +0 -247
- package/skills/taobao-search-product-form/scripts/visual_observation_batches.py +0 -117
- package/skills/taobao-search-product-form/scripts/workflow_ledger.py +0 -177
- package/skills/yuce-category-opportunity-report/SKILL.md +0 -151
- package/skills/yuce-category-opportunity-report/adapters/codex.json +0 -113
- package/skills/yuce-category-opportunity-report/adapters/sealseek.json +0 -39
- package/skills/yuce-category-opportunity-report/agents/openai.yaml +0 -3
- package/skills/yuce-category-opportunity-report/assets/chart-views.js +0 -80
- package/skills/yuce-category-opportunity-report/assets/package.json +0 -1
- package/skills/yuce-category-opportunity-report/assets/report-template.html +0 -62
- package/skills/yuce-category-opportunity-report/capabilities.json +0 -48
- package/skills/yuce-category-opportunity-report/references/SCHEMA.md +0 -5
- package/skills/yuce-category-opportunity-report/references/data-contract.md +0 -40
- package/skills/yuce-category-opportunity-report/references/index.md +0 -6
- package/skills/yuce-category-opportunity-report/references/log.md +0 -29
- package/skills/yuce-category-opportunity-report/references/queries/assess-category-opportunity.md +0 -7
- package/skills/yuce-category-opportunity-report/references/source-manifest.md +0 -14
- package/skills/yuce-category-opportunity-report/references/topics/category-opportunity-method.md +0 -39
- package/skills/yuce-category-opportunity-report/scripts/build_report.py +0 -779
- package/skills/yuce-category-opportunity-report/scripts/market_routes.py +0 -139
|
@@ -1,247 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Plan one-pass visual jobs and reduce open visual observations.
|
|
3
|
-
|
|
4
|
-
Planning groups images for review; it never infers a product form. Reduction
|
|
5
|
-
uses an explicit, versioned registry, so labels can change without rereading.
|
|
6
|
-
"""
|
|
7
|
-
|
|
8
|
-
import argparse
|
|
9
|
-
import hashlib
|
|
10
|
-
import json
|
|
11
|
-
from collections import Counter, defaultdict
|
|
12
|
-
from pathlib import Path
|
|
13
|
-
|
|
14
|
-
from PIL import Image, ImageDraw, ImageOps
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def digest(path):
|
|
18
|
-
value = hashlib.sha256()
|
|
19
|
-
with path.open("rb") as stream:
|
|
20
|
-
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
|
21
|
-
value.update(block)
|
|
22
|
-
return value.hexdigest()
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
def dhash(path):
|
|
26
|
-
with Image.open(path) as source:
|
|
27
|
-
image = ImageOps.exif_transpose(source).convert("L").resize((9, 8))
|
|
28
|
-
pixels = list(image.tobytes())
|
|
29
|
-
value = 0
|
|
30
|
-
for row in range(8):
|
|
31
|
-
for col in range(8):
|
|
32
|
-
value = (value << 1) | int(pixels[row * 9 + col] > pixels[row * 9 + col + 1])
|
|
33
|
-
return value
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def read_json(path):
|
|
37
|
-
return json.loads(Path(path).read_text(encoding="utf-8"))
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def write_json(path, value):
|
|
41
|
-
with path.open("x", encoding="utf-8") as stream:
|
|
42
|
-
json.dump(value, stream, ensure_ascii=False, indent=2)
|
|
43
|
-
stream.write("\n")
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def plan(args):
|
|
47
|
-
manifest_path = Path(args.manifest).resolve()
|
|
48
|
-
manifest = read_json(manifest_path)
|
|
49
|
-
source_root = manifest_path.parent
|
|
50
|
-
if args.population != "natural":
|
|
51
|
-
raise ValueError("only the natural-position form population is currently implemented")
|
|
52
|
-
ids = manifest["selected_organic_ids"]
|
|
53
|
-
if args.limit:
|
|
54
|
-
ids = ids[:args.limit]
|
|
55
|
-
lookup = {str(p["product_id"]): p for p in manifest["products"]}
|
|
56
|
-
items = []
|
|
57
|
-
for pid in ids:
|
|
58
|
-
fetched = manifest["fetches"].get(pid, {})
|
|
59
|
-
if fetched.get("status") != "ok":
|
|
60
|
-
items.append({"product_id": pid, "status": "image_unavailable", "image_path": None,
|
|
61
|
-
"unavailable_reason": fetched.get("status") or "missing_fetch_record"})
|
|
62
|
-
continue
|
|
63
|
-
path = (source_root / fetched["path"]).resolve()
|
|
64
|
-
if not path.is_file() or not path.is_relative_to(source_root):
|
|
65
|
-
raise ValueError(f"missing or escaping image path for product {pid}")
|
|
66
|
-
items.append({"product_id": pid, "status": "ready", "image_path": str(path),
|
|
67
|
-
"source_image_url": lookup[pid].get("image_url"),
|
|
68
|
-
"image_sha256": digest(path), "image_dhash": f"{dhash(path):016x}",
|
|
69
|
-
"source_rank": lookup[pid]["organic_rank"]})
|
|
70
|
-
if len(ids) != len(set(ids)):
|
|
71
|
-
raise ValueError("duplicate selected natural-position product ID")
|
|
72
|
-
out = Path(args.output_dir).resolve()
|
|
73
|
-
if out.exists():
|
|
74
|
-
raise FileExistsError(out)
|
|
75
|
-
out.mkdir(parents=True)
|
|
76
|
-
ready = [item for item in items if item["status"] == "ready"]
|
|
77
|
-
# dHash is a cheap grouping hint only. It is never a classification label.
|
|
78
|
-
ready.sort(key=lambda item: (item["image_dhash"], item["product_id"]))
|
|
79
|
-
batches = []
|
|
80
|
-
for start in range(0, len(ready), args.batch_size):
|
|
81
|
-
members = ready[start:start + args.batch_size]
|
|
82
|
-
batch_id = f"B{len(batches) + 1:03d}"
|
|
83
|
-
for item in members:
|
|
84
|
-
item["batch_id"] = batch_id
|
|
85
|
-
batches.append({"batch_id": batch_id, "product_ids": [item["product_id"] for item in members]})
|
|
86
|
-
columns, cell_w, cell_h = 3, 350, 385
|
|
87
|
-
rows = (len(members) + columns - 1) // columns
|
|
88
|
-
sheet = Image.new("RGB", (columns * cell_w, rows * cell_h), "white")
|
|
89
|
-
draw = ImageDraw.Draw(sheet)
|
|
90
|
-
for n, item in enumerate(members):
|
|
91
|
-
with Image.open(item["image_path"]) as source:
|
|
92
|
-
image = ImageOps.exif_transpose(source).convert("RGB")
|
|
93
|
-
image.thumbnail((330, 345))
|
|
94
|
-
x, y = (n % columns) * cell_w, (n // columns) * cell_h
|
|
95
|
-
sheet.paste(image, (x + (cell_w - image.width) // 2, y + 4))
|
|
96
|
-
draw.text((x + 8, y + 354), f'{item["product_id"]} N{item["source_rank"]}', fill="black")
|
|
97
|
-
sheet_name = f"{batch_id}.jpg"
|
|
98
|
-
sheet.save(out / sheet_name, "JPEG", quality=90)
|
|
99
|
-
batches[-1]["contact_sheet"] = str(out / sheet_name)
|
|
100
|
-
plan_doc = {"schema_version": 1, "strategy": "visual-form-open-v1",
|
|
101
|
-
"semantic_inspection_unit": "original_product_image",
|
|
102
|
-
"contact_sheet_role": "orientation_and_coverage_audit",
|
|
103
|
-
"source_manifest": str(manifest_path), "source_manifest_sha256": digest(manifest_path),
|
|
104
|
-
"population": "natural", "scope": {"selected": len(ids), "ready": len(ready),
|
|
105
|
-
"image_unavailable": len(ids) - len(ready), "manifest_natural_total": manifest["scope"]["organic_total"]},
|
|
106
|
-
"batch_size": args.batch_size, "items": items, "batches": batches}
|
|
107
|
-
write_json(out / "visual_batch_plan.json", plan_doc)
|
|
108
|
-
print(json.dumps({"plan": str(out / "visual_batch_plan.json"), "selected": len(ids),
|
|
109
|
-
"ready": len(ready), "batches": len(batches)}, ensure_ascii=False))
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
def bind(args):
|
|
113
|
-
plan_doc = read_json(args.plan)
|
|
114
|
-
draft = read_json(args.draft)
|
|
115
|
-
ready = {item["product_id"]: item for item in plan_doc["items"] if item["status"] == "ready"}
|
|
116
|
-
observations = draft["observations"]
|
|
117
|
-
ids = [str(row["product_id"]) for row in observations]
|
|
118
|
-
if len(ids) != len(set(ids)) or set(ids) != set(ready):
|
|
119
|
-
raise ValueError("draft must cover each ready product exactly once")
|
|
120
|
-
bound = []
|
|
121
|
-
for row in observations:
|
|
122
|
-
item = ready[str(row["product_id"])]
|
|
123
|
-
bound.append({**row, "batch_id": item["batch_id"], "image_sha256": item["image_sha256"]})
|
|
124
|
-
output = Path(args.output).resolve()
|
|
125
|
-
write_json(output, {"schema_version": 1, "plan_sha256": digest(Path(args.plan)),
|
|
126
|
-
"observations": bound})
|
|
127
|
-
print(json.dumps({"observations": str(output), "count": len(bound)}, ensure_ascii=False))
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
def reduce(args):
|
|
131
|
-
plan_doc = read_json(args.plan)
|
|
132
|
-
raw = read_json(args.observations)
|
|
133
|
-
registry = read_json(args.registry)
|
|
134
|
-
if raw.get("schema_version") != 1 or registry.get("schema_version") != 1:
|
|
135
|
-
raise ValueError("unsupported observations or registry schema")
|
|
136
|
-
if raw.get("plan_sha256") != digest(Path(args.plan)):
|
|
137
|
-
raise ValueError("observations were bound to a different plan")
|
|
138
|
-
expected = {item["product_id"]: item for item in plan_doc["items"] if item["status"] == "ready"}
|
|
139
|
-
records = raw["observations"]
|
|
140
|
-
ids = [str(record["product_id"]) for record in records]
|
|
141
|
-
if len(ids) != len(set(ids)):
|
|
142
|
-
raise ValueError("a product has more than one semantic observation")
|
|
143
|
-
if set(ids) != set(expected):
|
|
144
|
-
raise ValueError(f"observation coverage mismatch: missing={sorted(set(expected)-set(ids))}, extra={sorted(set(ids)-set(expected))}")
|
|
145
|
-
forms = registry.get("primary_forms", {})
|
|
146
|
-
alias_owner = {}
|
|
147
|
-
for canonical, entry in forms.items():
|
|
148
|
-
for alias in [canonical, *entry.get("aliases", [])]:
|
|
149
|
-
key = alias.strip().casefold()
|
|
150
|
-
if key in alias_owner and alias_owner[key] != canonical:
|
|
151
|
-
raise ValueError(f"alias collision: {alias}")
|
|
152
|
-
alias_owner[key] = canonical
|
|
153
|
-
result = []
|
|
154
|
-
unresolved = Counter()
|
|
155
|
-
uncertain_candidates = Counter()
|
|
156
|
-
candidates = Counter()
|
|
157
|
-
batches = defaultdict(Counter)
|
|
158
|
-
for record in records:
|
|
159
|
-
pid = str(record["product_id"])
|
|
160
|
-
item = expected[pid]
|
|
161
|
-
if record.get("image_sha256") != item["image_sha256"] or record.get("batch_id") != item["batch_id"]:
|
|
162
|
-
raise ValueError(f"image/batch provenance mismatch: {pid}")
|
|
163
|
-
status = record.get("status")
|
|
164
|
-
if status not in {"tagged", "uncertain"}:
|
|
165
|
-
raise ValueError(f"invalid observation status: {pid}")
|
|
166
|
-
evidence = str(record.get("visible_evidence", "")).strip()
|
|
167
|
-
if not evidence:
|
|
168
|
-
raise ValueError(f"missing visible evidence: {pid}")
|
|
169
|
-
candidate = str(record.get("primary_form_candidate", "")).strip()
|
|
170
|
-
if status == "tagged" and not candidate:
|
|
171
|
-
raise ValueError(f"missing form candidate: {pid}")
|
|
172
|
-
facets = record.get("facets", {})
|
|
173
|
-
if not isinstance(facets, dict):
|
|
174
|
-
raise ValueError(f"facets must be an object: {pid}")
|
|
175
|
-
canonical = alias_owner.get(candidate.casefold()) if candidate else None
|
|
176
|
-
if candidate:
|
|
177
|
-
candidates[candidate] += 1
|
|
178
|
-
if status == "tagged" and canonical is None:
|
|
179
|
-
unresolved[candidate] += 1
|
|
180
|
-
if status == "uncertain" and candidate and canonical is None:
|
|
181
|
-
uncertain_candidates[candidate] += 1
|
|
182
|
-
if canonical:
|
|
183
|
-
batches[item["batch_id"]][canonical] += 1
|
|
184
|
-
result.append({"product_id": pid, "batch_id": item["batch_id"],
|
|
185
|
-
"image_sha256": item["image_sha256"], "status": status,
|
|
186
|
-
"primary_form_candidate": candidate or None,
|
|
187
|
-
"primary_form": canonical, "visible_evidence": evidence,
|
|
188
|
-
"facets": facets})
|
|
189
|
-
out = Path(args.output_dir).resolve()
|
|
190
|
-
if out.exists():
|
|
191
|
-
raise FileExistsError(out)
|
|
192
|
-
out.mkdir(parents=True)
|
|
193
|
-
counts = Counter(row["primary_form"] for row in result if row["primary_form"])
|
|
194
|
-
report = {"schema_version": 1, "strategy": "visual-form-open-v1",
|
|
195
|
-
"plan_sha256": digest(Path(args.plan)), "observations_sha256": digest(Path(args.observations)),
|
|
196
|
-
"registry_sha256": digest(Path(args.registry)), "registry_version": registry.get("version"),
|
|
197
|
-
"scope": plan_doc["scope"], "observed": len(result),
|
|
198
|
-
"candidate_labels": dict(candidates), "candidate_label_count": len(candidates),
|
|
199
|
-
"canonical_forms": dict(counts), "canonical_form_count": len(counts),
|
|
200
|
-
"representative_product_ids": {form: [row["product_id"] for row in result
|
|
201
|
-
if row["primary_form"] == form][:3]
|
|
202
|
-
for form in counts},
|
|
203
|
-
"singleton_forms": [form for form, count in counts.items() if count == 1],
|
|
204
|
-
"unresolved_candidates": dict(unresolved),
|
|
205
|
-
"uncertain_candidates": dict(uncertain_candidates),
|
|
206
|
-
"uncertain_count": sum(row["status"] == "uncertain" for row in result),
|
|
207
|
-
"batch_form_counts": {key: dict(value) for key, value in batches.items()},
|
|
208
|
-
"qa": {"one_record_per_ready_product": True, "image_hash_and_batch_match": True,
|
|
209
|
-
"all_labels_resolved": not unresolved}}
|
|
210
|
-
write_json(out / "visual_form_assignments.json", {"schema_version": 1, "records": result})
|
|
211
|
-
write_json(out / "visual_form_summary.json", report)
|
|
212
|
-
print(json.dumps({"output_dir": str(out), "observed": len(result),
|
|
213
|
-
"candidate_labels": len(candidates), "canonical_forms": len(counts),
|
|
214
|
-
"unresolved": dict(unresolved)}, ensure_ascii=False))
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
def main():
|
|
218
|
-
parser = argparse.ArgumentParser()
|
|
219
|
-
commands = parser.add_subparsers(dest="command", required=True)
|
|
220
|
-
planning = commands.add_parser("plan")
|
|
221
|
-
planning.add_argument("--manifest", required=True)
|
|
222
|
-
planning.add_argument("--output-dir", required=True)
|
|
223
|
-
planning.add_argument("--population", default="natural")
|
|
224
|
-
planning.add_argument("--limit", type=int)
|
|
225
|
-
planning.add_argument("--batch-size", type=int, default=10)
|
|
226
|
-
binding = commands.add_parser("bind")
|
|
227
|
-
binding.add_argument("--plan", required=True)
|
|
228
|
-
binding.add_argument("--draft", required=True)
|
|
229
|
-
binding.add_argument("--output", required=True)
|
|
230
|
-
reducing = commands.add_parser("reduce")
|
|
231
|
-
reducing.add_argument("--plan", required=True)
|
|
232
|
-
reducing.add_argument("--observations", required=True)
|
|
233
|
-
reducing.add_argument("--registry", required=True)
|
|
234
|
-
reducing.add_argument("--output-dir", required=True)
|
|
235
|
-
args = parser.parse_args()
|
|
236
|
-
if args.command == "plan":
|
|
237
|
-
if args.batch_size < 1 or args.limit is not None and args.limit < 1:
|
|
238
|
-
parser.error("batch size and limit must be positive")
|
|
239
|
-
plan(args)
|
|
240
|
-
elif args.command == "bind":
|
|
241
|
-
bind(args)
|
|
242
|
-
else:
|
|
243
|
-
reduce(args)
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
if __name__ == "__main__":
|
|
247
|
-
main()
|
|
@@ -1,117 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Checkpoint image-grounded observations one visual batch at a time.
|
|
3
|
-
|
|
4
|
-
This tool validates membership and provenance. It never generates semantic labels.
|
|
5
|
-
"""
|
|
6
|
-
|
|
7
|
-
import argparse
|
|
8
|
-
import hashlib
|
|
9
|
-
import json
|
|
10
|
-
from pathlib import Path
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
def read(path):
|
|
14
|
-
return json.loads(Path(path).read_text(encoding="utf-8"))
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def digest(path):
|
|
18
|
-
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def ready_batches(plan):
|
|
22
|
-
items = {str(item["product_id"]): item for item in plan["items"] if item["status"] == "ready"}
|
|
23
|
-
batches = {batch["batch_id"]: batch for batch in plan["batches"]}
|
|
24
|
-
if len(items) != plan["scope"]["ready"]:
|
|
25
|
-
raise ValueError("ready-image count mismatch")
|
|
26
|
-
return items, batches
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
def validate_batch(plan_path, batch_id, doc):
|
|
30
|
-
plan = read(plan_path)
|
|
31
|
-
items, batches = ready_batches(plan)
|
|
32
|
-
if batch_id not in batches:
|
|
33
|
-
raise ValueError(f"unknown batch: {batch_id}")
|
|
34
|
-
expected = set(batches[batch_id]["product_ids"])
|
|
35
|
-
rows = doc.get("observations")
|
|
36
|
-
if not isinstance(rows, list):
|
|
37
|
-
raise ValueError("observations must be an array")
|
|
38
|
-
ids = [str(row.get("product_id", "")) for row in rows]
|
|
39
|
-
if len(ids) != len(set(ids)) or set(ids) != expected:
|
|
40
|
-
raise ValueError(f"batch coverage mismatch: missing={sorted(expected-set(ids))}, extra={sorted(set(ids)-expected)}")
|
|
41
|
-
bound = []
|
|
42
|
-
for row in rows:
|
|
43
|
-
item = items[str(row["product_id"])]
|
|
44
|
-
if item["batch_id"] != batch_id:
|
|
45
|
-
raise ValueError(f"batch membership mismatch: {row['product_id']}")
|
|
46
|
-
if row.get("batch_id", batch_id) != batch_id or row.get("image_sha256", item["image_sha256"]) != item["image_sha256"]:
|
|
47
|
-
raise ValueError(f"image provenance mismatch: {row['product_id']}")
|
|
48
|
-
if row.get("status") not in ("tagged", "uncertain") or not str(row.get("visible_evidence", "")).strip():
|
|
49
|
-
raise ValueError(f"missing status or visible evidence: {row['product_id']}")
|
|
50
|
-
if any(not isinstance(row.get(axis), dict) for axis in ("form", "style", "audience")):
|
|
51
|
-
raise ValueError(f"three axes required: {row['product_id']}")
|
|
52
|
-
bound.append({**row, "product_id": str(row["product_id"]), "batch_id": batch_id,
|
|
53
|
-
"image_sha256": item["image_sha256"]})
|
|
54
|
-
return {"schema_version": 2, "batch_id": batch_id, "plan_sha256": digest(plan_path),
|
|
55
|
-
"observations": bound}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def checked_files(plan_path, batch_dir):
|
|
59
|
-
plan = read(plan_path)
|
|
60
|
-
_, batches = ready_batches(plan)
|
|
61
|
-
folder = Path(batch_dir)
|
|
62
|
-
found = {}
|
|
63
|
-
if folder.exists():
|
|
64
|
-
for path in sorted(folder.glob("B*.json")):
|
|
65
|
-
batch_id = path.stem
|
|
66
|
-
if batch_id not in batches:
|
|
67
|
-
raise ValueError(f"unexpected batch file: {path}")
|
|
68
|
-
doc = read(path)
|
|
69
|
-
if doc.get("plan_sha256") != digest(plan_path) or doc.get("batch_id") != batch_id:
|
|
70
|
-
raise ValueError(f"stale or mismatched batch file: {path}")
|
|
71
|
-
validated = validate_batch(plan_path, batch_id, doc)
|
|
72
|
-
found[batch_id] = validated["observations"]
|
|
73
|
-
return batches, found
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
def main():
|
|
77
|
-
parser = argparse.ArgumentParser(description=__doc__)
|
|
78
|
-
commands = parser.add_subparsers(dest="command", required=True)
|
|
79
|
-
for name in ("commit", "status", "merge"):
|
|
80
|
-
command = commands.add_parser(name)
|
|
81
|
-
command.add_argument("--plan", required=True)
|
|
82
|
-
command.add_argument("--batch-dir", required=True)
|
|
83
|
-
if name == "commit":
|
|
84
|
-
command.add_argument("--batch-id", required=True)
|
|
85
|
-
command.add_argument("--draft", required=True)
|
|
86
|
-
if name == "merge":
|
|
87
|
-
command.add_argument("--output", required=True)
|
|
88
|
-
args = parser.parse_args()
|
|
89
|
-
plan_path = Path(args.plan).resolve()
|
|
90
|
-
batch_dir = Path(args.batch_dir).resolve()
|
|
91
|
-
batches, found = checked_files(plan_path, batch_dir)
|
|
92
|
-
if args.command == "commit":
|
|
93
|
-
if args.batch_id in found:
|
|
94
|
-
raise FileExistsError(batch_dir / f"{args.batch_id}.json")
|
|
95
|
-
validated = validate_batch(plan_path, args.batch_id, read(args.draft))
|
|
96
|
-
batch_dir.mkdir(parents=True, exist_ok=True)
|
|
97
|
-
path = batch_dir / f"{args.batch_id}.json"
|
|
98
|
-
with path.open("x", encoding="utf-8") as stream:
|
|
99
|
-
json.dump(validated, stream, ensure_ascii=False, indent=2)
|
|
100
|
-
stream.write("\n")
|
|
101
|
-
found[args.batch_id] = validated["observations"]
|
|
102
|
-
missing = [batch_id for batch_id in batches if batch_id not in found]
|
|
103
|
-
if args.command == "merge":
|
|
104
|
-
if missing:
|
|
105
|
-
raise ValueError(f"incomplete visual observations: {len(missing)} batches remain; next={missing[0]}")
|
|
106
|
-
rows = [row for batch_id in batches for row in found[batch_id]]
|
|
107
|
-
output = Path(args.output).resolve()
|
|
108
|
-
with output.open("x", encoding="utf-8") as stream:
|
|
109
|
-
json.dump({"schema_version": 2, "observations": rows}, stream, ensure_ascii=False, indent=2)
|
|
110
|
-
stream.write("\n")
|
|
111
|
-
print(json.dumps({"completed_batches": len(found), "total_batches": len(batches),
|
|
112
|
-
"observed_products": sum(len(rows) for rows in found.values()),
|
|
113
|
-
"next_batch": missing[0] if missing else None}, ensure_ascii=False))
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
if __name__ == "__main__":
|
|
117
|
-
main()
|
|
@@ -1,177 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Small, recoverable stage ledger for development and end-to-end runs."""
|
|
3
|
-
|
|
4
|
-
import argparse
|
|
5
|
-
import hashlib
|
|
6
|
-
import json
|
|
7
|
-
import os
|
|
8
|
-
import shutil
|
|
9
|
-
import tempfile
|
|
10
|
-
import uuid
|
|
11
|
-
from datetime import datetime, timezone
|
|
12
|
-
from pathlib import Path
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
STAGES = ("input", "clean", "classify", "compare", "decide", "report")
|
|
16
|
-
STATUSES = ("not_started", "in_progress", "awaiting_review", "approved", "verified", "blocked", "skipped")
|
|
17
|
-
DESIGN_STATES = ("open", "defined", "validated")
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def now():
|
|
21
|
-
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def sha256(path):
|
|
25
|
-
digest = hashlib.sha256()
|
|
26
|
-
with path.open("rb") as handle:
|
|
27
|
-
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
28
|
-
digest.update(chunk)
|
|
29
|
-
return digest.hexdigest()
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
def stage_record():
|
|
33
|
-
return {"design_state": "open", "status": "not_started", "strategy": None,
|
|
34
|
-
"artifacts": [], "review": None, "note": None, "updated_at": None}
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
def artifact_intact(item):
|
|
38
|
-
path = Path(item["path"])
|
|
39
|
-
return path.is_file() and path.stat().st_size == item["bytes"] and sha256(path) == item["sha256"]
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
def command_init(args):
|
|
43
|
-
source = Path(args.input).expanduser().resolve()
|
|
44
|
-
if not source.is_file():
|
|
45
|
-
raise ValueError(f"input workbook not found: {source}")
|
|
46
|
-
run_dir = Path(args.run_dir).expanduser().resolve()
|
|
47
|
-
ledger = run_dir / "ledger.json"
|
|
48
|
-
if ledger.exists() or ledger.is_symlink():
|
|
49
|
-
raise FileExistsError(f"ledger already exists: {ledger}")
|
|
50
|
-
run_dir.mkdir(parents=True, exist_ok=True)
|
|
51
|
-
timestamp = now()
|
|
52
|
-
document = {
|
|
53
|
-
"schema_version": 1,
|
|
54
|
-
"mode": args.mode,
|
|
55
|
-
"created_at": timestamp,
|
|
56
|
-
"updated_at": timestamp,
|
|
57
|
-
"input": {"path": str(source), "sha256": sha256(source), "bytes": source.stat().st_size},
|
|
58
|
-
"stages": {name: stage_record() for name in STAGES},
|
|
59
|
-
"events": [{"at": timestamp, "action": "init", "note": "Run ledger initialized; stages await design or execution."}],
|
|
60
|
-
}
|
|
61
|
-
with ledger.open("x", encoding="utf-8") as handle:
|
|
62
|
-
json.dump(document, handle, ensure_ascii=False, indent=2)
|
|
63
|
-
handle.write("\n")
|
|
64
|
-
print(json.dumps({"ledger": str(ledger), "mode": args.mode, "stages": list(STAGES)}, ensure_ascii=False))
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
def command_show(args):
|
|
68
|
-
ledger = Path(args.ledger).expanduser().resolve()
|
|
69
|
-
document = json.loads(ledger.read_text(encoding="utf-8"))
|
|
70
|
-
input_path = Path(document["input"]["path"])
|
|
71
|
-
input_intact = input_path.is_file() and sha256(input_path) == document["input"]["sha256"]
|
|
72
|
-
result = {"ledger": str(ledger), "mode": document["mode"], "input": document["input"],
|
|
73
|
-
"input_intact": input_intact,
|
|
74
|
-
"stages": {key: {"design_state": value["design_state"], "status": value["status"],
|
|
75
|
-
"strategy": value["strategy"], "artifacts": value["artifacts"],
|
|
76
|
-
"artifacts_intact": all(artifact_intact(item) for item in value["artifacts"]),
|
|
77
|
-
"note": value["note"]} for key, value in document["stages"].items()}}
|
|
78
|
-
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
def command_record(args):
|
|
82
|
-
ledger = Path(args.ledger).expanduser().resolve()
|
|
83
|
-
if ledger.is_symlink() or not ledger.is_file():
|
|
84
|
-
raise ValueError(f"ledger must be an existing regular file: {ledger}")
|
|
85
|
-
document = json.loads(ledger.read_text(encoding="utf-8"))
|
|
86
|
-
stage_id = args.stage
|
|
87
|
-
if stage_id not in document["stages"]:
|
|
88
|
-
if not stage_id.startswith("classify/") or len(stage_id) <= len("classify/"):
|
|
89
|
-
raise ValueError(f"unknown stage: {stage_id}; use classify/NAME for a module")
|
|
90
|
-
document["stages"][stage_id] = stage_record()
|
|
91
|
-
stage = document["stages"][stage_id]
|
|
92
|
-
if args.status == "approved" and (document["mode"] != "development" or not args.reviewer or not args.note):
|
|
93
|
-
raise ValueError("approved requires development mode, --reviewer and --note recording confirmation")
|
|
94
|
-
if args.status == "verified" and document["mode"] != "run":
|
|
95
|
-
raise ValueError("verified is reserved for run-mode automated QA")
|
|
96
|
-
if args.status == "awaiting_review" and document["mode"] != "development":
|
|
97
|
-
raise ValueError("awaiting_review is reserved for development mode")
|
|
98
|
-
if args.status == "blocked" and not args.note:
|
|
99
|
-
raise ValueError("blocked requires --note")
|
|
100
|
-
artifacts = []
|
|
101
|
-
for raw in args.artifact:
|
|
102
|
-
target = Path(raw).expanduser().resolve()
|
|
103
|
-
if not target.is_file():
|
|
104
|
-
raise ValueError(f"artifact does not exist as a file: {target}")
|
|
105
|
-
artifacts.append({"path": str(target), "sha256": sha256(target), "bytes": target.stat().st_size})
|
|
106
|
-
if args.status is not None:
|
|
107
|
-
stage["status"] = args.status
|
|
108
|
-
if args.design_state is not None:
|
|
109
|
-
stage["design_state"] = args.design_state
|
|
110
|
-
if args.strategy is not None:
|
|
111
|
-
stage["strategy"] = args.strategy
|
|
112
|
-
for artifact in artifacts:
|
|
113
|
-
stage["artifacts"] = [old for old in stage["artifacts"] if old["path"] != artifact["path"]]
|
|
114
|
-
stage["artifacts"].append(artifact)
|
|
115
|
-
if args.note is not None:
|
|
116
|
-
stage["note"] = args.note
|
|
117
|
-
if args.status == "approved":
|
|
118
|
-
stage["review"] = {"reviewer": args.reviewer, "at": now(), "note": args.note}
|
|
119
|
-
elif args.status in ("not_started", "in_progress", "awaiting_review", "blocked"):
|
|
120
|
-
stage["review"] = None
|
|
121
|
-
timestamp = now()
|
|
122
|
-
stage["updated_at"] = timestamp
|
|
123
|
-
document["updated_at"] = timestamp
|
|
124
|
-
document["events"].append({"at": timestamp, "action": "record", "stage": stage_id,
|
|
125
|
-
"status": stage["status"], "design_state": stage["design_state"],
|
|
126
|
-
"strategy": stage["strategy"], "artifacts_added": artifacts,
|
|
127
|
-
"note": args.note, "reviewer": args.reviewer})
|
|
128
|
-
backup = ledger.with_name(ledger.name + ".bak-" + uuid.uuid4().hex[:12])
|
|
129
|
-
shutil.copy2(ledger, backup)
|
|
130
|
-
if not backup.is_file() or backup.stat().st_size == 0:
|
|
131
|
-
raise OSError("ledger backup verification failed")
|
|
132
|
-
fd, temporary = tempfile.mkstemp(prefix=".ledger-", suffix=".json", dir=ledger.parent)
|
|
133
|
-
try:
|
|
134
|
-
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
|
135
|
-
json.dump(document, handle, ensure_ascii=False, indent=2)
|
|
136
|
-
handle.write("\n")
|
|
137
|
-
handle.flush()
|
|
138
|
-
os.fsync(handle.fileno())
|
|
139
|
-
os.chmod(temporary, ledger.stat().st_mode & 0o777)
|
|
140
|
-
os.replace(temporary, ledger)
|
|
141
|
-
finally:
|
|
142
|
-
if os.path.exists(temporary):
|
|
143
|
-
os.unlink(temporary)
|
|
144
|
-
print(json.dumps({"ledger": str(ledger), "backup": str(backup), "stage": stage_id,
|
|
145
|
-
"status": stage["status"], "artifacts": stage["artifacts"]}, ensure_ascii=False))
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
def main():
|
|
149
|
-
parser = argparse.ArgumentParser(description=__doc__)
|
|
150
|
-
commands = parser.add_subparsers(dest="command", required=True)
|
|
151
|
-
init = commands.add_parser("init")
|
|
152
|
-
init.add_argument("--run-dir", required=True)
|
|
153
|
-
init.add_argument("--mode", choices=("development", "run"), required=True)
|
|
154
|
-
init.add_argument("--input", required=True)
|
|
155
|
-
init.set_defaults(func=command_init)
|
|
156
|
-
show = commands.add_parser("show")
|
|
157
|
-
show.add_argument("--ledger", required=True)
|
|
158
|
-
show.set_defaults(func=command_show)
|
|
159
|
-
record = commands.add_parser("record")
|
|
160
|
-
record.add_argument("--ledger", required=True)
|
|
161
|
-
record.add_argument("--stage", required=True)
|
|
162
|
-
record.add_argument("--status", choices=STATUSES)
|
|
163
|
-
record.add_argument("--design-state", choices=DESIGN_STATES)
|
|
164
|
-
record.add_argument("--strategy")
|
|
165
|
-
record.add_argument("--artifact", action="append", default=[])
|
|
166
|
-
record.add_argument("--note")
|
|
167
|
-
record.add_argument("--reviewer")
|
|
168
|
-
record.set_defaults(func=command_record)
|
|
169
|
-
args = parser.parse_args()
|
|
170
|
-
try:
|
|
171
|
-
args.func(args)
|
|
172
|
-
except (OSError, ValueError, KeyError, json.JSONDecodeError) as error:
|
|
173
|
-
parser.exit(2, f"{error}\n")
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
if __name__ == "__main__":
|
|
177
|
-
main()
|