@petercjl/taobao-ai-ops 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -22
- package/package.json +2 -7
- package/requirements.txt +1 -0
- package/skills/taobao-ai-operations/SKILL.md +8 -1
- package/src/cli.mjs +16 -6
- package/src/components.mjs +73 -0
- package/src/core.mjs +5 -1
- package/src/runtime.mjs +18 -16
- package/src/skill-manager.mjs +4 -4
- package/src/update.mjs +5 -5
- package/src/workflow.mjs +3 -3
- package/suite-manifest.json +186 -10
- package/skills/taobao-search-product-form/SKILL.md +0 -74
- package/skills/taobao-search-product-form/adapters/codex.json +0 -139
- package/skills/taobao-search-product-form/adapters/sealseek.json +0 -24
- package/skills/taobao-search-product-form/agents/openai.yaml +0 -3
- package/skills/taobao-search-product-form/capabilities.json +0 -148
- package/skills/taobao-search-product-form/references/SCHEMA.md +0 -3
- package/skills/taobao-search-product-form/references/index.md +0 -10
- package/skills/taobao-search-product-form/references/log.md +0 -35
- package/skills/taobao-search-product-form/references/queries/evaluate-white-label-forms.md +0 -3
- package/skills/taobao-search-product-form/references/queries/identify-advertisers.md +0 -5
- package/skills/taobao-search-product-form/references/queries/identify-geography.md +0 -5
- package/skills/taobao-search-product-form/references/queries/identify-price-bands.md +0 -3
- package/skills/taobao-search-product-form/references/queries/identify-product-forms.md +0 -3
- package/skills/taobao-search-product-form/references/queries/identify-title-roots.md +0 -5
- package/skills/taobao-search-product-form/references/report-output.md +0 -23
- package/skills/taobao-search-product-form/references/source-manifest.md +0 -37
- package/skills/taobao-search-product-form/references/title-lexicon.json +0 -55
- package/skills/taobao-search-product-form/references/topics/advertiser-analysis-method.md +0 -43
- package/skills/taobao-search-product-form/references/topics/geography-analysis-method.md +0 -23
- package/skills/taobao-search-product-form/references/topics/price-band-analysis-method.md +0 -27
- package/skills/taobao-search-product-form/references/topics/title-root-analysis-method.md +0 -37
- package/skills/taobao-search-product-form/references/topics/visual-form-method.md +0 -31
- package/skills/taobao-search-product-form/references/topics/white-label-evidence.md +0 -38
- package/skills/taobao-search-product-form/references/workflow-ledger.md +0 -28
- package/skills/taobao-search-product-form/report_runtime/VERSION +0 -1
- package/skills/taobao-search-product-form/report_runtime/renderer.py +0 -485
- package/skills/taobao-search-product-form/report_runtime/templates/workbench.html +0 -67
- package/skills/taobao-search-product-form/report_runtime/validator.py +0 -117
- package/skills/taobao-search-product-form/scripts/analyze.py +0 -119
- package/skills/taobao-search-product-form/scripts/analyze_advertisers.py +0 -352
- package/skills/taobao-search-product-form/scripts/analyze_geography.py +0 -316
- package/skills/taobao-search-product-form/scripts/analyze_price_bands.py +0 -257
- package/skills/taobao-search-product-form/scripts/analyze_title_roots.py +0 -327
- package/skills/taobao-search-product-form/scripts/assemble_opportunity_evidence.py +0 -322
- package/skills/taobao-search-product-form/scripts/build_report_viewmodel.py +0 -822
- package/skills/taobao-search-product-form/scripts/build_visual_review_viewmodel.py +0 -177
- package/skills/taobao-search-product-form/scripts/clean_search_export.mjs +0 -61
- package/skills/taobao-search-product-form/scripts/cluster_visual_forms.py +0 -193
- package/skills/taobao-search-product-form/scripts/compile_style_prototypes.py +0 -115
- package/skills/taobao-search-product-form/scripts/compile_visual_review.py +0 -60
- package/skills/taobao-search-product-form/scripts/compile_visual_selection.py +0 -148
- package/skills/taobao-search-product-form/scripts/prepare.py +0 -180
- package/skills/taobao-search-product-form/scripts/render_report.py +0 -31
- package/skills/taobao-search-product-form/scripts/validate_opportunity_cards.py +0 -84
- package/skills/taobao-search-product-form/scripts/validate_report_viewmodel.py +0 -152
- package/skills/taobao-search-product-form/scripts/visual_form_pipeline.py +0 -247
- package/skills/taobao-search-product-form/scripts/visual_observation_batches.py +0 -117
- package/skills/taobao-search-product-form/scripts/workflow_ledger.py +0 -177
- package/skills/yuce-category-opportunity-report/SKILL.md +0 -151
- package/skills/yuce-category-opportunity-report/adapters/codex.json +0 -113
- package/skills/yuce-category-opportunity-report/adapters/sealseek.json +0 -39
- package/skills/yuce-category-opportunity-report/agents/openai.yaml +0 -3
- package/skills/yuce-category-opportunity-report/assets/chart-views.js +0 -80
- package/skills/yuce-category-opportunity-report/assets/package.json +0 -1
- package/skills/yuce-category-opportunity-report/assets/report-template.html +0 -62
- package/skills/yuce-category-opportunity-report/capabilities.json +0 -48
- package/skills/yuce-category-opportunity-report/references/SCHEMA.md +0 -5
- package/skills/yuce-category-opportunity-report/references/data-contract.md +0 -40
- package/skills/yuce-category-opportunity-report/references/index.md +0 -6
- package/skills/yuce-category-opportunity-report/references/log.md +0 -29
- package/skills/yuce-category-opportunity-report/references/queries/assess-category-opportunity.md +0 -7
- package/skills/yuce-category-opportunity-report/references/source-manifest.md +0 -14
- package/skills/yuce-category-opportunity-report/references/topics/category-opportunity-method.md +0 -39
- package/skills/yuce-category-opportunity-report/scripts/build_report.py +0 -779
- package/skills/yuce-category-opportunity-report/scripts/market_routes.py +0 -139
|
@@ -1,148 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Validate and compile three-axis image observations for selection review.
|
|
3
|
-
|
|
4
|
-
This script checks provenance and evidence contracts; it does not infer labels
|
|
5
|
-
from pixels or declare product-market opportunity.
|
|
6
|
-
"""
|
|
7
|
-
|
|
8
|
-
import argparse
|
|
9
|
-
import hashlib
|
|
10
|
-
import json
|
|
11
|
-
from collections import Counter, defaultdict
|
|
12
|
-
from pathlib import Path
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
def load(path):
|
|
16
|
-
return json.loads(Path(path).read_text(encoding="utf-8"))
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
def sha256(path):
|
|
20
|
-
h = hashlib.sha256()
|
|
21
|
-
with Path(path).open("rb") as stream:
|
|
22
|
-
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
23
|
-
h.update(chunk)
|
|
24
|
-
return h.hexdigest()
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def save(path, value):
|
|
28
|
-
with Path(path).open("x", encoding="utf-8") as stream:
|
|
29
|
-
json.dump(value, stream, ensure_ascii=False, indent=2)
|
|
30
|
-
stream.write("\n")
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def map_labels(registry, dimension):
|
|
34
|
-
owner = {}
|
|
35
|
-
for canonical, info in registry.get(dimension, {}).items():
|
|
36
|
-
for alias in [canonical, *info.get("aliases", [])]:
|
|
37
|
-
key = alias.strip().casefold()
|
|
38
|
-
if not key or key in owner and owner[key] != canonical:
|
|
39
|
-
raise ValueError(f"invalid or conflicting {dimension} alias: {alias}")
|
|
40
|
-
owner[key] = canonical
|
|
41
|
-
return owner
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def clean_text(value):
|
|
45
|
-
return str(value or "").strip()
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
def compile_run(args):
|
|
49
|
-
plan = load(args.plan)
|
|
50
|
-
draft = load(args.observations)
|
|
51
|
-
registry = load(args.registry)
|
|
52
|
-
if draft.get("schema_version") != 2 or registry.get("schema_version") != 2:
|
|
53
|
-
raise ValueError("three-axis observations and registry require schema_version 2")
|
|
54
|
-
expected = {str(item["product_id"]): item for item in plan["items"] if item["status"] == "ready"}
|
|
55
|
-
entries = draft.get("observations", [])
|
|
56
|
-
ids = [str(item.get("product_id", "")) for item in entries]
|
|
57
|
-
if len(ids) != len(set(ids)) or set(ids) != set(expected):
|
|
58
|
-
raise ValueError(f"one observation per ready image required; missing={sorted(set(expected)-set(ids))}, extra={sorted(set(ids)-set(expected))}")
|
|
59
|
-
maps = {axis: map_labels(registry, axis) for axis in ("form", "style", "audience")}
|
|
60
|
-
raw = []
|
|
61
|
-
products = []
|
|
62
|
-
unresolved = defaultdict(Counter)
|
|
63
|
-
candidates = defaultdict(Counter)
|
|
64
|
-
counts = defaultdict(Counter)
|
|
65
|
-
examples = defaultdict(lambda: defaultdict(list))
|
|
66
|
-
for entry in entries:
|
|
67
|
-
pid = str(entry["product_id"])
|
|
68
|
-
source = expected[pid]
|
|
69
|
-
if not clean_text(entry.get("visible_evidence")):
|
|
70
|
-
raise ValueError(f"missing visible evidence: {pid}")
|
|
71
|
-
status = entry.get("status")
|
|
72
|
-
if status not in ("tagged", "uncertain"):
|
|
73
|
-
raise ValueError(f"invalid status: {pid}")
|
|
74
|
-
bound = {**entry, "product_id": pid, "batch_id": source["batch_id"],
|
|
75
|
-
"image_sha256": source["image_sha256"], "image_path": source["image_path"],
|
|
76
|
-
"source_rank": source["source_rank"]}
|
|
77
|
-
row = {"product_id": pid, "source_rank": source["source_rank"],
|
|
78
|
-
"batch_id": source["batch_id"], "image_path": source["image_path"],
|
|
79
|
-
"image_sha256": source["image_sha256"], "status": status,
|
|
80
|
-
"visible_evidence": entry["visible_evidence"],
|
|
81
|
-
"image_presentation": clean_text(entry.get("image_presentation"))}
|
|
82
|
-
for axis in ("form", "style", "audience"):
|
|
83
|
-
item = entry.get(axis)
|
|
84
|
-
if not isinstance(item, dict):
|
|
85
|
-
raise ValueError(f"missing {axis} object: {pid}")
|
|
86
|
-
candidate = clean_text(item.get("candidate"))
|
|
87
|
-
if axis == "form" and status == "tagged" and not candidate:
|
|
88
|
-
raise ValueError(f"tagged product needs form candidate: {pid}")
|
|
89
|
-
if axis == "style" and candidate:
|
|
90
|
-
cues = item.get("product_cues", [])
|
|
91
|
-
if not isinstance(cues, list) or len([x for x in cues if clean_text(x)]) < 2 or not clean_text(item.get("physical_delta")):
|
|
92
|
-
raise ValueError(f"style requires two product cues and physical delta: {pid}")
|
|
93
|
-
if axis == "audience" and candidate:
|
|
94
|
-
cues = item.get("task_cues", [])
|
|
95
|
-
if not isinstance(cues, list) or not any(clean_text(x) for x in cues) or not clean_text(item.get("physical_delta")):
|
|
96
|
-
raise ValueError(f"audience requires task cue and physical delta: {pid}")
|
|
97
|
-
if item.get("evidence_tier") not in ("image_only", "combined"):
|
|
98
|
-
raise ValueError(f"audience evidence tier required: {pid}")
|
|
99
|
-
canonical = maps[axis].get(candidate.casefold()) if candidate else None
|
|
100
|
-
if candidate:
|
|
101
|
-
candidates[axis][candidate] += 1
|
|
102
|
-
if canonical is None:
|
|
103
|
-
unresolved[axis][candidate] += 1
|
|
104
|
-
if canonical:
|
|
105
|
-
counts[axis][canonical] += 1
|
|
106
|
-
if len(examples[axis][canonical]) < 3:
|
|
107
|
-
examples[axis][canonical].append(pid)
|
|
108
|
-
row[axis] = {**item, "candidate": candidate or None, "label": canonical}
|
|
109
|
-
raw.append(bound)
|
|
110
|
-
products.append(row)
|
|
111
|
-
products.sort(key=lambda item: item["source_rank"])
|
|
112
|
-
unavailable_reasons = Counter(item.get("unavailable_reason", "unknown")
|
|
113
|
-
for item in plan["items"] if item["status"] == "image_unavailable")
|
|
114
|
-
summary = {"schema_version": 2, "strategy": "visual-selection-three-axis-v1",
|
|
115
|
-
"source_plan_sha256": sha256(args.plan), "source_observations_sha256": sha256(args.observations),
|
|
116
|
-
"registry_sha256": sha256(args.registry), "registry_version": registry.get("version"),
|
|
117
|
-
"scope": plan["scope"], "reviewed_products": len(products),
|
|
118
|
-
"image_unavailable_by_reason": dict(unavailable_reasons),
|
|
119
|
-
"axes": {axis: {"candidate_labels": dict(candidates[axis]),
|
|
120
|
-
"canonical_counts": dict(counts[axis]),
|
|
121
|
-
"examples": dict(examples[axis]),
|
|
122
|
-
"unresolved": dict(unresolved[axis]),
|
|
123
|
-
"unclassified_products": sum(not row[axis]["label"] for row in products)}
|
|
124
|
-
for axis in ("form", "style", "audience")},
|
|
125
|
-
"qa": {"one_record_per_ready_image": True, "image_hash_bound": True,
|
|
126
|
-
"unavailable_reasons_reconciled": sum(unavailable_reasons.values()) == plan["scope"]["image_unavailable"],
|
|
127
|
-
"all_candidates_resolved": not any(unresolved.values()),
|
|
128
|
-
"physical_deltas_checked": True},
|
|
129
|
-
"interpretation_boundary": "Pilot product labels and supply descriptions; not actual buyer demographics or proven white-label opportunity."}
|
|
130
|
-
out = Path(args.output_dir).resolve()
|
|
131
|
-
if out.exists():
|
|
132
|
-
raise FileExistsError(out)
|
|
133
|
-
out.mkdir(parents=True)
|
|
134
|
-
save(out / "raw_observations.json", {"schema_version": 2, "records": raw})
|
|
135
|
-
save(out / "product_labels.json", {"schema_version": 2, "records": products})
|
|
136
|
-
save(out / "label_summary.json", summary)
|
|
137
|
-
print(json.dumps({"output_dir": str(out), "products": len(products),
|
|
138
|
-
"counts": {axis: dict(counts[axis]) for axis in counts},
|
|
139
|
-
"unresolved": {axis: dict(unresolved[axis]) for axis in unresolved}}, ensure_ascii=False))
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
if __name__ == "__main__":
|
|
143
|
-
parser = argparse.ArgumentParser(description=__doc__)
|
|
144
|
-
parser.add_argument("--plan", required=True)
|
|
145
|
-
parser.add_argument("--observations", required=True)
|
|
146
|
-
parser.add_argument("--registry", required=True)
|
|
147
|
-
parser.add_argument("--output-dir", required=True)
|
|
148
|
-
compile_run(parser.parse_args())
|
|
@@ -1,180 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Normalize a Taobao search export and prepare auditable image contact sheets."""
|
|
3
|
-
|
|
4
|
-
import argparse
|
|
5
|
-
import hashlib
|
|
6
|
-
import io
|
|
7
|
-
import json
|
|
8
|
-
import re
|
|
9
|
-
import urllib.parse
|
|
10
|
-
import urllib.request
|
|
11
|
-
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
12
|
-
from datetime import datetime, timezone
|
|
13
|
-
from pathlib import Path
|
|
14
|
-
|
|
15
|
-
import pandas as pd
|
|
16
|
-
from PIL import Image, ImageDraw, ImageOps
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
REQUIRED = ["商品ID", "商品名称", "商品图片链接", "占位类型"]
|
|
20
|
-
ALLOWED_IMAGE_HOSTS = ("alicdn.com", "alicdn.cn")
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
def scalar(value):
|
|
24
|
-
if pd.isna(value):
|
|
25
|
-
return None
|
|
26
|
-
if hasattr(value, "item"):
|
|
27
|
-
value = value.item()
|
|
28
|
-
if isinstance(value, (int, float, bool, str)):
|
|
29
|
-
return value
|
|
30
|
-
return str(value)
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def number(value):
|
|
34
|
-
try:
|
|
35
|
-
return float(value)
|
|
36
|
-
except (TypeError, ValueError):
|
|
37
|
-
return None
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def safe_image_url(url):
|
|
41
|
-
parsed = urllib.parse.urlparse(str(url))
|
|
42
|
-
host = (parsed.hostname or "").lower()
|
|
43
|
-
return parsed.scheme == "https" and any(host == suffix or host.endswith("." + suffix) for suffix in ALLOWED_IMAGE_HOSTS)
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def fetch_image(product, cache_dir):
|
|
47
|
-
pid = product["product_id"]
|
|
48
|
-
url = product["image_url"]
|
|
49
|
-
if not safe_image_url(url):
|
|
50
|
-
return pid, {"status": "blocked_host", "path": None}
|
|
51
|
-
request = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
|
52
|
-
try:
|
|
53
|
-
with urllib.request.urlopen(request, timeout=15) as response:
|
|
54
|
-
blob = response.read(10_000_001)
|
|
55
|
-
if len(blob) > 10_000_000:
|
|
56
|
-
return pid, {"status": "too_large", "path": None}
|
|
57
|
-
image = Image.open(io.BytesIO(blob))
|
|
58
|
-
image.load()
|
|
59
|
-
image = ImageOps.exif_transpose(image).convert("RGB")
|
|
60
|
-
image.thumbnail((500, 500))
|
|
61
|
-
digest = hashlib.sha256(pid.encode()).hexdigest()[:16]
|
|
62
|
-
path = cache_dir / f"{digest}.jpg"
|
|
63
|
-
image.save(path, "JPEG", quality=88, optimize=True)
|
|
64
|
-
return pid, {"status": "ok", "path": str(path.relative_to(cache_dir.parent)), "width": image.width, "height": image.height}
|
|
65
|
-
except Exception as exc: # fetch errors are reported, never converted into visual labels
|
|
66
|
-
return pid, {"status": "fetch_failed", "path": None, "error": type(exc).__name__}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
def make_sheets(products, fetches, out_dir, prefix):
|
|
70
|
-
cell_w, cell_h, columns, per_sheet = 180, 190, 8, 80
|
|
71
|
-
included = [p for p in products if fetches.get(p["product_id"], {}).get("status") == "ok"]
|
|
72
|
-
result = []
|
|
73
|
-
for start in range(0, len(included), per_sheet):
|
|
74
|
-
batch = included[start:start + per_sheet]
|
|
75
|
-
rows = (len(batch) + columns - 1) // columns
|
|
76
|
-
sheet = Image.new("RGB", (cell_w * columns, cell_h * rows), "white")
|
|
77
|
-
draw = ImageDraw.Draw(sheet)
|
|
78
|
-
for i, product in enumerate(batch):
|
|
79
|
-
x, y = (i % columns) * cell_w, (i // columns) * cell_h
|
|
80
|
-
path = out_dir / fetches[product["product_id"]]["path"]
|
|
81
|
-
image = Image.open(path)
|
|
82
|
-
image.thumbnail((160, 158))
|
|
83
|
-
sheet.paste(image, (x + (cell_w - image.width) // 2, y + 4))
|
|
84
|
-
draw.text((x + 5, y + 164), f"{product['product_id']}", fill="black")
|
|
85
|
-
rank = product["first_natural"] if product["first_natural"] is not None else product["first_ad"]
|
|
86
|
-
draw.text((x + 5, y + 177), f"{'N' if product['first_natural'] is not None else 'A'} {rank}", fill="black")
|
|
87
|
-
path = out_dir / f"{prefix}-{start // per_sheet + 1:02d}.jpg"
|
|
88
|
-
sheet.save(path, "JPEG", quality=88)
|
|
89
|
-
result.append(path.name)
|
|
90
|
-
return result
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
def main():
|
|
94
|
-
parser = argparse.ArgumentParser()
|
|
95
|
-
parser.add_argument("--input", required=True)
|
|
96
|
-
parser.add_argument("--output-dir", required=True)
|
|
97
|
-
parser.add_argument("--search-term")
|
|
98
|
-
parser.add_argument("--max-organic", type=int)
|
|
99
|
-
parser.add_argument("--max-ad-only", type=int, default=50)
|
|
100
|
-
args = parser.parse_args()
|
|
101
|
-
source = Path(args.input).expanduser().resolve()
|
|
102
|
-
out = Path(args.output_dir).expanduser().resolve()
|
|
103
|
-
if not source.is_file() or source.suffix.lower() != ".xlsx":
|
|
104
|
-
parser.error("input must be an existing .xlsx file")
|
|
105
|
-
if out.exists():
|
|
106
|
-
parser.error("output directory already exists; select a new path")
|
|
107
|
-
if args.max_organic is not None and args.max_organic < 1:
|
|
108
|
-
parser.error("--max-organic must be positive")
|
|
109
|
-
frame = pd.read_excel(source)
|
|
110
|
-
missing = [column for column in REQUIRED if column not in frame.columns]
|
|
111
|
-
if missing:
|
|
112
|
-
parser.error("missing required columns: " + ", ".join(missing))
|
|
113
|
-
out.mkdir(parents=True)
|
|
114
|
-
cache = out / "images"
|
|
115
|
-
cache.mkdir()
|
|
116
|
-
raw_rows = []
|
|
117
|
-
products = {}
|
|
118
|
-
for index, (_, row) in enumerate(frame.iterrows(), start=1):
|
|
119
|
-
pid = str(scalar(row["商品ID"]))
|
|
120
|
-
position = int(number(row.get("序号")) or index)
|
|
121
|
-
placement = str(scalar(row["占位类型"]) or "")
|
|
122
|
-
if placement not in ("自然位", "广告位"):
|
|
123
|
-
raise ValueError(f"unknown placement at row {index}: {placement}")
|
|
124
|
-
raw = {"product_id": pid, "display_position": position, "placement": placement}
|
|
125
|
-
raw_rows.append(raw)
|
|
126
|
-
if pid not in products:
|
|
127
|
-
products[pid] = {
|
|
128
|
-
"product_id": pid,
|
|
129
|
-
"title": str(scalar(row["商品名称"]) or ""),
|
|
130
|
-
"image_url": str(scalar(row["商品图片链接"]) or ""),
|
|
131
|
-
"item_url": str(scalar(row.get("商品链接")) or ""),
|
|
132
|
-
"price": number(row.get("现价")),
|
|
133
|
-
"payment_count": number(row.get("付款人数")),
|
|
134
|
-
"shop": str(scalar(row.get("店铺名称")) or ""),
|
|
135
|
-
"shop_type": str(scalar(row.get("店铺类型")) or ""),
|
|
136
|
-
"platform_category": str(scalar(row.get("类目")) or ""),
|
|
137
|
-
"natural_positions": [], "ad_positions": [],
|
|
138
|
-
}
|
|
139
|
-
key = "natural_positions" if placement == "自然位" else "ad_positions"
|
|
140
|
-
products[pid][key].append(position)
|
|
141
|
-
entries = list(products.values())
|
|
142
|
-
for product in entries:
|
|
143
|
-
product["first_natural"] = min(product["natural_positions"], default=None)
|
|
144
|
-
product["first_ad"] = min(product["ad_positions"], default=None)
|
|
145
|
-
organic = sorted((p for p in entries if p["first_natural"] is not None), key=lambda p: p["first_natural"])
|
|
146
|
-
for rank, product in enumerate(organic, start=1):
|
|
147
|
-
product["organic_rank"] = rank
|
|
148
|
-
ad_only = sorted((p for p in entries if p["first_natural"] is None), key=lambda p: p["first_ad"])
|
|
149
|
-
selected_organic = organic[:args.max_organic] if args.max_organic else organic
|
|
150
|
-
selected_ad = ad_only[:args.max_ad_only]
|
|
151
|
-
selected = selected_organic + selected_ad
|
|
152
|
-
fetches = {}
|
|
153
|
-
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
154
|
-
futures = [pool.submit(fetch_image, p, cache) for p in selected]
|
|
155
|
-
for future in as_completed(futures):
|
|
156
|
-
pid, result = future.result()
|
|
157
|
-
fetches[pid] = result
|
|
158
|
-
sheets = make_sheets(selected_organic, fetches, out, "organic") + make_sheets(selected_ad, fetches, out, "ad-only")
|
|
159
|
-
stem = source.stem
|
|
160
|
-
inferred = re.search(r"市场分析[-_](.+?)[-_]\d{8}$", stem)
|
|
161
|
-
term = args.search_term or (inferred.group(1) if inferred else "")
|
|
162
|
-
manifest = {
|
|
163
|
-
"schema_version": 1,
|
|
164
|
-
"source": {"name": source.name, "sha256": hashlib.sha256(source.read_bytes()).hexdigest(), "rows": len(raw_rows), "captured_at": datetime.now(timezone.utc).isoformat()},
|
|
165
|
-
"search_term": term,
|
|
166
|
-
"search_term_inferred": args.search_term is None,
|
|
167
|
-
"scope": {"kind": "pilot" if args.max_organic else "full", "organic_selected": len(selected_organic), "organic_total": len(organic), "ad_only_selected": len(selected_ad), "ad_only_total": len(ad_only)},
|
|
168
|
-
"source_rows": raw_rows,
|
|
169
|
-
"products": entries,
|
|
170
|
-
"selected_organic_ids": [p["product_id"] for p in selected_organic],
|
|
171
|
-
"selected_ad_only_ids": [p["product_id"] for p in selected_ad],
|
|
172
|
-
"fetches": fetches,
|
|
173
|
-
"contact_sheets": sheets,
|
|
174
|
-
}
|
|
175
|
-
(out / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
176
|
-
print(json.dumps({"manifest": str(out / "manifest.json"), "rows": len(raw_rows), "unique_products": len(entries), "organic_products": len(organic), "ad_only_products": len(ad_only), "images_ok": sum(v["status"] == "ok" for v in fetches.values()), "images_requested": len(selected), "contact_sheets": sheets}, ensure_ascii=False))
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
if __name__ == "__main__":
|
|
180
|
-
main()
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Render and validate a report with this package's pinned HTML runtime."""
|
|
3
|
-
|
|
4
|
-
import argparse
|
|
5
|
-
import json
|
|
6
|
-
import sys
|
|
7
|
-
from pathlib import Path
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
RUNTIME = Path(__file__).resolve().parents[1] / "report_runtime"
|
|
11
|
-
sys.path.insert(0, str(RUNTIME))
|
|
12
|
-
|
|
13
|
-
from renderer import render_file # noqa: E402
|
|
14
|
-
from validator import validate_file # noqa: E402
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def main():
|
|
18
|
-
parser = argparse.ArgumentParser(description=__doc__)
|
|
19
|
-
parser.add_argument("--input", required=True, help="Existing compact-workbench@1.0 ViewModel JSON")
|
|
20
|
-
parser.add_argument("--output", required=True, help="New standalone HTML report path")
|
|
21
|
-
args = parser.parse_args()
|
|
22
|
-
rendered = render_file(args.input, args.output)
|
|
23
|
-
checked = validate_file(args.output)
|
|
24
|
-
result = {"ok": checked["ok"], "render": rendered, "validation": checked}
|
|
25
|
-
print(json.dumps(result, ensure_ascii=False))
|
|
26
|
-
if not checked["ok"]:
|
|
27
|
-
raise SystemExit(1)
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
if __name__ == "__main__":
|
|
31
|
-
main()
|
|
@@ -1,84 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Validate opportunity-card structure and links to joined product evidence.
|
|
3
|
-
|
|
4
|
-
This is a mechanical check. It cannot decide whether natural-language claims are
|
|
5
|
-
commercially justified; the Agent must audit figures and alternative explanations.
|
|
6
|
-
"""
|
|
7
|
-
|
|
8
|
-
import argparse
|
|
9
|
-
import hashlib
|
|
10
|
-
import json
|
|
11
|
-
from pathlib import Path
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
def validate(cards, evidence, evidence_sha256):
|
|
15
|
-
if cards.get("schema_version") != 2 or not isinstance(cards.get("cards"), list):
|
|
16
|
-
raise ValueError("cards must have schema_version 2 and a cards array")
|
|
17
|
-
if cards.get("source_evidence_sha256") != evidence_sha256:
|
|
18
|
-
raise ValueError("opportunity cards do not match the current comparison evidence hash")
|
|
19
|
-
if not evidence.get("sources", {}).get("prototype_labels", {}).get("sha256"):
|
|
20
|
-
raise ValueError("comparison evidence lacks whole-product prototype labels")
|
|
21
|
-
candidates = {c["candidate_id"]: c for c in evidence["candidates"]}
|
|
22
|
-
if not any(c.get("axis") == "product_style" for c in candidates.values()):
|
|
23
|
-
raise ValueError("comparison evidence lacks product-style candidates")
|
|
24
|
-
products = {p["product_id"]: p for p in evidence["products"]}
|
|
25
|
-
seen = set()
|
|
26
|
-
orders = set()
|
|
27
|
-
for card in cards["cards"]:
|
|
28
|
-
cid = card.get("candidate_id")
|
|
29
|
-
if cid not in candidates or cid in seen:
|
|
30
|
-
raise ValueError(f"unknown or repeated candidate_id: {cid}")
|
|
31
|
-
seen.add(cid)
|
|
32
|
-
candidate = candidates[cid]
|
|
33
|
-
if candidate.get("axis") != "product_style":
|
|
34
|
-
raise ValueError(f"{cid}: opportunity card must use a whole-product style prototype")
|
|
35
|
-
for field in ("definition", "parent_comparison", "scope", "interpretation"):
|
|
36
|
-
if not isinstance(card.get(field), str) or not card[field].strip():
|
|
37
|
-
raise ValueError(f"{cid}: missing {field}")
|
|
38
|
-
if "score" in card or "entry_decision" in card:
|
|
39
|
-
raise ValueError(f"{cid}: card cannot contain a composite score or entry verdict")
|
|
40
|
-
for field in ("observations", "supporting_evidence", "counterevidence", "unknowns", "next_checks"):
|
|
41
|
-
if not isinstance(card.get(field), list) or not card[field]:
|
|
42
|
-
raise ValueError(f"{cid}: missing {field}")
|
|
43
|
-
for item in card["observations"] + card["supporting_evidence"] + card["counterevidence"]:
|
|
44
|
-
if not isinstance(item.get("claim"), str) or not item["claim"].strip():
|
|
45
|
-
raise ValueError(f"{cid}: evidence item lacks claim")
|
|
46
|
-
if not isinstance(item.get("refs"), list) or not item["refs"]:
|
|
47
|
-
raise ValueError(f"{cid}: evidence item lacks references")
|
|
48
|
-
for ref in item["refs"]:
|
|
49
|
-
if ref.isdigit():
|
|
50
|
-
if ref not in products or ref not in candidate["product_ids"]:
|
|
51
|
-
raise ValueError(f"{cid}: product reference is not a member: {ref}")
|
|
52
|
-
continue
|
|
53
|
-
target_id, separator, field_path = ref.partition(".")
|
|
54
|
-
if not separator or target_id not in candidates:
|
|
55
|
-
raise ValueError(f"{cid}: invalid candidate reference: {ref}")
|
|
56
|
-
target = candidates[target_id]
|
|
57
|
-
for key in field_path.split("."):
|
|
58
|
-
if not isinstance(target, dict) or key not in target:
|
|
59
|
-
raise ValueError(f"{cid}: missing evidence field: {ref}")
|
|
60
|
-
target = target[key]
|
|
61
|
-
for check in card["next_checks"]:
|
|
62
|
-
if not check.get("question") or not check.get("changes_assessment_if"):
|
|
63
|
-
raise ValueError(f"{cid}: next check lacks a discriminating result")
|
|
64
|
-
research_order = card.get("research_order")
|
|
65
|
-
if research_order is not None:
|
|
66
|
-
position = research_order.get("position")
|
|
67
|
-
if not isinstance(position, int) or position < 1 or position in orders or not research_order.get("reason"):
|
|
68
|
-
raise ValueError(f"{cid}: invalid or duplicate research order")
|
|
69
|
-
orders.add(position)
|
|
70
|
-
if not cards["cards"]:
|
|
71
|
-
raise ValueError("no opportunity cards")
|
|
72
|
-
return {"ok": True, "cards": len(seen), "linked_candidates": len(candidates),
|
|
73
|
-
"linked_products": len(products)}
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
if __name__ == "__main__":
|
|
77
|
-
parser = argparse.ArgumentParser(description=__doc__)
|
|
78
|
-
parser.add_argument("--cards", required=True)
|
|
79
|
-
parser.add_argument("--evidence", required=True)
|
|
80
|
-
args = parser.parse_args()
|
|
81
|
-
cards = json.loads(Path(args.cards).read_text(encoding="utf-8"))
|
|
82
|
-
evidence = json.loads(Path(args.evidence).read_text(encoding="utf-8"))
|
|
83
|
-
evidence_sha256 = hashlib.sha256(Path(args.evidence).read_bytes()).hexdigest()
|
|
84
|
-
print(json.dumps(validate(cards, evidence, evidence_sha256), ensure_ascii=False))
|
|
@@ -1,152 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Audit the comprehensive report ViewModel against its module sources."""
|
|
3
|
-
|
|
4
|
-
from __future__ import annotations
|
|
5
|
-
|
|
6
|
-
import argparse
|
|
7
|
-
import hashlib
|
|
8
|
-
import json
|
|
9
|
-
from pathlib import Path
|
|
10
|
-
|
|
11
|
-
from build_report_viewmodel import check_report_bindings, md_blocks, read_json, read_product_links, source_path
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
MODULE_VIEWS = {
|
|
15
|
-
"advertising": "advertising",
|
|
16
|
-
"title": "title_roots",
|
|
17
|
-
"geography": "geography",
|
|
18
|
-
"price": "price",
|
|
19
|
-
"visual": "visual",
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
def validate(manifest_path: Path, viewmodel_path: Path) -> dict:
|
|
24
|
-
manifest = read_json(manifest_path)
|
|
25
|
-
loaded, srcs = {}, {}
|
|
26
|
-
for key in ("visual", "decision"):
|
|
27
|
-
loaded[key], srcs[key] = {}, {}
|
|
28
|
-
for role, value in manifest["modules"][key].items():
|
|
29
|
-
path = source_path(manifest_path, value)
|
|
30
|
-
loaded[key][role] = path.read_text(encoding="utf-8") if path.suffix == ".md" else read_json(path)
|
|
31
|
-
srcs[key][role] = {"sha256": hashlib.sha256(path.read_bytes()).hexdigest()}
|
|
32
|
-
check_report_bindings(loaded, srcs)
|
|
33
|
-
report = read_json(viewmodel_path)
|
|
34
|
-
expected_binding = {
|
|
35
|
-
"prototype_labels_sha256": srcs["visual"]["labels"]["sha256"],
|
|
36
|
-
"prototype_summary_sha256": srcs["visual"]["prototypes"]["sha256"],
|
|
37
|
-
"comparison_evidence_sha256": srcs["decision"]["evidence"]["sha256"],
|
|
38
|
-
"opportunity_cards_sha256": srcs["decision"]["cards"]["sha256"],
|
|
39
|
-
}
|
|
40
|
-
if report.get("meta", {}).get("artifact_binding") != expected_binding:
|
|
41
|
-
raise ValueError("ViewModel is not bound to current classification, comparison and cards")
|
|
42
|
-
views = {view["id"]: view for view in report["views"]}
|
|
43
|
-
expected_views = {"overview", "sample", "advertising", "title_roots", "geography", "price",
|
|
44
|
-
"visual", "candidates", "opportunities", "evidence", "method"}
|
|
45
|
-
if not expected_views <= views.keys():
|
|
46
|
-
raise ValueError(f"missing views: {sorted(expected_views - views.keys())}")
|
|
47
|
-
retained_sections = {}
|
|
48
|
-
for module, view_id in MODULE_VIEWS.items():
|
|
49
|
-
item = manifest["modules"][module]
|
|
50
|
-
if "interpretation" not in item:
|
|
51
|
-
continue
|
|
52
|
-
source = source_path(manifest_path, item["interpretation"])
|
|
53
|
-
interpretation = loaded[module]["interpretation"] if module == "visual" else source.read_text(encoding="utf-8")
|
|
54
|
-
expected = md_blocks(interpretation)
|
|
55
|
-
actual = views[view_id]["blocks"]
|
|
56
|
-
missing = [block for block in expected if block not in actual]
|
|
57
|
-
if missing:
|
|
58
|
-
raise ValueError(f"{module}: {len(missing)} reviewed interpretation block(s) missing")
|
|
59
|
-
retained_sections[module] = len(expected)
|
|
60
|
-
decision = manifest["modules"]["decision"]
|
|
61
|
-
evidence = loaded["decision"]["evidence"]
|
|
62
|
-
cards = loaded["decision"]["cards"]["cards"]
|
|
63
|
-
candidate_table = next(b for b in views["candidates"]["blocks"] if b.get("title") == "全部候选组合")
|
|
64
|
-
product_table = next(b for b in views["evidence"]["blocks"] if b.get("title") == "全部自然位商品")
|
|
65
|
-
tabs = next(b for b in views["opportunities"]["blocks"] if b["type"] == "tabs")["tabs"]
|
|
66
|
-
if len(candidate_table["rows"]) != len(evidence["candidates"]):
|
|
67
|
-
raise ValueError("candidate group coverage mismatch")
|
|
68
|
-
if len(product_table["rows"]) != len(evidence["products"]):
|
|
69
|
-
raise ValueError("natural product coverage mismatch")
|
|
70
|
-
source_links = read_product_links(source_path(manifest_path, manifest["cleaned_workbook"]))
|
|
71
|
-
visual_summary = read_json(source_path(manifest_path, manifest["modules"]["visual"]["summary"]))
|
|
72
|
-
prototypes = loaded["visual"]["prototypes"]
|
|
73
|
-
visual_tabs = next(b for b in views["visual"]["blocks"] if b.get("type") == "tabs")["tabs"]
|
|
74
|
-
expected_family_names = {item["name"] for item in prototypes}
|
|
75
|
-
actual_tab_names = {tab["label"] for tab in visual_tabs}
|
|
76
|
-
if not expected_family_names <= actual_tab_names:
|
|
77
|
-
raise ValueError("visual gallery missing product-style menus")
|
|
78
|
-
gallery_ids = []
|
|
79
|
-
gallery_by_label = {}
|
|
80
|
-
for tab in visual_tabs:
|
|
81
|
-
gallery = next(b for b in tab["blocks"] if b.get("type") == "products")
|
|
82
|
-
if gallery.get("page_size") != 50:
|
|
83
|
-
raise ValueError("visual gallery must show 50 items per page")
|
|
84
|
-
keys = {option["key"] for option in gallery.get("sort_options", [])}
|
|
85
|
-
if not {"price", "payment", "score"} <= keys:
|
|
86
|
-
raise ValueError("visual gallery missing required sort dimensions")
|
|
87
|
-
for item in gallery["items"]:
|
|
88
|
-
if not str(item.get("image_url", "")).startswith("https://"):
|
|
89
|
-
raise ValueError("visual gallery product missing HTTPS image")
|
|
90
|
-
pid = item.get("copy_value")
|
|
91
|
-
if item.get("item_url") != source_links.get(pid):
|
|
92
|
-
raise ValueError(f"visual gallery product link mismatch for {pid}")
|
|
93
|
-
gallery_ids.append(pid)
|
|
94
|
-
gallery_by_label[tab["label"]] = {item["copy_value"] for item in gallery["items"]}
|
|
95
|
-
for prototype in prototypes:
|
|
96
|
-
if gallery_by_label.get(prototype["name"]) != set(map(str, prototype["product_ids"])):
|
|
97
|
-
raise ValueError(f"visual gallery prototype membership mismatch: {prototype['style_id']}")
|
|
98
|
-
expected_ids = {product["product_id"] for product in evidence["products"]}
|
|
99
|
-
if set(gallery_ids) != expected_ids:
|
|
100
|
-
raise ValueError("visual gallery product coverage mismatch")
|
|
101
|
-
card_tabs = [tab for tab in tabs if tab["id"] != "narrative"]
|
|
102
|
-
if len(card_tabs) != len(cards):
|
|
103
|
-
raise ValueError("opportunity card coverage mismatch")
|
|
104
|
-
if {tab["id"] for tab in card_tabs} != {card["candidate_id"] for card in cards}:
|
|
105
|
-
raise ValueError("opportunity card IDs mismatch")
|
|
106
|
-
if "interpretation" in decision:
|
|
107
|
-
source = source_path(manifest_path, decision["interpretation"])
|
|
108
|
-
expected = md_blocks(loaded["decision"]["interpretation"])
|
|
109
|
-
narrative = next((tab for tab in tabs if tab["id"] == "narrative"), None)
|
|
110
|
-
if not narrative or narrative["blocks"] != expected:
|
|
111
|
-
raise ValueError("reviewed opportunity interpretation missing")
|
|
112
|
-
retained_sections["decision"] = len(expected)
|
|
113
|
-
numbered_points = 0
|
|
114
|
-
for view in views.values():
|
|
115
|
-
for block in iter_blocks(view.get("blocks", [])):
|
|
116
|
-
if block.get("type") == "text":
|
|
117
|
-
passages = [block.get("text", ""), *block.get("paragraphs", [])]
|
|
118
|
-
if any(len(passage) > 120 for passage in passages):
|
|
119
|
-
raise ValueError(f"{view['id']}: oversized unstructured prose block")
|
|
120
|
-
if block.get("type") == "actions":
|
|
121
|
-
numbered_points += len(block.get("items", []))
|
|
122
|
-
for item in block.get("items", []):
|
|
123
|
-
if not item.get("title") or not item.get("text"):
|
|
124
|
-
raise ValueError(f"{view['id']}: incomplete numbered judgment")
|
|
125
|
-
if item.get("source_chunk") and item["text"] not in item["source_chunk"]:
|
|
126
|
-
raise ValueError(f"{view['id']}: numbered point lost its source explanation")
|
|
127
|
-
if numbered_points == 0:
|
|
128
|
-
raise ValueError("no numbered interpretation points")
|
|
129
|
-
return {"views": len(views), "reviewed_interpretation_blocks": retained_sections,
|
|
130
|
-
"natural_products": len(product_table["rows"]), "candidate_groups": len(candidate_table["rows"]),
|
|
131
|
-
"opportunity_cards": len(card_tabs), "visual_gallery_products": len(gallery_ids),
|
|
132
|
-
"visual_form_menus": len(visual_tabs), "numbered_points": numbered_points}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
def iter_blocks(blocks: list[dict]):
|
|
136
|
-
for block in blocks:
|
|
137
|
-
yield block
|
|
138
|
-
yield from iter_blocks(block.get("blocks", []))
|
|
139
|
-
for tab in block.get("tabs", []):
|
|
140
|
-
yield from iter_blocks(tab.get("blocks", []))
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
def main() -> None:
|
|
144
|
-
parser = argparse.ArgumentParser(description=__doc__)
|
|
145
|
-
parser.add_argument("--manifest", type=Path, required=True)
|
|
146
|
-
parser.add_argument("--viewmodel", type=Path, required=True)
|
|
147
|
-
args = parser.parse_args()
|
|
148
|
-
print(json.dumps(validate(args.manifest, args.viewmodel), ensure_ascii=False))
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
if __name__ == "__main__":
|
|
152
|
-
main()
|