@topy-ai/maggie 0.7.53 → 0.7.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README-zh-TW.md +28 -50
- package/README.md +85 -64
- package/bin/maggie.js +51 -7
- package/bundled-references/maggiedash-booking/QA-TEST-PLAN.md +1 -1
- package/bundled-references/maggiedash-booking/RUNTIME-OPERATIONS.md +2 -2
- package/bundled-skills/maggie-blog/SKILL.md +8 -0
- package/bundled-skills/maggie-booking/SKILL.md +20 -12
- package/bundled-skills/maggie-content-localization/SKILL.md +26 -0
- package/bundled-skills/maggie-dash/SKILL.md +28 -2
- package/bundled-skills/maggie-deployment/SKILL.md +27 -0
- package/bundled-skills/maggie-design/SKILL.md +81 -0
- package/bundled-skills/maggie-feedback/SKILL.md +10 -2
- package/bundled-skills/maggie-marketplace/SKILL.md +1 -1
- package/bundled-skills/maggie-marketplace/references/directory-search.md +1 -1
- package/bundled-skills/maggie-marketplace/references/directory.mjs +2 -2
- package/bundled-skills/maggie-mobile-app/SKILL.md +29 -0
- package/bundled-skills/maggie-social-share/SKILL.md +30 -0
- package/bundled-tools/clis/maggie_booking.py +9 -8
- package/bundled-tools/clis/maggie_dash.py +173 -3
- package/bundled-tools/clis/maggie_deployment_ops.py +64 -1
- package/bundled-tools/clis/maggie_design.py +7 -1
- package/bundled-tools/clis/maggie_design_bulk.py +290 -0
- package/bundled-tools/clis/maggie_design_lint.py +131 -1
- package/bundled-tools/clis/maggie_feedback.py +41 -13
- package/bundled-tools/clis/maggie_live_urls.py +138 -0
- package/bundled-tools/clis/maggie_localization.py +3 -0
- package/bundled-tools/clis/maggie_localization_keys.py +151 -0
- package/bundled-tools/clis/maggie_media_generate.py +174 -0
- package/bundled-tools/clis/maggie_mobile.py +84 -1
- package/bundled-tools/clis/maggie_product_release.py +49 -1
- package/bundled-tools/clis/maggie_social_share.py +122 -1
- package/bundled-tools/integrations/maggie-api-pull.md +22 -0
- package/package.json +1 -1
- package/bundled-contracts/maggiedash/execution-board.json +0 -5463
- package/bundled-references/maggiedash-booking/CURRENT-STATE.md +0 -210
- package/bundled-references/maggiedash-booking/EXECUTION-BOARD.json +0 -5463
- package/bundled-references/maggiedash-booking/ORAWELLNESS-INTEGRATION-AUDIT.md +0 -270
- package/bundled-references/maggiedash-booking/PROGRESS.md +0 -279
- package/bundled-references/maggiedash-booking/README.md +0 -288
- package/bundled-references/maggiedash-booking/SKILLS-AND-CLI.md +0 -551
- package/bundled-references/maggiedash-booking/TASK-RUNBOOK.md +0 -107
- package/bundled-references/maggiedash-booking/TASKS.md +0 -178
|
@@ -20,6 +20,10 @@ from urllib.request import Request, urlopen
|
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
DEFAULT_ENDPOINT = "https://feedback.noblox.app/api/feedback"
|
|
23
|
+
# Both hosts accept the same anonymous contract and de-duplicate on feedbackId,
|
|
24
|
+
# so a draft is offered to the next one only when the previous host failed
|
|
25
|
+
# before storing it (network error or a 5xx such as Cloudflare's 52x).
|
|
26
|
+
DEFAULT_ENDPOINTS = (DEFAULT_ENDPOINT, "https://feedback.cmo.so/api/feedback")
|
|
23
27
|
SECRET_RE = re.compile(r"(?i)(bearer\s+|(?:api[_-]?key|token|secret|password|npm_config[^=]*|authorization)\s*[=:]\s*)[^\s,;]+")
|
|
24
28
|
LOCAL_PATH_RE = re.compile(
|
|
25
29
|
r"(?<![A-Za-z0-9])(?:~/(?:[^\s,;:)\]}]+/)*[^\s,;:)\]}]+|"
|
|
@@ -348,28 +352,52 @@ def hosted_payload(data: dict) -> dict:
|
|
|
348
352
|
return payload
|
|
349
353
|
|
|
350
354
|
|
|
355
|
+
def submit_endpoints(value: object) -> list[str]:
|
|
356
|
+
endpoints = [value] if isinstance(value, str) else list(value or DEFAULT_ENDPOINTS)
|
|
357
|
+
result = []
|
|
358
|
+
for endpoint in endpoints:
|
|
359
|
+
endpoint = str(endpoint).rstrip("/")
|
|
360
|
+
parsed = urlparse(endpoint)
|
|
361
|
+
if parsed.scheme != "https" or not parsed.netloc:
|
|
362
|
+
raise ValueError("endpoint must be an absolute HTTPS URL")
|
|
363
|
+
if endpoint not in result:
|
|
364
|
+
result.append(endpoint)
|
|
365
|
+
return result
|
|
366
|
+
|
|
367
|
+
|
|
351
368
|
def submit(args: argparse.Namespace) -> int:
|
|
352
369
|
if not args.confirm:
|
|
353
370
|
raise ValueError("submit requires --confirm; feedback is never sent implicitly")
|
|
354
371
|
path = Path(args.feedback)
|
|
355
372
|
data = read_feedback(path)
|
|
356
|
-
|
|
357
|
-
parsed = urlparse(endpoint)
|
|
358
|
-
if parsed.scheme != "https" or not parsed.netloc:
|
|
359
|
-
raise ValueError("endpoint must be an absolute HTTPS URL")
|
|
373
|
+
endpoints = submit_endpoints(args.endpoint)
|
|
360
374
|
payload = hosted_payload(data)
|
|
361
375
|
payload["projectContext"] = data.get("projectContext") if data.get("privacy", {}).get("projectContextAllowed") else None
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
376
|
+
body = json.dumps(payload, ensure_ascii=False).encode()
|
|
377
|
+
failures = []
|
|
378
|
+
for endpoint in endpoints:
|
|
379
|
+
request = Request(endpoint, data=body, headers={"Content-Type": "application/json", "Accept": "application/json", "User-Agent": "MaggieFeedback/0.1"}, method="POST")
|
|
380
|
+
try:
|
|
381
|
+
with urlopen(request, timeout=20) as response:
|
|
382
|
+
raw = response.read(1_000_000).decode("utf-8", "replace")
|
|
383
|
+
result = json.loads(raw) if raw else {}
|
|
384
|
+
break
|
|
385
|
+
except HTTPError as error:
|
|
386
|
+
failures.append({"endpoint": endpoint, "error": f"HTTP {error.code}"})
|
|
387
|
+
# A 4xx is the host rejecting this draft; another host would too.
|
|
388
|
+
if error.code < 500:
|
|
389
|
+
raise RuntimeError(f"feedback submission failed: {endpoint} answered HTTP {error.code}") from error
|
|
390
|
+
except (URLError, TimeoutError, OSError) as error:
|
|
391
|
+
failures.append({"endpoint": endpoint, "error": safe_text(getattr(error, "reason", error))})
|
|
392
|
+
else:
|
|
393
|
+
tried = "; ".join(f"{item['endpoint']} ({item['error']})" for item in failures)
|
|
394
|
+
raise RuntimeError(f"feedback submission failed on every endpoint: {tried}")
|
|
369
395
|
ack = acknowledgement(result)
|
|
370
396
|
data["submission"] = {"status": "submitted", "endpoint": endpoint, "submittedAt": now(), "acknowledgement": ack}
|
|
397
|
+
if failures:
|
|
398
|
+
data["submission"]["failedEndpoints"] = failures
|
|
371
399
|
path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
372
|
-
print(json.dumps({"feedbackId": data["feedbackId"], "status": "submitted", "acknowledgement": ack}, indent=2, ensure_ascii=False))
|
|
400
|
+
print(json.dumps({"feedbackId": data["feedbackId"], "status": "submitted", "endpoint": endpoint, "failedEndpoints": failures, "acknowledgement": ack}, indent=2, ensure_ascii=False))
|
|
373
401
|
return 0
|
|
374
402
|
|
|
375
403
|
|
|
@@ -815,7 +843,7 @@ def main() -> int:
|
|
|
815
843
|
preview_parser.add_argument("--format", choices=("json", "markdown"), default="json")
|
|
816
844
|
submit_parser = sub.add_parser("submit")
|
|
817
845
|
submit_parser.add_argument("feedback")
|
|
818
|
-
submit_parser.add_argument("--endpoint", default
|
|
846
|
+
submit_parser.add_argument("--endpoint", action="append", help="HTTPS endpoint; repeat for an ordered fallback list (default: the NoBlox host, then feedback.cmo.so)")
|
|
819
847
|
submit_parser.add_argument("--confirm", action="store_true")
|
|
820
848
|
list_parser = sub.add_parser("list")
|
|
821
849
|
list_parser.add_argument("--project", default=argparse.SUPPRESS)
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Record where API Pull deliveries went live, from the site's sitemap.
|
|
3
|
+
|
|
4
|
+
A delivered post is useful to other channels (social posts, internal links)
|
|
5
|
+
only once its live URL is known. The site should report it when it publishes;
|
|
6
|
+
this matcher fills the gaps from the sitemap with two rules:
|
|
7
|
+
|
|
8
|
+
* additive only: a record that already has a live URL keeps it, and a record
|
|
9
|
+
that matches nothing in this run is left as it was (never marked unmatched),
|
|
10
|
+
so the match can run on a schedule;
|
|
11
|
+
* the original page first: when translated copies share the slug
|
|
12
|
+
(``/blog/x`` and ``/zh-hans/blog/x``), the URL without a locale segment wins.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import argparse
|
|
17
|
+
import json
|
|
18
|
+
import re
|
|
19
|
+
import sys
|
|
20
|
+
import xml.etree.ElementTree as ET
|
|
21
|
+
from datetime import datetime, timezone
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from urllib.parse import urlsplit
|
|
24
|
+
from urllib.request import Request, urlopen
|
|
25
|
+
|
|
26
|
+
LOCALE_SEGMENT = re.compile(r"^[a-z]{2}(?:[-_][a-z0-9]{2,8})?$", re.I)
|
|
27
|
+
MAX_SITEMAPS = 50
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def now() -> str:
|
|
31
|
+
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def fetch(location: str) -> str:
|
|
35
|
+
if not location.startswith(("http://", "https://")):
|
|
36
|
+
return Path(location).read_text(encoding="utf-8")
|
|
37
|
+
with urlopen(Request(location, headers={"User-Agent": "MaggieLiveUrls/1.0"}), timeout=20) as response:
|
|
38
|
+
return response.read(20_000_000).decode("utf-8", "replace")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def sitemap_urls(location: str) -> list[str]:
|
|
42
|
+
"""Every <loc> of a sitemap, following sitemap indexes (bounded)."""
|
|
43
|
+
seen: set[str] = set()
|
|
44
|
+
urls: list[str] = []
|
|
45
|
+
queue = [location]
|
|
46
|
+
while queue and len(seen) < MAX_SITEMAPS:
|
|
47
|
+
current = queue.pop(0)
|
|
48
|
+
if current in seen:
|
|
49
|
+
continue
|
|
50
|
+
seen.add(current)
|
|
51
|
+
root = ET.fromstring(fetch(current))
|
|
52
|
+
tag = root.tag.rsplit("}", 1)[-1]
|
|
53
|
+
locs = [element.text.strip() for element in root.iter() if element.tag.rsplit("}", 1)[-1] == "loc" and element.text]
|
|
54
|
+
if tag == "sitemapindex":
|
|
55
|
+
queue.extend(locs)
|
|
56
|
+
else:
|
|
57
|
+
urls.extend(locs)
|
|
58
|
+
return list(dict.fromkeys(urls))
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def has_locale_segment(url: str) -> bool:
|
|
62
|
+
parts = [part for part in urlsplit(url).path.split("/") if part]
|
|
63
|
+
return len(parts) > 1 and bool(LOCALE_SEGMENT.match(parts[0]))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def best_match(slug: str, urls: list[str]) -> str | None:
|
|
67
|
+
candidates = [url for url in urls if [part for part in urlsplit(url).path.split("/") if part][-1:] == [slug]]
|
|
68
|
+
if not candidates:
|
|
69
|
+
return None
|
|
70
|
+
# Original before translated copies, then the shortest path.
|
|
71
|
+
return sorted(candidates, key=lambda url: (has_locale_segment(url), len(urlsplit(url).path), url))[0]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def match(state: dict, urls: list[str]) -> dict:
|
|
75
|
+
records = state.get("content", {}) if isinstance(state.get("content"), dict) else {}
|
|
76
|
+
matched, kept, missing = [], [], []
|
|
77
|
+
for key, record in records.items():
|
|
78
|
+
if not isinstance(record, dict):
|
|
79
|
+
continue
|
|
80
|
+
if record.get("live_url"):
|
|
81
|
+
kept.append({"remote_content_id": record.get("remote_content_id") or key, "live_url": record["live_url"], "source": record.get("live_url_source", "")})
|
|
82
|
+
continue
|
|
83
|
+
slug = str(record.get("slug") or "").strip("/")
|
|
84
|
+
url = best_match(slug, urls) if slug else None
|
|
85
|
+
if url:
|
|
86
|
+
record["live_url"] = url
|
|
87
|
+
record["live_url_source"] = "sitemap"
|
|
88
|
+
record["live_url_matched_at"] = now()
|
|
89
|
+
matched.append({"remote_content_id": record.get("remote_content_id") or key, "slug": slug, "live_url": url})
|
|
90
|
+
else:
|
|
91
|
+
missing.append({"remote_content_id": record.get("remote_content_id") or key, "slug": slug})
|
|
92
|
+
return {"matched": matched, "kept": kept, "notFound": missing}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def main() -> int:
|
|
96
|
+
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
97
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
98
|
+
run = sub.add_parser("match", help="fill missing live URLs of delivered posts from the sitemap (additive)")
|
|
99
|
+
run.add_argument("--project", default=".")
|
|
100
|
+
run.add_argument("--state", help="lifecycle state (default: .maggie/ai-cmo-lifecycle.json)")
|
|
101
|
+
run.add_argument("--sitemap", required=True, help="sitemap URL or file (indexes are followed)")
|
|
102
|
+
run.add_argument("--write", action="store_true", help="save the new live URLs into the state file")
|
|
103
|
+
run.add_argument("--report-body", help="write a report-state body for the newly matched posts (send with `maggie api report-state --body-file`)")
|
|
104
|
+
record = sub.add_parser("record", help="record a live URL the site reported when it published a delivered post")
|
|
105
|
+
record.add_argument("--project", default=".")
|
|
106
|
+
record.add_argument("--state")
|
|
107
|
+
record.add_argument("--content-id", required=True)
|
|
108
|
+
record.add_argument("--url", required=True)
|
|
109
|
+
args = parser.parse_args()
|
|
110
|
+
state_path = Path(args.state) if args.state else Path(args.project) / ".maggie" / "ai-cmo-lifecycle.json"
|
|
111
|
+
try:
|
|
112
|
+
state = json.loads(state_path.read_text(encoding="utf-8"))
|
|
113
|
+
if args.command == "record":
|
|
114
|
+
if urlsplit(args.url).scheme not in {"http", "https"}:
|
|
115
|
+
raise ValueError("--url must be an absolute http(s) URL")
|
|
116
|
+
target = state.get("content", {}).get(args.content_id)
|
|
117
|
+
if not isinstance(target, dict):
|
|
118
|
+
raise ValueError(f"no delivered post {args.content_id} in {state_path}")
|
|
119
|
+
# The site's own report is authoritative and replaces a sitemap guess.
|
|
120
|
+
target.update({"live_url": args.url, "live_url_source": "site", "live_url_matched_at": now()})
|
|
121
|
+
state_path.write_text(json.dumps(state, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
122
|
+
print(json.dumps({"status": "recorded", "remote_content_id": args.content_id, "live_url": args.url}, indent=2))
|
|
123
|
+
return 0
|
|
124
|
+
result = match(state, sitemap_urls(args.sitemap))
|
|
125
|
+
if args.write:
|
|
126
|
+
state_path.write_text(json.dumps(state, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
127
|
+
if args.report_body and result["matched"]:
|
|
128
|
+
body = {"items": [{"content_id": item["remote_content_id"], "url": item["live_url"], "status": "published"} for item in result["matched"]][:500]}
|
|
129
|
+
Path(args.report_body).write_text(json.dumps(body, indent=2) + "\n", encoding="utf-8")
|
|
130
|
+
print(json.dumps({"status": "matched" if args.write else "dry-run", **{key: value for key, value in result.items()}, "counts": {key: len(value) for key, value in result.items()}}, indent=2, ensure_ascii=False))
|
|
131
|
+
return 0
|
|
132
|
+
except (OSError, ValueError, json.JSONDecodeError, ET.ParseError) as error:
|
|
133
|
+
print(f"maggie live-urls: {error}", file=sys.stderr)
|
|
134
|
+
return 1
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
if __name__ == "__main__":
|
|
138
|
+
raise SystemExit(main())
|
|
@@ -17,6 +17,7 @@ from localization_runner import generate, process_adapter
|
|
|
17
17
|
|
|
18
18
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
19
19
|
import maggie_email_localization as email_localization
|
|
20
|
+
import maggie_localization_keys as localization_keys
|
|
20
21
|
|
|
21
22
|
|
|
22
23
|
def generate_job(args: argparse.Namespace) -> int:
|
|
@@ -399,6 +400,7 @@ def main() -> int:
|
|
|
399
400
|
for name in ("stale", "glossary"):
|
|
400
401
|
command = sub.add_parser(name); command.add_argument("--project", type=Path, default=Path("."))
|
|
401
402
|
email_localization.add_parsers(sub)
|
|
403
|
+
localization_keys.add_parsers(sub)
|
|
402
404
|
args = parser.parse_args()
|
|
403
405
|
try:
|
|
404
406
|
if args.command == "generate": return generate_job(args)
|
|
@@ -411,6 +413,7 @@ def main() -> int:
|
|
|
411
413
|
if args.command == "publish": return publish(args.job, args.confirm)
|
|
412
414
|
if args.command == "stale": return stale_jobs(args.project)
|
|
413
415
|
if args.command in ("email-matrix", "lifecycle"): return email_localization.run(args)
|
|
416
|
+
if args.command in ("key-check", "fragments"): return localization_keys.run(args)
|
|
414
417
|
return glossary(args.project)
|
|
415
418
|
except (OSError, ValueError, json.JSONDecodeError) as error:
|
|
416
419
|
print(f"ERROR: {error}", file=sys.stderr); return 1
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""Translation keys for generated pages: text keys are site-wide, positions are not.
|
|
2
|
+
|
|
3
|
+
A translation stored under its source text applies to every page that shows
|
|
4
|
+
that text. That is right for a shared header, wrong for generic phrases on
|
|
5
|
+
generated pages ("your practice" is a clinic on one page and a firm on
|
|
6
|
+
another). `key-check` reads per-page translation rows and lists the texts that
|
|
7
|
+
need position keys. `fragments` reads rendered HTML and lists text split
|
|
8
|
+
around an inline span (a styled brand name) into pieces too short to translate
|
|
9
|
+
on their own ("Can", "No.", "What").
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import json
|
|
15
|
+
import re
|
|
16
|
+
from html.parser import HTMLParser
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
INLINE = {"span", "strong", "em", "b", "i", "a", "mark", "small", "sup", "sub", "u"}
|
|
20
|
+
BLOCK = {"p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "td", "th", "dt", "dd", "figcaption", "blockquote", "button", "label", "summary", "caption"}
|
|
21
|
+
SKIP = {"script", "style", "noscript", "template", "svg", "code", "pre"}
|
|
22
|
+
WORD = re.compile(r"[\w'’-]+", re.UNICODE)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def load_rows(path: Path) -> list[dict]:
|
|
26
|
+
value = json.loads(path.read_text(encoding="utf-8"))
|
|
27
|
+
rows = value.get("rows", []) if isinstance(value, dict) else value
|
|
28
|
+
if not isinstance(rows, list):
|
|
29
|
+
raise ValueError("rows must be a list of {page, position?, text, translation}")
|
|
30
|
+
return [row for row in rows if isinstance(row, dict)]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def key_check(rows: list[dict], locale: str | None = None) -> dict:
|
|
34
|
+
"""Source texts whose translation differs between pages cannot share a text key."""
|
|
35
|
+
by_text: dict[str, dict[str, set[str]]] = {}
|
|
36
|
+
for row in rows:
|
|
37
|
+
if locale and row.get("locale") not in (None, locale):
|
|
38
|
+
continue
|
|
39
|
+
text = " ".join(str(row.get("text") or "").split())
|
|
40
|
+
translation = " ".join(str(row.get("translation") or "").split())
|
|
41
|
+
if not text or not translation:
|
|
42
|
+
continue
|
|
43
|
+
by_text.setdefault(text, {}).setdefault(translation, set()).add(str(row.get("page") or "?"))
|
|
44
|
+
conflicts = []
|
|
45
|
+
for text, translations in sorted(by_text.items()):
|
|
46
|
+
if len(translations) > 1:
|
|
47
|
+
pages = sorted({page for group in translations.values() for page in group})
|
|
48
|
+
conflicts.append({"text": text, "translations": {translation: sorted(group)[:5] for translation, group in translations.items()}, "pages": len(pages)})
|
|
49
|
+
shared = sum(1 for translations in by_text.values() if len({page for group in translations.values() for page in group}) > 1)
|
|
50
|
+
return {
|
|
51
|
+
"valid": not conflicts,
|
|
52
|
+
"texts": len(by_text),
|
|
53
|
+
"sharedAcrossPages": shared,
|
|
54
|
+
"needPositionKeys": len(conflicts),
|
|
55
|
+
"conflicts": conflicts[:200],
|
|
56
|
+
"advice": "Store these under per-page position keys (render the source page, then key each node by page + position with a source hash); keep text keys only as a fallback for genuinely shared text." if conflicts else "",
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class FragmentParser(HTMLParser):
|
|
61
|
+
"""Collect, per block element, the run of text pieces and inline spans."""
|
|
62
|
+
|
|
63
|
+
def __init__(self, brand: re.Pattern[str]) -> None:
|
|
64
|
+
super().__init__(convert_charrefs=True)
|
|
65
|
+
self.brand = brand
|
|
66
|
+
self.stack: list[str] = []
|
|
67
|
+
self.blocks: list[list[tuple[str, str]]] = [] # (kind, text): kind is "text" or "inline"
|
|
68
|
+
self.inline_depth = 0
|
|
69
|
+
self.inline_text = ""
|
|
70
|
+
self.skip = 0
|
|
71
|
+
self.findings: list[dict] = []
|
|
72
|
+
|
|
73
|
+
def handle_starttag(self, tag: str, attrs) -> None: # noqa: ANN001
|
|
74
|
+
if tag in SKIP:
|
|
75
|
+
self.skip += 1
|
|
76
|
+
return
|
|
77
|
+
if tag in BLOCK:
|
|
78
|
+
self.blocks.append([])
|
|
79
|
+
elif tag in INLINE and self.blocks:
|
|
80
|
+
if self.inline_depth == 0:
|
|
81
|
+
self.inline_text = ""
|
|
82
|
+
self.inline_depth += 1
|
|
83
|
+
|
|
84
|
+
def handle_endtag(self, tag: str) -> None:
|
|
85
|
+
if tag in SKIP:
|
|
86
|
+
self.skip = max(0, self.skip - 1)
|
|
87
|
+
return
|
|
88
|
+
if tag in INLINE and self.inline_depth:
|
|
89
|
+
self.inline_depth -= 1
|
|
90
|
+
if self.inline_depth == 0 and self.blocks:
|
|
91
|
+
self.blocks[-1].append(("inline", self.inline_text))
|
|
92
|
+
elif tag in BLOCK and self.blocks:
|
|
93
|
+
self.finish(self.blocks.pop())
|
|
94
|
+
|
|
95
|
+
def handle_data(self, data: str) -> None:
|
|
96
|
+
if self.skip or not self.blocks:
|
|
97
|
+
return
|
|
98
|
+
if self.inline_depth:
|
|
99
|
+
self.inline_text += data
|
|
100
|
+
elif data.strip():
|
|
101
|
+
self.blocks[-1].append(("text", data))
|
|
102
|
+
|
|
103
|
+
def finish(self, parts: list[tuple[str, str]]) -> None:
|
|
104
|
+
if not any(kind == "inline" and self.brand.search(text) for kind, text in parts):
|
|
105
|
+
return
|
|
106
|
+
sentence = " ".join(" ".join(text.split()) for _, text in parts).strip()
|
|
107
|
+
for index, (kind, text) in enumerate(parts):
|
|
108
|
+
if kind != "text":
|
|
109
|
+
continue
|
|
110
|
+
neighbours = [parts[i] for i in (index - 1, index + 1) if 0 <= i < len(parts)]
|
|
111
|
+
if not any(other_kind == "inline" and self.brand.search(other_text) for other_kind, other_text in neighbours):
|
|
112
|
+
continue
|
|
113
|
+
words = WORD.findall(text)
|
|
114
|
+
if 1 <= len(words) <= 2:
|
|
115
|
+
self.findings.append({"fragment": " ".join(text.split()), "words": len(words), "sentence": sentence[:200]})
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def fragments(paths: list[Path], brand: str) -> dict:
|
|
119
|
+
pattern = re.compile(re.escape(brand), re.I)
|
|
120
|
+
findings = []
|
|
121
|
+
for path in paths:
|
|
122
|
+
parser = FragmentParser(pattern)
|
|
123
|
+
parser.feed(path.read_text(encoding="utf-8", errors="replace"))
|
|
124
|
+
findings.extend({"file": str(path), **item} for item in parser.findings)
|
|
125
|
+
return {
|
|
126
|
+
"valid": not findings,
|
|
127
|
+
"files": len(paths),
|
|
128
|
+
"fragments": findings[:500],
|
|
129
|
+
"advice": f"Translate the whole sentence around {brand!r} as one unit (keep the span as a placeholder such as {{brand}}) instead of storing these short pieces." if findings else "",
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def add_parsers(sub: argparse._SubParsersAction) -> None:
|
|
134
|
+
keys = sub.add_parser("key-check", help="list source texts translated differently on different pages (they need per-page position keys)")
|
|
135
|
+
keys.add_argument("rows", type=Path, help='JSON rows [{"page", "position"?, "text", "translation", "locale"?}]')
|
|
136
|
+
keys.add_argument("--locale")
|
|
137
|
+
split = sub.add_parser("fragments", help="find text split around an inline brand span into pieces too short to translate")
|
|
138
|
+
split.add_argument("--html", type=Path, nargs="+", required=True, help="rendered HTML files or directories (searched for *.html)")
|
|
139
|
+
split.add_argument("--brand", required=True, help="the inline span's text, e.g. the brand name")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def run(args: argparse.Namespace) -> int:
|
|
143
|
+
if args.command == "key-check":
|
|
144
|
+
result = key_check(load_rows(args.rows), args.locale)
|
|
145
|
+
else:
|
|
146
|
+
files: list[Path] = []
|
|
147
|
+
for item in args.html:
|
|
148
|
+
files.extend(sorted(item.rglob("*.html")) if item.is_dir() else [item])
|
|
149
|
+
result = fragments(files, args.brand)
|
|
150
|
+
print(json.dumps(result, indent=2, ensure_ascii=False))
|
|
151
|
+
return 0 if result["valid"] else 1
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Generate page pictures with a chosen engine, and plan a large run first.
|
|
3
|
+
|
|
4
|
+
Engines:
|
|
5
|
+
|
|
6
|
+
* ``codex`` — the local ``codex`` CLI's image tool on the signed-in ChatGPT
|
|
7
|
+
plan. No per-image charge, but it draws from the plan's quota and is slow
|
|
8
|
+
(about 75 s per picture observed). Accepts reference pictures (``--image``).
|
|
9
|
+
* ``gemini`` — Gemini image generation with the project's own
|
|
10
|
+
``GEMINI_API_KEY`` (billed to that Google account). Faster (about 19 s at 2K
|
|
11
|
+
observed). No reference pictures in this tool.
|
|
12
|
+
|
|
13
|
+
``engine-plan`` states the time and cost of each engine for a run before it
|
|
14
|
+
starts; ``generate`` draws one picture and never runs without ``--confirm``.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import argparse
|
|
19
|
+
import base64
|
|
20
|
+
import json
|
|
21
|
+
import math
|
|
22
|
+
import os
|
|
23
|
+
import shutil
|
|
24
|
+
import subprocess
|
|
25
|
+
import sys
|
|
26
|
+
import tempfile
|
|
27
|
+
import time
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from urllib.error import HTTPError, URLError
|
|
30
|
+
from urllib.request import Request, urlopen
|
|
31
|
+
|
|
32
|
+
GEMINI_MODEL = "gemini-3.1-flash-image"
|
|
33
|
+
GEMINI_ENDPOINT = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent"
|
|
34
|
+
RATIOS = ("21:9", "16:9", "3:2", "4:3", "5:4", "1:1", "4:5", "3:4", "2:3", "9:16")
|
|
35
|
+
# Observed seconds per picture (one worker); override with --seconds-codex / --seconds-gemini.
|
|
36
|
+
OBSERVED_SECONDS = {"codex": 75.0, "gemini": 19.0}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def parse_size(value: str) -> tuple[int, int]:
|
|
40
|
+
try:
|
|
41
|
+
width, height = (int(part) for part in value.lower().split("x"))
|
|
42
|
+
except ValueError as error:
|
|
43
|
+
raise ValueError("--size must look like 2400x1000") from error
|
|
44
|
+
if width < 64 or height < 64:
|
|
45
|
+
raise ValueError("--size is too small")
|
|
46
|
+
return width, height
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def nearest_ratio(width: int, height: int) -> str:
|
|
50
|
+
target = width / height
|
|
51
|
+
return min(RATIOS, key=lambda ratio: abs(math.log((int(ratio.split(":")[0]) / int(ratio.split(":")[1])) / target)))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def image_size(width: int, height: int) -> str:
|
|
55
|
+
longest = max(width, height)
|
|
56
|
+
return "4K" if longest > 2600 else "2K" if longest > 1300 else "1K"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def engine_plan(count: int, width: int, height: int, workers: int, references: bool, seconds: dict[str, float], gemini_unit_cost: float | None, currency: str) -> dict:
|
|
60
|
+
engines = []
|
|
61
|
+
for engine in ("codex", "gemini"):
|
|
62
|
+
per = seconds[engine]
|
|
63
|
+
wall = math.ceil(count / max(1, workers)) * per
|
|
64
|
+
row = {"engine": engine, "secondsPerPicture": per, "workers": workers, "estimatedMinutes": round(wall / 60, 1)}
|
|
65
|
+
if engine == "codex":
|
|
66
|
+
row.update({"billing": "ChatGPT plan quota (no per-picture charge)", "quotaPictures": count, "cost": None, "referenceImages": True})
|
|
67
|
+
else:
|
|
68
|
+
row.update({"billing": "the project's GEMINI_API_KEY (its Google account)", "aspectRatio": nearest_ratio(width, height), "imageSize": image_size(width, height),
|
|
69
|
+
"cost": round(count * gemini_unit_cost, 2) if gemini_unit_cost is not None else None, "currency": currency if gemini_unit_cost is not None else None,
|
|
70
|
+
"costNote": None if gemini_unit_cost is not None else "pass --gemini-unit-cost from the current Gemini price list to state the cost", "referenceImages": False})
|
|
71
|
+
engines.append(row)
|
|
72
|
+
usable = [row for row in engines if not (references and not row["referenceImages"])]
|
|
73
|
+
fastest = min(usable, key=lambda row: row["estimatedMinutes"])["engine"] if usable else None
|
|
74
|
+
return {"schemaVersion": "maggie-media-engine-plan.v1", "pictures": count, "size": f"{width}x{height}", "needsReferenceImages": references, "engines": engines, "fastest": fastest,
|
|
75
|
+
"confirmBeforeRunning": True, "note": "State this plan to the person and wait for a yes before a large run."}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def gemini_generate(prompt: str, width: int, height: int, model: str, timeout: float) -> bytes:
|
|
79
|
+
key = os.environ.get("GEMINI_API_KEY", "")
|
|
80
|
+
if not key:
|
|
81
|
+
raise RuntimeError("--engine gemini needs GEMINI_API_KEY in the environment")
|
|
82
|
+
endpoint = os.environ.get("GEMINI_API_ENDPOINT") or GEMINI_ENDPOINT.format(model=model)
|
|
83
|
+
body = json.dumps({"contents": [{"role": "user", "parts": [{"text": prompt}]}],
|
|
84
|
+
"generationConfig": {"responseModalities": ["IMAGE"], "imageConfig": {"aspectRatio": nearest_ratio(width, height), "imageSize": image_size(width, height)}}}).encode()
|
|
85
|
+
for attempt in range(1, 5):
|
|
86
|
+
request = Request(endpoint, data=body, method="POST", headers={"content-type": "application/json", "x-goog-api-key": key})
|
|
87
|
+
status, data = 0, {}
|
|
88
|
+
try:
|
|
89
|
+
with urlopen(request, timeout=timeout) as response:
|
|
90
|
+
status, data = response.status, json.loads(response.read() or b"{}")
|
|
91
|
+
except HTTPError as error:
|
|
92
|
+
status = error.code
|
|
93
|
+
try:
|
|
94
|
+
data = json.loads(error.read() or b"{}")
|
|
95
|
+
except json.JSONDecodeError:
|
|
96
|
+
data = {}
|
|
97
|
+
except (URLError, TimeoutError, OSError) as error:
|
|
98
|
+
data = {"error": {"message": type(error).__name__}}
|
|
99
|
+
parts = (((data.get("candidates") or [{}])[0].get("content") or {}).get("parts") or [])
|
|
100
|
+
image = next((part["inlineData"]["data"] for part in parts if isinstance(part, dict) and part.get("inlineData", {}).get("data")), None)
|
|
101
|
+
if 200 <= status < 300 and image:
|
|
102
|
+
return base64.b64decode(image)
|
|
103
|
+
if attempt == 4 or (400 <= status < 500 and status != 429):
|
|
104
|
+
reason = (data.get("error") or {}).get("message") or ((data.get("candidates") or [{}])[0].get("finishReason")) or f"HTTP {status}"
|
|
105
|
+
raise RuntimeError(f"Gemini: {str(reason)[:300]}")
|
|
106
|
+
time.sleep(float(os.environ.get("MAGGIE_MEDIA_RETRY_SECONDS", attempt * 15)))
|
|
107
|
+
raise RuntimeError("Gemini: no image")
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def codex_generate(prompt: str, width: int, height: int, images: list[Path], timeout: float) -> bytes:
|
|
111
|
+
if not shutil.which("codex"):
|
|
112
|
+
raise RuntimeError("--engine codex needs the codex CLI on PATH (signed in to ChatGPT)")
|
|
113
|
+
shape = "wide landscape (about 3:2 or wider)" if width >= height * 1.4 else "landscape" if width > height else "portrait" if width < height else "square"
|
|
114
|
+
with tempfile.TemporaryDirectory(prefix="maggie-media-") as work:
|
|
115
|
+
reference = "Use the attached image only as a reference for layout and content. " if images else ""
|
|
116
|
+
instruction = f"Use your image generation tool to create exactly one image, {shape}. {reference}{prompt} Save the final image into the current directory as out.png. Reply only with the saved path."
|
|
117
|
+
# `-i` takes several files, so `--` ends the list before the prompt.
|
|
118
|
+
argv = ["codex", "exec", "--skip-git-repo-check", "--sandbox", "workspace-write", *[item for path in images for item in ("-i", str(path.resolve()))], *(["--"] if images else []), instruction]
|
|
119
|
+
subprocess.run(argv, cwd=work, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, timeout=timeout, check=False)
|
|
120
|
+
out = Path(work) / "out.png"
|
|
121
|
+
if not out.is_file():
|
|
122
|
+
raise RuntimeError("codex did not save out.png")
|
|
123
|
+
return out.read_bytes()
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def main(argv: list[str] | None = None) -> int:
|
|
127
|
+
parser = argparse.ArgumentParser(prog="maggie media", description=__doc__.splitlines()[0])
|
|
128
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
129
|
+
plan = sub.add_parser("engine-plan", help="time and cost of each engine for a run, before it starts")
|
|
130
|
+
plan.add_argument("--count", type=int, required=True)
|
|
131
|
+
plan.add_argument("--size", default="1600x1200")
|
|
132
|
+
plan.add_argument("--workers", type=int, default=1)
|
|
133
|
+
plan.add_argument("--reference-images", action="store_true", help="the run redraws screenshots (codex only)")
|
|
134
|
+
plan.add_argument("--seconds-codex", type=float, default=OBSERVED_SECONDS["codex"])
|
|
135
|
+
plan.add_argument("--seconds-gemini", type=float, default=OBSERVED_SECONDS["gemini"])
|
|
136
|
+
plan.add_argument("--gemini-unit-cost", type=float, help="price of one picture at this size from the current Gemini price list")
|
|
137
|
+
plan.add_argument("--currency", default="USD")
|
|
138
|
+
gen = sub.add_parser("generate", help="draw one picture with the chosen engine")
|
|
139
|
+
gen.add_argument("--engine", choices=("codex", "gemini"), default="codex")
|
|
140
|
+
gen.add_argument("--prompt", required=True)
|
|
141
|
+
gen.add_argument("--size", default="1600x1200")
|
|
142
|
+
gen.add_argument("--image", type=Path, action="append", default=[], help="reference picture (codex only, repeatable)")
|
|
143
|
+
gen.add_argument("--output", type=Path, required=True, help="PNG path to write")
|
|
144
|
+
gen.add_argument("--model", default=GEMINI_MODEL)
|
|
145
|
+
gen.add_argument("--timeout", type=float, default=600)
|
|
146
|
+
gen.add_argument("--confirm", action="store_true")
|
|
147
|
+
args = parser.parse_args(argv)
|
|
148
|
+
try:
|
|
149
|
+
width, height = parse_size(args.size)
|
|
150
|
+
if args.command == "engine-plan":
|
|
151
|
+
print(json.dumps(engine_plan(args.count, width, height, args.workers, args.reference_images, {"codex": args.seconds_codex, "gemini": args.seconds_gemini}, args.gemini_unit_cost, args.currency), indent=2))
|
|
152
|
+
return 0
|
|
153
|
+
if not args.confirm:
|
|
154
|
+
print("CONFIRMATION_REQUIRED: generating a picture uses plan quota (codex) or a paid API (gemini); rerun with --confirm", file=sys.stderr)
|
|
155
|
+
return 2
|
|
156
|
+
for path in args.image:
|
|
157
|
+
if not path.is_file():
|
|
158
|
+
raise ValueError(f"--image {path} not found")
|
|
159
|
+
if args.engine == "gemini" and args.image:
|
|
160
|
+
raise ValueError("--image needs the codex engine")
|
|
161
|
+
started = time.monotonic()
|
|
162
|
+
data = gemini_generate(args.prompt, width, height, args.model, args.timeout) if args.engine == "gemini" else codex_generate(args.prompt, width, height, args.image, args.timeout)
|
|
163
|
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
164
|
+
args.output.write_bytes(data)
|
|
165
|
+
print(json.dumps({"engine": args.engine, "output": str(args.output), "bytes": len(data), "seconds": round(time.monotonic() - started, 1), "aspectRatio": nearest_ratio(width, height) if args.engine == "gemini" else None, "imageSize": image_size(width, height) if args.engine == "gemini" else None,
|
|
166
|
+
"next": "resize/crop to the exact size and convert to WebP before use"}, indent=2))
|
|
167
|
+
return 0
|
|
168
|
+
except (OSError, ValueError, RuntimeError, subprocess.TimeoutExpired) as error:
|
|
169
|
+
print(f"BLOCKED: maggie media: {error}", file=sys.stderr)
|
|
170
|
+
return 1
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
if __name__ == "__main__":
|
|
174
|
+
raise SystemExit(main())
|