@topy-ai/maggie 0.7.47 → 0.7.49

package/bundled-tools/clis/maggie_service_import.py

@@ -0,0 +1,520 @@
+#!/usr/bin/env python3
+"""Provider-neutral contracts for long-running service imports.
+
+Commands (routed by ``maggie service``):
+
+- ``import-stages-validate``: validate a multi-stage import state snapshot
+  (source → enrichment/research → verification) — idempotent stage keys,
+  bounded progress and retries, heartbeat-based stale detection, cancellation
+  propagation, and out-of-order completion against superseded inputs.
+- ``import-readiness``: fail before a job is queued when the host worker lacks
+  Python modules, a browser/driver, crawler configuration (by env name only),
+  a UTF-8 runtime, or normalizable country labels. A rate-limit probe runs only
+  when explicitly requested.
+- ``fact-provenance``: merge per-field facts from the user and providers.
+  User-entered facts are authoritative; provider disagreement becomes a
+  reviewable conflict artifact instead of a silent overwrite, and one failing
+  provider never blocks the others.
+
+All commands are read-only apart from their explicit ``--output`` report.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+import locale
+import os
+import re
+import shutil
+import subprocess
+import sys
+import unicodedata
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlsplit
+from urllib.request import Request, urlopen
+
+STAGES_SCHEMA = "maggie-service-import-stages.v1"
+READINESS_SCHEMA = "maggie-service-import-readiness.v1"
+PROVENANCE_SCHEMA = "maggie-service-fact-provenance.v1"
+MERGE_SCHEMA = "maggie-service-fact-merge.v1"
+
+STAGE_NAMES = ("source", "enrichment", "research", "verification")
+DEFAULT_DEPENDENCIES = {"source": [], "enrichment": ["source"], "research": ["source"], "verification": ["enrichment", "research"]}
+STAGE_STATES = {"queued", "running", "succeeded", "failed", "skipped", "cancelled"}
+IMPORT_STATES = {"queued", "running", "succeeded", "partial", "failed", "cancelled", "stale"}
+SAFE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$")
+SAFE_ERROR = re.compile(r"^[a-z0-9][a-z0-9._-]{0,79}$")
+SAFE_SOURCE = re.compile(r"^[a-z][a-z0-9_-]{0,39}$")
+DEFAULT_PRECEDENCE = ["user", "fresha", "google", "website"]
+
+
+def _time(value: object, label: str, errors: list[str]) -> datetime | None:
+    if value in (None, ""):
+        return None
+    if not isinstance(value, str):
+        errors.append(f"{label} must be an ISO-8601 timestamp")
+        return None
+    try:
+        parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError:
+        errors.append(f"{label} must be an ISO-8601 timestamp")
+        return None
+    if parsed.tzinfo is None:
+        errors.append(f"{label} must include a UTC offset")
+        return None
+    return parsed
+
+
+def _read(path: Path) -> dict[str, Any]:
+    value = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(value, dict):
+        raise ValueError(f"{path.name} root must be an object")
+    return value
+
+
+# ---------------------------------------------------------------------------
+# D24: multi-stage import state
+
+
+def derive_import_state(stages: dict[str, dict], cancellation_requested: bool) -> str:
+    states = {name: str(stage.get("state")) for name, stage in stages.items()}
+    if cancellation_requested and not any(state in {"running", "queued"} for state in states.values()):
+        return "cancelled"
+    if all(state in {"succeeded", "skipped"} for state in states.values()):
+        return "succeeded"
+    exhausted = [name for name, stage in stages.items() if stage.get("state") == "failed" and int(stage.get("attempt") or 0) >= int(stage.get("maxAttempts") or 0)]
+    if exhausted and not any(state in {"running", "queued"} for state in states.values()):
+        return "partial" if any(state == "succeeded" for state in states.values()) else "failed"
+    if any(state in {"running", "succeeded", "failed"} for state in states.values()):
+        return "running"
+    return "queued"
+
+
+def validate_stages(payload: dict[str, Any], now: datetime | None = None) -> dict[str, Any]:
+    errors: list[str] = []; warnings: list[str] = []
+    if payload.get("schemaVersion") != STAGES_SCHEMA:
+        errors.append(f"schemaVersion must be {STAGES_SCHEMA}")
+    import_id = payload.get("importId")
+    if not isinstance(import_id, str) or not SAFE_ID.fullmatch(import_id):
+        errors.append("importId must be a safe identifier"); import_id = ""
+    if not isinstance(payload.get("idempotencyKey"), str) or not SAFE_ID.fullmatch(payload["idempotencyKey"]):
+        errors.append("idempotencyKey must be a safe identifier")
+    declared_state = payload.get("state")
+    if declared_state not in IMPORT_STATES:
+        errors.append("state is unsupported")
+    stale_after = payload.get("staleAfterSeconds")
+    if not isinstance(stale_after, int) or isinstance(stale_after, bool) or not 30 <= stale_after <= 86400:
+        errors.append("staleAfterSeconds must be an integer between 30 and 86400"); stale_after = 900
+    now = now or _time(payload.get("evaluatedAt"), "evaluatedAt", errors) or datetime.now(timezone.utc)
+    cancellation = payload.get("cancellation") if isinstance(payload.get("cancellation"), dict) else {}
+    cancel_requested = cancellation.get("requested") is True
+    cancel_at = _time(cancellation.get("requestedAt"), "cancellation.requestedAt", errors) if cancel_requested else None
+    if cancel_requested and cancel_at is None:
+        errors.append("cancellation.requestedAt is required when cancellation is requested")
+
+    raw = payload.get("stages")
+    if not isinstance(raw, list):
+        errors.append("stages must be an array"); raw = []
+    stages: dict[str, dict] = {}; keys: set[str] = set()
+    for index, stage in enumerate(raw):
+        label = f"stages[{index}]"
+        if not isinstance(stage, dict):
+            errors.append(f"{label} must be an object"); continue
+        name = stage.get("name")
+        if name not in STAGE_NAMES:
+            errors.append(f"{label}.name must be one of {', '.join(STAGE_NAMES)}"); continue
+        if name in stages:
+            errors.append(f"{label}.name {name!r} is duplicated"); continue
+        stages[name] = stage
+        key = stage.get("stageKey")
+        if not isinstance(key, str) or not SAFE_ID.fullmatch(key) or (import_id and not key.startswith(import_id + ":")):
+            errors.append(f"{name}.stageKey must be a safe identifier prefixed with '<importId>:'")
+        elif key in keys:
+            errors.append(f"{name}.stageKey is duplicated; retries must reuse, not mint, keys per stage")
+        else:
+            keys.add(key)
+        if stage.get("state") not in STAGE_STATES:
+            errors.append(f"{name}.state is unsupported")
+        attempt, maximum = stage.get("attempt"), stage.get("maxAttempts")
+        if not isinstance(attempt, int) or isinstance(attempt, bool) or attempt < 0 or not isinstance(maximum, int) or isinstance(maximum, bool) or maximum < 1 or attempt > maximum:
+            errors.append(f"{name}.attempt must be 0..maxAttempts and maxAttempts >= 1")
+        progress = stage.get("progress")
+        if not isinstance(progress, dict) or not all(isinstance(progress.get(k), int) and not isinstance(progress.get(k), bool) for k in ("done", "total")) or not 0 <= progress["done"] <= progress["total"]:
+            errors.append(f"{name}.progress must be {{done, total}} with 0 <= done <= total")
+        elif stage.get("state") == "succeeded" and progress["done"] != progress["total"]:
+            errors.append(f"{name} succeeded with incomplete progress {progress['done']}/{progress['total']}")
+        if stage.get("state") == "failed" and not (isinstance(stage.get("errorCode"), str) and SAFE_ERROR.fullmatch(stage["errorCode"])):
+            errors.append(f"{name}.errorCode is required for a failed stage")
+    missing = [name for name in STAGE_NAMES if name not in stages]
+    if missing:
+        errors.append("stages must include " + ", ".join(missing))
+
+    stale: list[str] = []; out_of_order: list[str] = []
+    times: dict[str, dict[str, datetime | None]] = {}
+    for name, stage in stages.items():
+        times[name] = {field: _time(stage.get(field), f"{name}.{field}", errors) for field in ("startedAt", "completedAt", "heartbeatAt")}
+    for name, stage in stages.items():
+        deps = stage.get("dependsOn", DEFAULT_DEPENDENCIES[name])
+        if not isinstance(deps, list) or any(dep not in STAGE_NAMES or dep == name for dep in deps):
+            errors.append(f"{name}.dependsOn must list other stage names"); deps = []
+        if name == "source" and deps:
+            errors.append("source cannot depend on another stage")
+        state = stage.get("state")
+        if state in {"running", "succeeded"}:
+            unmet = [dep for dep in deps if dep in stages and stages[dep].get("state") not in {"succeeded", "skipped"}]
+            if unmet:
+                errors.append(f"{name} is {state} before dependencies finished: {', '.join(unmet)}")
+            started = times[name]["startedAt"]
+            for dep in deps:
+                finished = times.get(dep, {}).get("completedAt")
+                if started and finished and started < finished:
+                    out_of_order.append(f"{name} started before {dep} completed")
+        if state == "running":
+            beat = times[name]["heartbeatAt"] or times[name]["startedAt"]
+            if beat is None:
+                errors.append(f"{name} is running without heartbeatAt/startedAt")
+            elif now - beat > timedelta(seconds=stale_after):
+                stale.append(name)
+        if state in {"succeeded", "failed"} and times[name]["completedAt"] is None:
+            errors.append(f"{name}.completedAt is required once the stage finished")
+        # A downstream stage records which source revision it consumed. A late
+        # result built from a superseded source revision must not be applied.
+        if name != "source" and state == "succeeded" and "source" in stages:
+            consumed = stage.get("inputRevision")
+            current = stages["source"].get("resultRevision")
+            if consumed is None or current is None:
+                errors.append(f"{name}.inputRevision and source.resultRevision are required to reject superseded results")
+            elif consumed != current and stage.get("resultDiscarded") is not True:
+                out_of_order.append(f"{name} applied a result from source revision {consumed!r}; current is {current!r}")
+        if cancel_requested and cancel_at and state == "succeeded" and times[name]["completedAt"] and times[name]["completedAt"] > cancel_at and stage.get("resultDiscarded") is not True:
+            errors.append(f"{name} completed after cancellation was requested; mark resultDiscarded instead of applying it")
+    if cancel_requested and cancellation.get("propagated") is True:
+        still = [name for name, stage in stages.items() if stage.get("state") in {"queued", "running"}]
+        if still:
+            errors.append("cancellation is marked propagated but stages are still active: " + ", ".join(still))
+    if out_of_order:
+        errors.append("out-of-order completion: " + "; ".join(out_of_order))
+
+    derived = derive_import_state(stages, cancel_requested) if not missing else None
+    if stale:
+        derived = "stale"
+        if declared_state != "stale":
+            errors.append("stale running stage(s) without heartbeat for more than staleAfterSeconds: " + ", ".join(stale) + "; mark the import stale and requeue with the same stageKey")
+        if not isinstance(payload.get("nextAction"), str) or not payload["nextAction"].strip():
+            errors.append("nextAction is required for a stale import")
+    if derived and declared_state in IMPORT_STATES and declared_state != derived:
+        errors.append(f"state {declared_state!r} does not match the stage-derived state {derived!r}")
+    if cancel_requested and not cancellation.get("propagated") and not any(stage.get("state") in {"queued", "running"} for stage in stages.values()):
+        warnings.append("cancellation requested and no stage is active; set cancellation.propagated")
+    progress_total = sum(stage["progress"]["total"] for stage in stages.values() if isinstance(stage.get("progress"), dict) and isinstance(stage["progress"].get("total"), int))
+    progress_done = sum(stage["progress"]["done"] for stage in stages.values() if isinstance(stage.get("progress"), dict) and isinstance(stage["progress"].get("done"), int))
+    return {
+        "schemaVersion": STAGES_SCHEMA + "-report", "passed": not errors, "importId": import_id or None,
+        "declaredState": declared_state, "derivedState": derived, "staleStages": stale,
+        "progress": {"done": progress_done, "total": progress_total, "ratio": round(progress_done / progress_total, 4) if progress_total else 0.0},
+        "stages": [{"name": name, "state": stages[name].get("state"), "attempt": stages[name].get("attempt"), "maxAttempts": stages[name].get("maxAttempts")} for name in STAGE_NAMES if name in stages],
+        "errors": errors, "warnings": warnings,
+    }
+
+
+# ---------------------------------------------------------------------------
+# D25: provider / crawler readiness
+
+COUNTRY_ALIASES = {
+    "GB": ["united kingdom", "uk", "u.k.", "great britain", "britain", "england", "scotland", "wales", "northern ireland", "vereinigtes königreich", "royaume-uni", "reino unido", "regno unito", "verenigd koninkrijk", "英国", "英國", "联合王国", "聯合王國"],
+    "IE": ["ireland", "republic of ireland", "irland", "irlande", "irlanda", "爱尔兰", "愛爾蘭"],
+    "US": ["united states", "united states of america", "usa", "u.s.", "us", "vereinigte staaten", "états-unis", "etats-unis", "estados unidos", "stati uniti", "美国", "美國"],
+    "CA": ["canada", "kanada", "加拿大"],
+    "AU": ["australia", "australien", "australie", "澳大利亚", "澳洲"],
+    "NZ": ["new zealand", "neuseeland", "nouvelle-zélande", "新西兰", "紐西蘭"],
+    "DE": ["germany", "deutschland", "allemagne", "alemania", "germania", "德国", "德國"],
+    "FR": ["france", "frankreich", "francia", "法国", "法國"],
+    "ES": ["spain", "españa", "espana", "spanien", "espagne", "spagna", "西班牙"],
+    "IT": ["italy", "italia", "italien", "italie", "意大利", "義大利"],
+    "NL": ["netherlands", "the netherlands", "nederland", "niederlande", "pays-bas", "países bajos", "荷兰", "荷蘭"],
+    "CN": ["china", "people's republic of china", "中国", "中國", "中华人民共和国"],
+    "HK": ["hong kong", "hong kong sar", "香港"],
+    "TW": ["taiwan", "台灣", "臺灣", "台湾"],
+    "SG": ["singapore", "新加坡"],
+    "JP": ["japan", "日本"],
+    "AE": ["united arab emirates", "uae", "阿联酋", "阿聯酋"],
+}
+_ALIAS_INDEX = {alias: code for code, aliases in COUNTRY_ALIASES.items() for alias in aliases}
+
+
+def repair_mojibake(value: str) -> str:
+    """Undo UTF-8 text that was decoded as Latin-1/CP1252 (e.g. ``Königreich``)."""
+    if not re.search(r"[ÃÂâ€]", value):
+        return value
+    for codec in ("cp1252", "latin-1"):
+        try:
+            repaired = value.encode(codec).decode("utf-8")
+        except (UnicodeEncodeError, UnicodeDecodeError):
+            continue
+        if repaired != value:
+            return repaired
+    return value
+
+
+def normalize_country(label: str) -> dict[str, Any]:
+    original = label
+    repaired = repair_mojibake(label)
+    text = unicodedata.normalize("NFC", repaired).strip()
+    simplified = re.sub(r"\s+", " ", text.casefold().replace("’", "'"))
+    code = None
+    if re.fullmatch(r"[a-z]{2}", simplified) and simplified.upper() in COUNTRY_ALIASES:
+        code = simplified.upper()
+    code = code or _ALIAS_INDEX.get(simplified)
+    if code is None:
+        try:
+            import pycountry  # type: ignore[import-not-found]
+
+            match = pycountry.countries.lookup(text)
+            code = match.alpha_2
+        except Exception:  # noqa: BLE001 - optional dependency, any lookup failure is "unknown"
+            code = None
+    return {"input": original, "repaired": repaired != original, "normalized": text, "iso2": code, "passed": code is not None}
+
+
+def _which_browser() -> tuple[str | None, str | None]:
+    for name in ("google-chrome", "google-chrome-stable", "chromium", "chromium-browser", "chrome", "firefox"):
+        path = shutil.which(name)
+        if path:
+            return name, path
+    for env in ("CHROME_PATH", "CHROME_BIN", "PUPPETEER_EXECUTABLE_PATH"):
+        value = os.environ.get(env, "")
+        if value and Path(value).is_file() and os.access(value, os.X_OK):
+            return env, value
+    return None, None
+
+
+def _version(binary: str) -> str | None:
+    try:
+        proc = subprocess.run([binary, "--version"], capture_output=True, text=True, timeout=15, check=False)
+    except (OSError, subprocess.SubprocessError):
+        return None
+    text = (proc.stdout or proc.stderr).strip()
+    return text[:120] if proc.returncode == 0 and text else None
+
+
+def _major(version: str | None) -> int | None:
+    match = re.search(r"(\d+)\.", version or "")
+    return int(match.group(1)) if match else None
+
+
+def readiness(modules: list[str], browser: bool, webdriver: bool, env_names: list[str], countries: list[str], probe_url: str | None, probe_timeout: float) -> dict[str, Any]:
+    checks: list[dict[str, Any]] = []
+
+    def add(check: str, passed: bool, detail: str, remediation: str = "") -> None:
+        checks.append({"check": check, "passed": passed, "detail": detail, **({"remediation": remediation} if not passed and remediation else {})})
+
+    for module in modules:
+        found = importlib.util.find_spec(module) is not None
+        add(f"python:{module}", found, "importable" if found else "not importable", f"install it in the worker environment (e.g. `python3 -m pip install {module}`) and restart the worker")
+    browser_version = None
+    if browser or webdriver:
+        name, path = _which_browser()
+        browser_version = _version(path) if path else None
+        add("browser", bool(path and browser_version), f"{name}: {browser_version}" if browser_version else ("found but `--version` failed" if path else "no Chrome/Chromium/Firefox executable found"), "install Chrome or Chromium on the worker host, or set CHROME_PATH to the executable")
+    if webdriver:
+        driver = shutil.which("chromedriver") or shutil.which("geckodriver")
+        driver_version = _version(driver) if driver else None
+        add("webdriver", bool(driver_version), driver_version or "no chromedriver/geckodriver on PATH", "install the driver matching the browser major version, or use Selenium Manager with network access")
+        if driver_version and browser_version and "chrome" in (driver or "") and _major(driver_version) != _major(browser_version):
+            add("webdriver-version", False, f"driver major {_major(driver_version)} != browser major {_major(browser_version)}", "install the chromedriver release that matches the installed Chrome major version")
+    for name in env_names:
+        if not re.fullmatch(r"[A-Z][A-Z0-9_]{1,63}", name):
+            add(f"env:{name}", False, "invalid environment variable name", "use an upper-case variable name")
+            continue
+        present = bool(os.environ.get(name, "").strip())
+        add(f"env:{name}", present, "set" if present else "missing or empty", f"configure {name} in the worker's secret store; values are never printed")
+    encoding = (locale.getpreferredencoding(False) or "").lower().replace("-", "")
+    fs_encoding = (sys.getfilesystemencoding() or "").lower().replace("-", "")
+    utf8 = encoding == "utf8" and fs_encoding == "utf8"
+    add("runtime-encoding", utf8, f"preferred={locale.getpreferredencoding(False)} filesystem={sys.getfilesystemencoding()}", "run the worker with LANG/LC_ALL set to a UTF-8 locale (e.g. C.UTF-8) or PYTHONUTF8=1")
+    normalized = [normalize_country(label) for label in countries]
+    for item in normalized:
+        add(f"country:{item['input']}", item["passed"], f"→ {item['iso2']}" + (" (repaired encoding)" if item["repaired"] else "") if item["passed"] else f"cannot map {item['normalized']!r} to an ISO 3166 code", "store ISO 3166-1 alpha-2 codes; translate locale display names before queuing")
+    probe = None
+    if probe_url:
+        if urlsplit(probe_url).scheme not in {"http", "https"}:
+            add("rate-limit-probe", False, "probe URL must be http(s)")
+        else:
+            try:
+                with urlopen(Request(probe_url, method="HEAD", headers={"User-Agent": "maggie-service-import-readiness/1"}), timeout=probe_timeout) as response:
+                    status = response.status; headers = response.headers
+            except HTTPError as error:
+                status = error.code; headers = error.headers
+            except (URLError, TimeoutError, OSError) as error:
+                status = None; headers = None
+                add("rate-limit-probe", False, f"unreachable: {getattr(error, 'reason', error)}", "check worker egress, DNS and proxy settings")
+            if status is not None:
+                retry_after = headers.get("Retry-After") if headers else None
+                remaining = (headers.get("X-RateLimit-Remaining") or headers.get("RateLimit-Remaining")) if headers else None
+                probe = {"status": status, "retryAfter": retry_after, "remaining": remaining}
+                limited = status == 429 or (remaining is not None and str(remaining).strip() == "0")
+                add("rate-limit-probe", not limited and status < 500, f"HTTP {status}" + (f", Retry-After {retry_after}" if retry_after else ""), "wait for the provider window to reset or lower worker concurrency before queuing")
+    failed = [item["check"] for item in checks if not item["passed"]]
+    return {"schemaVersion": READINESS_SCHEMA, "passed": not failed, "queueAllowed": not failed, "failedChecks": failed, "checks": checks, "countries": normalized, "rateLimitProbe": probe if probe_url else "not requested", "mutation": "not executed"}
+
+
+# ---------------------------------------------------------------------------
+# D26: field-level provenance and conflict resolution
+
+
+def _norm_value(value: Any) -> str:
+    if isinstance(value, str):
+        text = unicodedata.normalize("NFC", value).strip()
+        if re.match(r"^https?://", text, re.I):
+            parts = urlsplit(text)
+            return f"{parts.scheme.lower()}://{parts.netloc.lower().removeprefix('www.')}{parts.path.rstrip('/')}"
+        return re.sub(r"\s+", " ", text).casefold()
+    return json.dumps(value, sort_keys=True, ensure_ascii=False)
+
+
+def merge_facts(payload: dict[str, Any]) -> dict[str, Any]:
+    errors: list[str] = []
+    if payload.get("schemaVersion") != PROVENANCE_SCHEMA:
+        errors.append(f"schemaVersion must be {PROVENANCE_SCHEMA}")
+    policy = payload.get("policy") if isinstance(payload.get("policy"), dict) else {}
+    precedence = policy.get("precedence") or DEFAULT_PRECEDENCE
+    if not isinstance(precedence, list) or precedence[:1] != ["user"]:
+        errors.append("policy.precedence must start with 'user'; user-entered facts are authoritative")
+        precedence = DEFAULT_PRECEDENCE
+    field_precedence = policy.get("fieldPrecedence") if isinstance(policy.get("fieldPrecedence"), dict) else {}
+    for field, order in field_precedence.items():
+        if not isinstance(order, list) or order[:1] != ["user"]:
+            errors.append(f"policy.fieldPrecedence.{field} must start with 'user'")
+    min_confidence = policy.get("minConfidence", 0.6)
+    if not isinstance(min_confidence, (int, float)) or not 0 <= min_confidence <= 1:
+        errors.append("policy.minConfidence must be between 0 and 1"); min_confidence = 0.6
+    fields = payload.get("fields")
+    if not isinstance(fields, dict) or not fields:
+        errors.append("fields must map field paths to provenance records"); fields = {}
+    resolved: dict[str, Any] = {}; conflicts: list[dict[str, Any]] = []; unavailable: set[str] = set(); low_confidence: list[str] = []
+    for field, record in sorted(fields.items()):
+        if not isinstance(record, dict):
+            errors.append(f"fields.{field} must be an object"); continue
+        order = field_precedence.get(field) if isinstance(field_precedence.get(field), list) else precedence
+        rank = {source: index for index, source in enumerate(order)}
+        user = record.get("user")
+        candidates: list[dict[str, Any]] = []
+        for index, entry in enumerate(record.get("sources") or []):
+            label = f"fields.{field}.sources[{index}]"
+            if not isinstance(entry, dict) or not isinstance(entry.get("source"), str) or not SAFE_SOURCE.fullmatch(entry["source"]) or entry["source"] == "user":
+                errors.append(f"{label}.source must be a provider id other than 'user'"); continue
+            if entry.get("error"):
+                if not (isinstance(entry["error"], str) and SAFE_ERROR.fullmatch(entry["error"])):
+                    errors.append(f"{label}.error must be a short error code")
+                unavailable.add(entry["source"]); continue
+            confidence = entry.get("confidence")
+            if not isinstance(confidence, (int, float)) or isinstance(confidence, bool) or not 0 <= confidence <= 1:
+                errors.append(f"{label}.confidence must be between 0 and 1"); continue
+            if "value" not in entry:
+                errors.append(f"{label}.value is required"); continue
+            if entry.get("sourceRef") is not None and (not isinstance(entry["sourceRef"], str) or "?" in entry["sourceRef"] and re.search(r"(?:key|token|sig|signature)=", entry["sourceRef"], re.I)):
+                errors.append(f"{label}.sourceRef must not carry credentials")
+            candidates.append({"source": entry["source"], "value": entry["value"], "confidence": float(confidence), "observedAt": entry.get("observedAt"), "sourceRef": entry.get("sourceRef")})
+        if isinstance(user, dict) and "value" in user:
+            disagree = sorted({item["source"] for item in candidates if _norm_value(item["value"]) != _norm_value(user["value"])})
+            resolved[field] = {"value": user["value"], "source": "user", "confidence": 1.0, "decidedBy": "user-authoritative", "providerDisagreement": disagree}
+            if disagree:
+                conflicts.append({"field": field, "status": "resolved-user-authoritative", "chosen": "user", "candidates": candidates, "requiresReview": False})
+            continue
+        eligible = [item for item in candidates if item["confidence"] >= min_confidence]
+        if not eligible:
+            if candidates:
+                low_confidence.append(field)
+            resolved[field] = {"value": None, "source": None, "confidence": 0.0, "decidedBy": "no-eligible-source"}
+            continue
+        eligible.sort(key=lambda item: (rank.get(item["source"], len(rank)), -item["confidence"]))
+        groups = {_norm_value(item["value"]) for item in eligible}
+        top = eligible[0]
+        if len(groups) == 1:
+            resolved[field] = {"value": top["value"], "source": top["source"], "confidence": max(item["confidence"] for item in eligible), "decidedBy": "agreement" if len(eligible) > 1 else "single-source", "corroboratedBy": sorted({item["source"] for item in eligible} - {top["source"]})}
+        else:
+            resolved[field] = {"value": top["value"], "source": top["source"], "confidence": top["confidence"], "decidedBy": "precedence-pending-review"}
+            conflicts.append({"field": field, "status": "needs-review", "proposed": top["source"], "precedence": order, "candidates": eligible, "requiresReview": True})
+    needs_review = [item["field"] for item in conflicts if item["requiresReview"]]
+    return {
+        "schemaVersion": MERGE_SCHEMA, "entityId": payload.get("entityId"),
+        "passed": not errors and not needs_review, "valid": not errors,
+        "resolved": resolved, "conflicts": conflicts, "needsReview": needs_review,
+        "unavailableSources": sorted(unavailable), "lowConfidenceFields": low_confidence,
+        "policy": {"precedence": precedence, "fieldPrecedence": field_precedence, "minConfidence": min_confidence},
+        "errors": errors, "mutation": "not executed",
+    }
+
+
+# ---------------------------------------------------------------------------
+
+
+def _emit(report: dict[str, Any], output: Path | None, project: Path) -> None:
+    if output:
+        target = output if output.is_absolute() else project / output
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+        report = {**report, "output": str(target)}
+    print(json.dumps(report, indent=2, ensure_ascii=False))
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="maggie service", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    sub = parser.add_subparsers(dest="command", required=True)
+    stages = sub.add_parser("import-stages-validate", help="validate a multi-stage import state snapshot")
+    stages.add_argument("--manifest", type=Path, required=True)
+    stages.add_argument("--now", help="evaluation time (ISO-8601); defaults to evaluatedAt or the current time")
+    ready = sub.add_parser("import-readiness", help="fail before queuing when the import worker is not ready")
+    ready.add_argument("--requirements", type=Path, help="JSON with pythonModules, browser, webdriver, env, countries")
+    ready.add_argument("--python-module", action="append", default=[])
+    ready.add_argument("--browser", action="store_true", help="require a headless-capable browser")
+    ready.add_argument("--webdriver", action="store_true", help="require a matching WebDriver (Selenium crawlers)")
+    ready.add_argument("--env", action="append", default=[], help="required environment variable name (value never printed)")
+    ready.add_argument("--country", action="append", default=[], help="country label to normalize to ISO 3166")
+    ready.add_argument("--rate-limit-probe", metavar="URL", help="opt-in single HEAD request to read rate-limit headers")
+    ready.add_argument("--probe-timeout", type=float, default=10.0)
+    facts = sub.add_parser("fact-provenance", help="merge user and provider facts with per-field provenance")
+    facts.add_argument("--facts", type=Path, required=True)
+    for command in (stages, ready, facts):
+        command.add_argument("--project", type=Path, default=Path.cwd())
+        command.add_argument("--output", type=Path)
+    args = parser.parse_args(argv)
+    project = args.project.resolve()
+    try:
+        if args.command == "import-stages-validate":
+            errors: list[str] = []
+            now = _time(args.now, "--now", errors) if args.now else None
+            if errors:
+                raise ValueError("; ".join(errors))
+            manifest = args.manifest if args.manifest.is_absolute() else project / args.manifest
+            report = validate_stages(_read(manifest), now)
+        elif args.command == "import-readiness":
+            spec: dict[str, Any] = {}
+            if args.requirements:
+                spec = _read(args.requirements if args.requirements.is_absolute() else project / args.requirements)
+            report = readiness(
+                list(spec.get("pythonModules", [])) + args.python_module,
+                bool(spec.get("browser")) or args.browser,
+                bool(spec.get("webdriver")) or args.webdriver,
+                list(spec.get("env", [])) + args.env,
+                list(spec.get("countries", [])) + args.country,
+                args.rate_limit_probe or spec.get("rateLimitProbe"),
+                args.probe_timeout,
+            )
+        else:
+            facts_path = args.facts if args.facts.is_absolute() else project / args.facts
+            report = merge_facts(_read(facts_path))
+    except (OSError, ValueError, json.JSONDecodeError) as error:
+        print(f"BLOCKED: maggie service {args.command}: {error}", file=sys.stderr)
+        return 2
+    _emit(report, args.output, project)
+    return 0 if report["passed"] else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())