@daemux/store-automator 0.10.94 → 0.10.96

package/templates/scripts/ci/ios-native/asc_version_create.py

@@ -0,0 +1,365 @@
+#!/usr/bin/env python3
+# Note: this file has 11 functions (one over the 10-per-file cap) and a
+# ~91-line helper. Pre-existing in the source-of-truth; refactor (likely
+# a split of the 409-retry branches into their own module) tracked
+# separately to avoid bundling unrelated edits into round-13 trims.
+"""
+App Store Connect appStoreVersions POST with classified-409 retry.
+
+Split out of manage_marketing_version.py so the main module stays under
+the 400-line cap. This module owns:
+
+  - ``create_version``: thin POST wrapper that logs the full request body
+    to stderr before the call and returns the raw response with 409
+    allowed so the caller can inspect headers/body.
+  - ``create_or_reuse``: orchestrates the POST, the happy-path 2xx -> id
+    extraction, the 409 reconcile (re-fetch both /appStoreVersions and
+    /preReleaseVersions for a matching versionString), and classified
+    409 handling:
+      * reusable id in /appStoreVersions         -> REUSE
+      * version-collision 409 (CONFLICT.VERSION_EXISTS, ENTITY_ERROR.
+        UNIQUENESS.VERSION_STRING, or detail mentions versionString)
+        with no reusable id                       -> bump + retry
+      * any other 409 (RELATIONSHIP.INVALID,
+        ATTRIBUTE.INVALID, unknown codes)         -> fail fast with
+        SystemExit(4) and the ASC source.pointer quoted in stderr
+
+Why the classifier (CI run 24640760698)? 21 consecutive POSTs returned
+409 ENTITY_ERROR.RELATIONSHIP.INVALID as the script bumped 1.0.6 ->
+1.2.6. That is a payload bug, not a version collision -- bumping cannot
+fix it. Fail fast instead so the next CI run surfaces the real error
+with the ASC-reported JSON pointer intact.
+
+Exit codes:
+  3 -- retry cap hit (20 consecutive version-collision 409s)
+  4 -- non-version 409 (payload bug; inspect logged pointer)
+  5 -- stale editable cannot be renamed or deleted (CI run 24640907316):
+       see ``asc_version_reuse.reuse_stale_editable``.
+"""
+
+from __future__ import annotations
+
+import json as _json
+import sys
+
+from asc_common import request
+from asc_version_reuse import (
+    delete_version,
+    is_editable_exists_409,
+    patch_version,
+    reuse_stale_editable,
+)
+
+# Re-exported so callers that historically imported these from
+# ``asc_version_create`` continue to work after the split into
+# ``asc_version_reuse``. This is also the seam tests patch via
+# ``mock.patch.object(asc_version_create, "patch_version", ...)``.
+__all__ = (
+    "create_or_reuse",
+    "create_version",
+    "delete_version",
+    "is_editable_exists_409",
+    "patch_version",
+    "reuse_stale_editable",
+)
+
+
+_RETRY_CAP = 20
+
+# ASC error codes that mean "this versionString already exists" and thus
+# a bump-and-retry is the right response. The list is best-effort -- ASC
+# has not published a canonical enum. Confirmed from historic CI logs
+# and Apple's forum posts.
+_VERSION_COLLISION_CODES = {
+    "CONFLICT.VERSION_EXISTS",
+    "ENTITY_ERROR.UNIQUENESS.VERSION_STRING",
+    "ENTITY_ERROR.CONFLICT",  # legacy -- seen in run 24640430898
+}
+
+# Substrings in error.detail that indicate a version-string collision
+# when the `code` field is absent (older ASC responses). Case-insensitive
+# match against the stripped detail string.
+_VERSION_COLLISION_DETAIL_HINTS = (
+    "versionstring already exists",
+    "version with this versionstring",
+    "version already exists",
+)
+
+
+def _log_request_body(body: dict) -> None:
+    """Pretty-print the POST body to stderr before the call so CI logs
+    have the exact request even if the response never arrives."""
+    try:
+        pretty = _json.dumps(body, indent=2, sort_keys=True)
+    except (TypeError, ValueError):
+        pretty = repr(body)
+    print(f"[create-version] request body: {pretty}", file=sys.stderr)
+
+
+def create_version(app_id: str, version: str, token: str):
+    """POST /appStoreVersions for this app + versionString. 409 allowed
+    so the caller can inspect the body + classify the failure."""
+    body = {
+        "data": {
+            "type": "appStoreVersions",
+            "attributes": {
+                "platform": "IOS",
+                "versionString": version,
+                "releaseType": "AFTER_APPROVAL",
+            },
+            "relationships": {
+                "app": {"data": {"type": "apps", "id": str(app_id)}},
+            },
+        }
+    }
+    _log_request_body(body)
+    return request(
+        "POST", "/appStoreVersions", token,
+        json_body=body, allow_status={409},
+    )
+
+
+def _parse_errors(resp) -> list[dict]:
+    """Extract the errors[] array from an ASC error response. Returns []
+    if the body isn't JSON or lacks `errors`."""
+    text = getattr(resp, "text", "") or ""
+    if not isinstance(text, str) or not text.strip():
+        return []
+    try:
+        parsed = _json.loads(text)
+    except (ValueError, TypeError):
+        return []
+    errs = parsed.get("errors") if isinstance(parsed, dict) else None
+    return errs if isinstance(errs, list) else []
+
+
+def _error_pointers(errors: list[dict]) -> list[str]:
+    """Collect every source.pointer across errors. Empty list if none."""
+    out: list[str] = []
+    for e in errors:
+        src = e.get("source") if isinstance(e, dict) else None
+        ptr = src.get("pointer") if isinstance(src, dict) else None
+        if isinstance(ptr, str) and ptr:
+            out.append(ptr)
+    return out
+
+
+def _is_version_collision(errors: list[dict]) -> bool:
+    """True iff the 409 is attributable to a version-string collision.
+    Only then is bump-and-retry the right response."""
+    if not errors:
+        # No parseable body -- historic ASC responses behaved this way.
+        # Stay back-compatible: treat as collision (the only known reason
+        # we ever saw 409 here before run 24640760698).
+        return False
+    for e in errors:
+        if not isinstance(e, dict):
+            continue
+        code = (e.get("code") or "").strip()
+        if code in _VERSION_COLLISION_CODES:
+            return True
+        detail = (e.get("detail") or "").strip().lower()
+        if any(hint in detail for hint in _VERSION_COLLISION_DETAIL_HINTS):
+            return True
+    return False
+
+
+def _dump_409(resp, version: str) -> list[dict]:
+    """Log the full 409 response (headers + body, untruncated) + any
+    source.pointer values. Returns the parsed errors[] so the caller
+    doesn't have to re-parse. Defensive against Mock-style responses
+    that may not carry dict-like headers or a real str body."""
+    try:
+        headers = dict(resp.headers)
+    except Exception:  # noqa: BLE001 - resp may not carry real headers
+        headers = {}
+    body = resp.text if isinstance(getattr(resp, "text", None), str) else ""
+    print(
+        f"[create-version] response status=409 for {version}; "
+        f"headers={headers}",
+        file=sys.stderr,
+    )
+    # Full body -- no truncation. The 2KB cap used to hide the offending
+    # source.pointer past byte 2000 in some ASC responses.
+    print(
+        f"[create-version] response body: {body}",
+        file=sys.stderr,
+    )
+    errors = _parse_errors(resp)
+    pointers = _error_pointers(errors)
+    if pointers:
+        print(
+            f"[create-version] error pointers: {pointers}",
+            file=sys.stderr,
+        )
+    return errors
+
+
+def _fail_fast_non_version_409(
+    resp, version: str, errors: list[dict], attempted: list[str],
+) -> None:
+    """SystemExit(4) with a clear diagnostic when the 409 is NOT a
+    version collision. Bumping cannot fix payload/relationship bugs."""
+    codes = [
+        (e.get("code") or "").strip()
+        for e in errors if isinstance(e, dict)
+    ]
+    pointers = _error_pointers(errors)
+    print(
+        f"::error::POST /appStoreVersions returned 409 for {version} "
+        f"with non-version-collision error(s); bumping would not fix "
+        f"this. codes={codes} pointers={pointers}. Inspect the full "
+        f"response body above, fix the request shape, and retry. "
+        f"(attempted={attempted})",
+        file=sys.stderr,
+    )
+    raise SystemExit(4)
+
+
+def _safe_fetch(label: str, fn, *args) -> list:
+    """Call fn(*args); on any exception, log + return []."""
+    try:
+        return list(fn(*args) or [])
+    except Exception as exc:  # noqa: BLE001 - defensive only
+        print(f"[409-retry] {label} re-fetch failed: {exc!r}", file=sys.stderr)
+        return []
+
+
+def _reconcile_409(
+    app_id: str, token: str, version: str,
+    fetch_versions, fetch_prerelease_versions, fetch_builds_prerelease_versions,
+) -> tuple[str | None, bool]:
+    """Re-query all three collections after a 409. Returns
+    (reusable_id_or_None, is_known_on_secondary).
+
+    - reusable_id: the appStoreVersions row to reuse (caller returns it).
+    - is_known_on_secondary: True if the version is visible on
+      /preReleaseVersions or builds->preReleaseVersion but NOT on
+      /appStoreVersions; caller should bump.
+    """
+    rows = _safe_fetch("/appStoreVersions", fetch_versions, app_id, token)
+    for v in rows:
+        if isinstance(v, dict) and v.get("versionString") == version and v.get("id"):
+            return v["id"], False
+
+    known = set()
+    known.update(
+        vs for vs in _safe_fetch(
+            "/preReleaseVersions", fetch_prerelease_versions, app_id, token,
+        ) if vs
+    )
+    known.update(
+        vs for vs in _safe_fetch(
+            "builds->preReleaseVersion",
+            fetch_builds_prerelease_versions, app_id, token,
+        ) if vs
+    )
+    return None, version in known
+
+
+def _log_retry_reason(version: str, known_on_secondary: bool) -> None:
+    if known_on_secondary:
+        msg = (
+            f"[409-retry] {version} found on a secondary collection "
+            f"but not in /appStoreVersions; bumping"
+        )
+    else:
+        msg = (
+            f"[409-retry] {version} not found on /appStoreVersions, "
+            f"/preReleaseVersions, or builds->preReleaseVersion; "
+            f"assuming hidden ASC record and bumping"
+        )
+    print(msg, file=sys.stderr)
+
+
+def create_or_reuse(
+    app_id: str, version: str, token: str,
+    *,
+    bump_fn,
+    fetch_versions,
+    fetch_prerelease_versions,
+    fetch_builds_prerelease_versions,
+    post_fn=None,
+    stale_editable_id: str | None = None,
+    patch_fn=None,
+    delete_fn=None,
+) -> str:
+    """POST a new appStoreVersion; on 409 either reuse an existing id,
+    bump-and-retry (only for true version collisions), or fail fast
+    (non-version 409s).
+
+    Deps are injected so callers (and tests) decide which semver bumper,
+    POST function, and which fetchers to use. ``post_fn`` defaults to
+    this module's ``create_version`` when unset.
+
+    Callers can disarm the bump-retry path entirely by passing a
+    ``bump_fn`` that raises (e.g. ``SystemExit``). manage_marketing_version
+    does this so a hidden-record 409 never invents a new version --
+    the project's MARKETING_VERSION is the single source of truth.
+
+    When ``stale_editable_id`` is provided (caller detected an editable
+    appStoreVersion at-or-below highest_shipped), the first attempt uses
+    the PATCH-reuse path (``reuse_stale_editable``) instead of POST.
+    Falls back to the classic POST loop only on an unexpected flow.
+    """
+    post = post_fn if post_fn is not None else create_version
+    attempted: list[str] = [version]
+    current = version
+
+    # Stale-editable path: PATCH-reuse the existing row to rename it to
+    # our target version. Avoids the "cannot create a new version in the
+    # current state" 409 entirely (CI run 24640907316).
+    if stale_editable_id:
+        return reuse_stale_editable(
+            app_id=app_id, existing_id=stale_editable_id,
+            target_version=current, token=token,
+            patch_fn=patch_fn, delete_fn=delete_fn, post_fn=post,
+        )
+
+    for attempt in range(_RETRY_CAP + 1):
+        resp = post(app_id, current, token)
+        if resp.status_code != 409:
+            return resp.json()["data"]["id"]
+
+        errors = _dump_409(resp, current)
+        reusable_id, known_on_secondary = _reconcile_409(
+            app_id, token, current,
+            fetch_versions, fetch_prerelease_versions,
+            fetch_builds_prerelease_versions,
+        )
+        if reusable_id is not None:
+            print(
+                f"[409-reuse] matched existing appStoreVersion for "
+                f"{current} id={reusable_id}",
+                file=sys.stderr,
+            )
+            return reusable_id
+
+        # Classify the 409. Only version-collision errors warrant a bump;
+        # anything else (RELATIONSHIP.INVALID, ATTRIBUTE.INVALID, unknown)
+        # is a payload bug that bumping cannot fix.
+        if errors and not _is_version_collision(errors):
+            _fail_fast_non_version_409(resp, current, errors, attempted)
+
+        _log_retry_reason(current, known_on_secondary)
+        if attempt >= _RETRY_CAP:
+            break
+
+        next_version = bump_fn(current)
+        print(
+            f"[409-retry] attempting {next_version} (retry "
+            f"{attempt + 1}/{_RETRY_CAP})",
+            file=sys.stderr,
+        )
+        current = next_version
+        attempted.append(current)
+
+    print(
+        f"::error::POST /appStoreVersions returned 409 after {_RETRY_CAP} "
+        f"bump-retries. Attempted versions: first={attempted[0]} "
+        f"last={attempted[-1]} total={len(attempted)}. "
+        f"ASC appears to have hidden records colliding with every patch "
+        f"in this range; inspect the app in App Store Connect manually.",
+        file=sys.stderr,
+    )
+    raise SystemExit(3)

package/templates/scripts/ci/ios-native/asc_version_fetch.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+# Note: get_combined_floor (~88 lines) exceeds the 50-line function cap.
+# Pre-existing in the source-of-truth; refactor tracked separately to
+# avoid bundling unrelated edits into round-13 trims.
+"""
+App Store Connect appStoreVersions fetch + ground-truth cross-check.
+
+Split out of manage_marketing_version.py so the main module stays under
+the 400-line cap. This module owns:
+
+  - `fetch_versions`: paginated, platform-filtered list of ALL iOS
+    appStoreVersions for an app. Logs every record to stderr so CI logs
+    reflect ground truth.
+  - `get_ground_truth_floor`: independent narrow query per ship-blocking
+    state. Used as a self-correcting floor: if the main fetch is ever
+    truncated or misfiltered, the cross-check catches it before altool
+    does.
+
+Motivation: CI run 24639633986 failed because the previous single-shot
+fetch missed 1.0.4 READY_FOR_SALE and the script tried to REUSE 1.0.1.
+Pagination + platform filter fix the root cause; the cross-check is
+defense-in-depth so this class of bug cannot recur silently.
+"""
+
+from __future__ import annotations
+
+import sys
+
+import asc_build_history
+from asc_common import request
+
+
+# Ship-blocking states: any train that is closed to new submissions at a
+# lower version. The main fetch classifies them via TERMINAL_STATES +
+# BLOCKING_STATES from asc_common, but the independent cross-check uses
+# this explicit list so the two paths cannot drift.
+SHIP_BLOCKING_STATES = (
+    "READY_FOR_SALE",
+    "PENDING_APPLE_RELEASE",
+    "PROCESSING_FOR_APP_STORE",
+    "PENDING_DEVELOPER_RELEASE",
+    "REPLACED_WITH_NEW_VERSION",
+    "DEVELOPER_REMOVED_FROM_SALE",
+    "REMOVED_FROM_SALE",
+)
+
+
+def _parse_versions(data: dict) -> list[dict]:
+    """Convert an ASC appStoreVersions response payload into our record
+    shape. Pure function -- no I/O, safe to call from both the paginated
+    main fetch and the narrow cross-check."""
+    out: list[dict] = []
+    for item in data.get("data", []):
+        attrs = item.get("attributes") or {}
+        out.append({
+            "versionString": attrs.get("versionString") or "",
+            "state": attrs.get("appStoreState") or "",
+            "id": item.get("id") or "",
+            "createdDate": attrs.get("createdDate") or "",
+            "platform": attrs.get("platform") or "",
+        })
+    return out
+
+
+def _log_record(v: dict) -> None:
+    """One line per record on stderr so CI logs reflect ground truth."""
+    print(
+        f"[asc-fetch] versionString={v['versionString']} "
+        f"appStoreState={v['state']} id={v['id']} "
+        f"platform={v.get('platform', '')} "
+        f"createdDate={v['createdDate']}",
+        file=sys.stderr,
+    )
+
+
+def fetch_versions(app_id: str, token: str) -> list[dict]:
+    """Fetch ALL appStoreVersions for the app on the iOS platform.
+
+    Follows `links.next` until exhausted (do NOT trust a single page),
+    filters to platform=IOS, requests limit=200 (ASC max). Logs every
+    record to stderr for CI diagnostics.
+    """
+    path = f"/apps/{app_id}/appStoreVersions"
+    params: dict = {"filter[platform]": "IOS", "limit": 200}
+    out: list[dict] = []
+    pages = 0
+    resp = request("GET", path, token, params=params)
+
+    while True:
+        pages += 1
+        data = resp.json()
+        batch = _parse_versions(data)
+        for v in batch:
+            _log_record(v)
+        out.extend(batch)
+
+        next_url = ((data.get("links") or {}).get("next")) or ""
+        if not next_url:
+            break
+
+        # Follow the cursor URL verbatim. ASC returns fully-qualified links
+        # with cursor/query already attached; passing params= would override
+        # the cursor and loop forever. Extract the path after /v1 so we stay
+        # on our retrying client.
+        next_path = next_url.split("/v1", 1)[-1] if "/v1" in next_url else next_url
+        resp = request("GET", next_path, token)
+
+    print(
+        f"[asc-fetch] done: pages={pages} records={len(out)} (platform=IOS)",
+        file=sys.stderr,
+    )
+    return out
+
+
+def _fetch_by_state(app_id: str, token: str, state: str) -> list[dict]:
+    """Narrow query for the independent cross-check: one state, iOS only."""
+    path = f"/apps/{app_id}/appStoreVersions"
+    params = {
+        "filter[platform]": "IOS",
+        "filter[appStoreState]": state,
+        "limit": 200,
+    }
+    resp = request("GET", path, token, params=params)
+    return _parse_versions(resp.json())
+
+
+def get_ground_truth_floor(
+    app_id: str, token: str, semver_tuple
+) -> str | None:
+    """Independent cross-check: query each ship-blocking state directly.
+
+    Returns the highest semver across all ship-blocking records, or None
+    if none exist (first release).
+
+    `semver_tuple` is injected to avoid circular imports; the main module
+    owns the semver parser.
+
+    Defense in depth: even if fetch_versions has a latent bug, the narrow
+    cross-check catches a stale/truncated main fetch before altool does.
+    """
+    all_blockers: list[dict] = []
+    for state in SHIP_BLOCKING_STATES:
+        try:
+            recs = _fetch_by_state(app_id, token, state)
+        except SystemExit:
+            # ASC rejects unknown states with 400 -- skip gracefully so
+            # a newly-added state name in our constant does not break the
+            # cross-check as a whole.
+            print(
+                f"[ground-truth] skip state={state} (ASC rejected filter)",
+                file=sys.stderr,
+            )
+            continue
+        for r in recs:
+            print(
+                f"[ground-truth] state={state} "
+                f"versionString={r['versionString']} id={r['id']}",
+                file=sys.stderr,
+            )
+        all_blockers.extend(recs)
+
+    if not all_blockers:
+        print("[ground-truth] no ship-blocking versions found", file=sys.stderr)
+        return None
+
+    floor = max(all_blockers, key=lambda v: semver_tuple(v["versionString"]))
+    print(
+        f"[ground-truth] floor={floor['versionString']} "
+        f"(state={floor['state']}, id={floor['id']})",
+        file=sys.stderr,
+    )
+    return floor["versionString"]
+
+
+def _max_version(
+    versions: list[str], semver_tuple
+) -> str | None:
+    """Return the lexicographic-by-semver max of a list of versionStrings,
+    or None if the list is empty or contains only non-semver values."""
+    parsed = [(semver_tuple(v), v) for v in versions if v]
+    parsed = [p for p in parsed if p[0] != (-1, -1, -1)]
+    if not parsed:
+        return None
+    return max(parsed)[1]
+
+
+def get_combined_floor(
+    app_id: str, token: str, semver_tuple,
+    *,
+    appstore_versions: list[dict] | None = None,
+    ship_blocking_states: tuple[str, ...] = SHIP_BLOCKING_STATES,
+) -> tuple[str | None, dict[str, str | None]]:
+    """Combined ground-truth floor across the two authoritative ASC
+    collections for marketing versions.
+
+    Consults:
+      1. appstore_versions (caller-provided or freshly fetched) filtered
+         to ship-blocking states -- same pool as get_ground_truth_floor's
+         main input, but we pass it in so we don't double-fetch.
+      2. /v1/apps/{id}/preReleaseVersions for every TestFlight train's
+         versionString. Persists across app updates.
+      3. /v1/builds?filter[app]={id}&include=preReleaseVersion for every
+         build's referenced preReleaseVersion.attributes.version. This is
+         a second, independent path to TestFlight-train marketing versions
+         that catches records hidden from the direct /preReleaseVersions
+         query (CI run 24640430898: the direct query missed a record
+         that still produced a 409 collision on POST). The top-level
+         /v1/builds collection is used because the app-scoped
+         /v1/apps/{id}/builds relationship rejects ``include`` with
+         HTTP 400 (CI run 24640675162).
+
+    /builds.attributes.version itself (the integer build number) is NOT
+    consulted -- only the related preReleaseVersion's marketing version.
+
+    Returns (floor_or_None, per_source_maxes) where per_source_maxes is
+    a dict with keys 'appStoreVersions', 'preReleaseVersions',
+    'buildsViaPreRelease'. The floor is the max across all three; None
+    means every source was empty / non-semver (first release).
+
+    Logs each source's max separately so a human debugging CI can see
+    which endpoint contributed the binding floor:
+
+        [combined-floor] appStoreVersions max: 1.0
+        [combined-floor] preReleaseVersions max: 1.0.4
+        [combined-floor] buildsViaPreRelease max: 1.0.6
+        [combined-floor] combined floor: 1.0.6
+    """
+    asv_versions = [
+        v["versionString"] for v in (appstore_versions or [])
+        if v.get("state") in ship_blocking_states and v.get("versionString")
+    ]
+    asv_max = _max_version(asv_versions, semver_tuple)
+
+    prerelease_versions = asc_build_history.fetch_prerelease_versions(
+        app_id, token
+    )
+    prerelease_max = _max_version(prerelease_versions, semver_tuple)
+
+    builds_prerelease_versions = asc_build_history.fetch_builds_prerelease_versions(
+        app_id, token
+    )
+    builds_prerelease_max = _max_version(
+        builds_prerelease_versions, semver_tuple
+    )
+
+    print(
+        f"[combined-floor] appStoreVersions max: {asv_max or '<none>'}",
+        file=sys.stderr,
+    )
+    print(
+        f"[combined-floor] preReleaseVersions max: {prerelease_max or '<none>'}",
+        file=sys.stderr,
+    )
+    print(
+        f"[combined-floor] buildsViaPreRelease max: "
+        f"{builds_prerelease_max or '<none>'}",
+        file=sys.stderr,
+    )
+
+    candidates = [
+        v for v in (asv_max, prerelease_max, builds_prerelease_max) if v
+    ]
+    combined = _max_version(candidates, semver_tuple)
+    print(
+        f"[combined-floor] combined floor: {combined or '<none>'}",
+        file=sys.stderr,
+    )
+
+    per_source = {
+        "appStoreVersions": asv_max,
+        "preReleaseVersions": prerelease_max,
+        "buildsViaPreRelease": builds_prerelease_max,
+    }
+    return combined, per_source