@1aboveio/skills 1.0.16 → 1.0.18

package/skills/app-dev/maestro/scripts/failure_record.py

@@ -0,0 +1,110 @@
+"""The device failure record: shared shape, read and write, no device needed.
+
+Every layer that records a failed device case writes the same ``failure.json`` and
+the same one-shot ``NATIVE_FAILURE {...}`` line:
+
+    case, step, error, screenshot, uiDump, logcatTail, evidenceDir, cleanupErrors
+
+This module owns that shape, its constants and the file access. The Android capture
+helper (``android-development/scripts/failure_capture.py``), the ``device-run.sh``
+fallback and the framework adapters described in that skill's ``references/
+diagnosis.md`` share it, and the Maestro converter in this folder writes the record
+through it, so the shape cannot drift between the layers. Nothing here needs a
+device, adb or an SDK.
+
+``screenshot``, ``uiDump`` and ``logcatTail`` are file names relative to
+``evidenceDir`` and are ``null`` when that capture failed; the reason goes to
+``cleanupErrors`` and never replaces the original ``error``. A record is written
+once: the first failure is kept, and a later failure or a cleanup problem never
+overwrites it.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from dataclasses import dataclass, field
+
+RECORD_NAME = "failure.json"
+NATIVE_FAILURE_PREFIX = "NATIVE_FAILURE "
+
+RECORD_KEYS = (
+    "case",
+    "step",
+    "error",
+    "screenshot",
+    "uiDump",
+    "logcatTail",
+    "evidenceDir",
+    "cleanupErrors",
+)
+NATIVE_FAILURE_KEYS = ("case", "step", "error", "evidenceDir")
+
+
+@dataclass
+class Failure:
+    """One failed device case; also the payload of `failure.json`."""
+
+    case: str = None
+    step: str = None
+    error: str = ""
+    screenshot: str = None
+    uiDump: str = None
+    logcatTail: str = None
+    evidenceDir: str = None
+    cleanupErrors: list = field(default_factory=list)
+
+    def as_dict(self):
+        return {key: getattr(self, key) for key in RECORD_KEYS}
+
+    def native_failure_line(self):
+        compact = {key: getattr(self, key) for key in NATIVE_FAILURE_KEYS}
+        return NATIVE_FAILURE_PREFIX + json.dumps(compact, separators=(",", ":"))
+
+    @classmethod
+    def from_dict(cls, payload):
+        values = {key: payload[key] for key in RECORD_KEYS if key in payload}
+        errors = values.get("cleanupErrors")
+        values["cleanupErrors"] = list(errors) if isinstance(errors, list) else []
+        return cls(**values)
+
+
+def read_record(evidence_dir):
+    """The record already in `evidence_dir`, or None when there is no usable one."""
+    path = os.path.join(evidence_dir, RECORD_NAME)
+    try:
+        with open(path, "r", encoding="utf-8") as stream:
+            payload = json.load(stream)
+    except (OSError, ValueError):
+        return None
+    if not isinstance(payload, dict) or "error" not in payload:
+        return None
+    return Failure.from_dict(payload)
+
+
+def write_record(failure, evidence_dir):
+    """Store `failure` unless a record already exists; return the record on disk.
+
+    A valid earlier record wins, so a cleanup error or a second failing step cannot
+    replace the first failure. An unreadable file is not a record and is replaced,
+    with the replacement noted on the returned record's `cleanupErrors`.
+    """
+    os.makedirs(evidence_dir, exist_ok=True)
+    existing = read_record(evidence_dir)
+    if existing is not None:
+        return existing
+    path = os.path.join(evidence_dir, RECORD_NAME)
+    if os.path.exists(path):
+        failure.cleanupErrors.append("%s: replaced an unreadable record" % RECORD_NAME)
+    payload = json.dumps(failure.as_dict(), indent=2) + "\n"
+    handle, temporary = tempfile.mkstemp(dir=evidence_dir, prefix=".failure-", suffix=".json")
+    try:
+        with os.fdopen(handle, "w", encoding="utf-8") as stream:
+            stream.write(payload)
+        os.replace(temporary, path)
+    except BaseException:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+        raise
+    return failure

package/skills/app-dev/maestro/scripts/maestro_failure.py

@@ -0,0 +1,481 @@
+"""Convert a Maestro test-output session or Flow folder into the device failure record.
+
+Maestro's ``--test-output-dir`` session holds one self-contained folder per Flow
+(`test reports and artifacts <https://docs.maestro.dev/maestro-flows/workspace-management/test-reports-and-artifacts>`_):
+
+    <output>/2026-07-31_173617/            # session folder
+    ├── maestro.log
+    └── checkout_flow/                     # one folder per Flow
+        ├── manifest.json
+        ├── commands.json                  # one entry per executed step
+        ├── screenshots/                   # step-<NNN>-<type>.png, failing step only
+        ├── screen-hierarchy/              # step-<NNN>-<type>.json
+        └── logs/device-logcat.txt
+
+``--maestro-dir`` takes the session folder or one Flow folder. The converter picks the
+first failed step, writes ``failure.json`` through the shared record model in
+``failure_record.py`` (beside this file) and prints the one ``NATIVE_FAILURE {...}`` line,
+so the record shape cannot drift from the other layers. Nothing here needs a device, adb,
+an SDK or the Android skill:
+
+    python3 maestro_failure.py --maestro-dir "$LOG_DIR/2026-07-31_173617" --case checkout
+
+A passing Flow writes nothing. An existing ``failure.json`` is never replaced, so the first
+failure keeps its evidence. ``screenshot``, ``uiDump`` and ``logcatTail`` are paths relative
+to ``evidenceDir``: the Flow folder itself, unless ``--evidence-dir`` names another run
+folder, in which case the artifacts are copied there. A missing artifact is ``null`` and the
+reason goes to ``cleanupErrors``; it never replaces ``error``. An artifact path that leaves
+the Flow folder or the evidence folder is refused the same way, so a recorded path cannot
+read or overwrite a file outside them.
+
+A readable ``commands.json`` has to be a list of step objects whose ``status`` is one of the
+six Maestro writes; anything else is a conversion error and exits 2 rather than being
+dropped, because a dropped entry turns a failed run into an apparent pass. A Flow whose
+``commands.json`` cannot be read at all is reported as an incomplete conversion, while a
+readable failing Flow beside it still writes its record.
+
+Field mapping, with what could not be confirmed from the docs
+-------------------------------------------------------------
+
+Confirmed from Maestro's source (``TestOutputWriter.saveCommands`` and
+``CommandDebugMetadata``), because the docs name the fields but show no example:
+
+- ``commands.json`` is an array of ``{"command": <command object>, "metadata": {...}}``.
+- ``metadata.status`` is one of PENDING, RUNNING, COMPLETED, FAILED, WARNED, SKIPPED;
+  ``metadata.timestamp`` is epoch milliseconds; ``sequenceNumber`` starts at 0.
+- ``metadata.error`` is the exception serialized as ``{"message", "debugMessage"}``; keys
+  are omitted when empty.
+- ``metadata.artifacts`` is a list of ``{"type": "SCREENSHOT"|"SCREEN_HIERARCHY"|...,
+  "path": "<flow-folder-relative>"}``.
+
+Confirmed from the docs: ``screenshots/``, ``screen-hierarchy/``, ``logs/maestro.log``,
+``logs/device-logcat.txt`` (Android), ``manifest.json``, and that only the failing step is
+captured outside ``--analyze``.
+
+Assumed, and not confirmed from the docs:
+
+- The *step description*: ``commands.json`` stores the command object, not a rendered
+  description (Maestro's ``originalDescription`` is not serialized). ``step`` is the
+  command's own ``label`` when it has one, otherwise a mechanical reading of the command
+  type plus its first target (``text``/``textRegex``/``id``/``point``/``appId``/``link``/
+  ``path``). It is deliberately not a re-implementation of Maestro's phrasing.
+- Artifact file names ``step-<NNN>-<type>`` with ``NNN`` = ``sequenceNumber`` + 1. This was
+  observed in Maestro's own tests, not in the docs, so it is only the fallback when
+  ``metadata.artifacts`` does not name the file.
+- ``logs/device-simulator.log`` as the iOS fallback for the device log; the Android target
+  is ``logs/device-logcat.txt``.
+- Which Flow failed first in a multi-Flow session: the earliest failing step's
+  ``timestamp`` decides, with the folder name as the tie-break.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import shutil
+import sys
+
+from failure_record import Failure, read_record, write_record
+
+COMMANDS_JSON = "commands.json"
+SCREENSHOT_DIR = "screenshots"
+HIERARCHY_DIR = "screen-hierarchy"
+DEVICE_LOGS = ("logs/device-logcat.txt", "logs/device-simulator.log")
+FAILED_STATUS = "FAILED"
+# The statuses CommandDebugMetadata writes; anything else is not a step this can read.
+STATUSES = ("PENDING", "RUNNING", "COMPLETED", "FAILED", "WARNED", "SKIPPED")
+NO_ERROR_MESSAGE = "Maestro marked the step FAILED and gave no error message"
+TARGET_KEYS = ("text", "textRegex", "id", "point", "appId", "link", "path")
+# The record attribute, the Maestro artifact kind, and the reason when it is absent.
+RECORD_ARTIFACTS = (
+    ("screenshot", "SCREENSHOT", "no failing-step screenshot in the Maestro output"),
+    ("uiDump", "SCREEN_HIERARCHY", "no screen hierarchy in the Maestro output"),
+    ("logcatTail", "LOGCAT", "no device log in the Maestro output"),
+)
+
+_CAMEL_WORD = re.compile(r"(?<=[a-z0-9])([A-Z])")
+
+
+class MaestroOutputError(RuntimeError):
+    """The given folder holds no usable Maestro commands.json."""
+
+
+def _text(value):
+    if value is None:
+        return ""
+    return str(value).strip()
+
+
+def load_commands(flow_dir):
+    """The executed steps of `flow_dir`, or None when its commands.json cannot be read.
+
+    A readable file is a list of step objects in the nested or the documented flat shape.
+    An entry that is not an object, or whose status is not one of `STATUSES`, raises
+    `MaestroOutputError`: silently dropping it would report a failed run as a pass.
+    """
+    path = os.path.join(flow_dir, COMMANDS_JSON)
+    try:
+        with open(path, "r", encoding="utf-8") as stream:
+            payload = json.load(stream)
+    except (OSError, ValueError):
+        return None
+    if not isinstance(payload, list):
+        return None
+    for index, entry in enumerate(payload):
+        if not isinstance(entry, dict):
+            raise MaestroOutputError("%s entry %d is not an object" % (path, index))
+        _, metadata = _pieces(entry)
+        if _status(metadata) not in STATUSES:
+            raise MaestroOutputError(
+                "%s entry %d has unsupported status %r" % (path, index, metadata.get("status"))
+            )
+    return payload
+
+
+def _pieces(entry):
+    """(command, metadata) for one entry, in the current or the documented flat shape."""
+    metadata = entry.get("metadata")
+    if isinstance(metadata, dict):
+        return entry.get("command"), metadata
+    return entry.get("command"), entry
+
+
+def _status(metadata):
+    return _text(metadata.get("status")).upper()
+
+
+def failed_steps(entries):
+    """(command, metadata) for each step Maestro marked FAILED, in execution order."""
+    steps = []
+    for entry in entries:
+        command, metadata = _pieces(entry)
+        if _status(metadata) == FAILED_STATUS:
+            steps.append((command, metadata))
+    return steps
+
+
+def _timestamp(metadata):
+    """The step's epoch-millisecond timestamp, or infinity when Maestro did not record one."""
+    value = metadata.get("timestamp")
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return float("inf")
+    return float(value)
+
+
+def flow_dirs(root, max_depth=3):
+    """Folders holding a commands.json, `root` included, at most `max_depth` levels down."""
+    if os.path.isfile(os.path.join(root, COMMANDS_JSON)):
+        return [root]
+    found = []
+    base_depth = os.path.abspath(root).rstrip(os.sep).count(os.sep)
+    for current, directories, files in os.walk(root):
+        depth = os.path.abspath(current).rstrip(os.sep).count(os.sep) - base_depth
+        directories.sort()
+        if COMMANDS_JSON in files:
+            found.append(current)
+            # A Flow folder's own subfolders hold artifacts, never another Flow.
+            directories[:] = []
+        elif depth >= max_depth:
+            directories[:] = []
+    return found
+
+
+def first_failure(root, incomplete=None):
+    """(flow dir, command, metadata) for the earliest failing step under `root`, or None.
+
+    `incomplete` is an optional list collecting one note per Flow whose commands.json
+    cannot be read, so a caller can report the conversion as partial. Every unreadable
+    Flow is noted, even when another Flow supplies the failure.
+    """
+    directories = flow_dirs(root)
+    if not directories:
+        raise MaestroOutputError("no %s under %s" % (COMMANDS_JSON, root))
+    unreadable = []
+    unreadable_notes = []
+    failures = []
+    for directory in directories:
+        detail = None
+        try:
+            entries = load_commands(directory)
+        except MaestroOutputError as problem:
+            entries = None
+            detail = str(problem)
+        if entries is None:
+            unreadable.append(directory)
+            note = "unreadable %s in %s" % (COMMANDS_JSON, directory)
+            unreadable_notes.append(note if detail is None else "%s: %s" % (note, detail))
+            continue
+        steps = failed_steps(entries)
+        if steps:
+            command, metadata = steps[0]
+            failures.append((_timestamp(metadata), directory, command, metadata))
+    if unreadable and incomplete is not None:
+        incomplete.extend(unreadable_notes)
+    if not failures:
+        if unreadable:
+            raise MaestroOutputError(
+                "no readable %s: %s" % (COMMANDS_JSON, "; ".join(unreadable_notes))
+            )
+        return None
+    failures.sort(key=lambda failure: (failure[0], failure[1]))
+    _, directory, command, metadata = failures[0]
+    return directory, command, metadata
+
+
+def humanize(command_type):
+    """`tapOnElement` -> `Tap on element`, `assertScreenshotCommand` -> `Assert screenshot`."""
+    name = command_type[: -len("Command")] if command_type.endswith("Command") else command_type
+    spaced = _CAMEL_WORD.sub(r" \1", name).lower()
+    return spaced[:1].upper() + spaced[1:]
+
+
+def _target(value, depth=5):
+    """The first selector or target text inside a command, for a readable step."""
+    if depth <= 0:
+        return None
+    if isinstance(value, dict):
+        for key in TARGET_KEYS:
+            candidate = value.get(key)
+            if isinstance(candidate, bool):
+                continue
+            if isinstance(candidate, (str, int, float)) and _text(candidate):
+                return candidate
+        children = list(value.values())
+    elif isinstance(value, list):
+        children = value
+    else:
+        return None
+    for child in children:
+        found = _target(child, depth - 1)
+        if found is not None:
+            return found
+    return None
+
+
+def describe_command(command):
+    """The step recorded for the failing command, or None when Maestro stored none."""
+    if isinstance(command, str):
+        return _text(command) or None
+    if not isinstance(command, dict):
+        return None
+    for command_type, value in command.items():
+        if value is None:
+            continue
+        label = value.get("label") if isinstance(value, dict) else None
+        if _text(label):
+            return _text(label)
+        reading = humanize(command_type)
+        target = _target(value)
+        return reading if target is None else "%s %s" % (reading, json.dumps(target, ensure_ascii=False))
+    return None
+
+
+def error_text(metadata):
+    """The step's error message, keeping Maestro's debug detail when it adds anything."""
+    error = metadata.get("error")
+    if isinstance(error, dict):
+        message = _text(error.get("message"))
+        detail = _text(error.get("debugMessage"))
+        if message and detail and detail not in message:
+            return "%s: %s" % (message, detail)
+        return message or detail or NO_ERROR_MESSAGE
+    text = _text(error)
+    return text or NO_ERROR_MESSAGE
+
+
+def _listed_artifacts(metadata):
+    """{kind: relative path} from the step's artifacts list, in either shape."""
+    artifacts = metadata.get("artifacts")
+    if not isinstance(artifacts, list):
+        return {}
+    found = {}
+    for artifact in artifacts:
+        kind = ""
+        if isinstance(artifact, dict):
+            kind = _text(artifact.get("type")).upper()
+            path = artifact.get("path")
+        else:
+            path = artifact
+        if not isinstance(path, str) or not path:
+            continue
+        if not kind:
+            # The documented flat shape names the file without a kind.
+            lowered = path.lower()
+            if lowered.endswith(".png"):
+                kind = "SCREENSHOT"
+            elif lowered.endswith(".json") and "hierarch" in lowered:
+                kind = "SCREEN_HIERARCHY"
+        if kind:
+            found.setdefault(kind, path)
+    return found
+
+
+def _step_named_artifacts(flow_dir, metadata):
+    """Fallback: the files Maestro names after the step, step-<NNN>-<type>.(png|json)."""
+    number = metadata.get("sequenceNumber")
+    prefix = "step-%03d-" % (number + 1) if isinstance(number, int) and not isinstance(number, bool) else "step-"
+    found = {}
+    for kind, directory, suffix in (
+        ("SCREENSHOT", SCREENSHOT_DIR, ".png"),
+        ("SCREEN_HIERARCHY", HIERARCHY_DIR, ".json"),
+    ):
+        try:
+            names = sorted(os.listdir(os.path.join(flow_dir, directory)))
+        except OSError:
+            continue
+        for name in names:
+            if name.startswith(prefix) and name.endswith(suffix):
+                found[kind] = "%s/%s" % (directory, name)
+                break
+    return found
+
+
+def step_artifacts(flow_dir, metadata):
+    """{kind: relative path} for the failing step, as `commands.json` names it.
+
+    A path Maestro listed is kept even when the file is absent, so the record names the
+    artifact it expected instead of a generic gap. The step-named fallback only fills a
+    kind Maestro did not list, and only with a file that is actually there.
+    """
+    found = {}
+    listed = _listed_artifacts(metadata)
+    named = _step_named_artifacts(flow_dir, metadata)
+    for kind in ("SCREENSHOT", "SCREEN_HIERARCHY"):
+        relative = listed.get(kind) or named.get(kind)
+        if relative:
+            found[kind] = relative
+    for relative in DEVICE_LOGS:
+        if os.path.isfile(os.path.join(flow_dir, relative)):
+            found["LOGCAT"] = relative
+            break
+    return found
+
+
+def _contained(base_dir, relative):
+    """`relative` resolved under `base_dir`, or None when it can leave that folder.
+
+    An absolute path, an escaping `..` path, or an existing symlink that resolves outside the
+    folder is refused, so the resolved path is what the caller then reads or writes.
+    """
+    if not relative or "\x00" in relative:
+        return None
+    if os.path.isabs(relative) or os.path.splitdrive(relative)[0]:
+        return None
+    base = os.path.realpath(base_dir)
+    resolved = os.path.realpath(os.path.join(base, relative))
+    if resolved != base and not resolved.startswith(base + os.sep):
+        return None
+    return resolved
+
+
+def _store(flow_dir, evidence_dir, relative, failure, attribute):
+    """Copy `relative` into `evidence_dir` when they differ; record the failure otherwise."""
+    source = _contained(flow_dir, relative)
+    if source is None:
+        failure.cleanupErrors.append("%s: %s is outside the Maestro output" % (attribute, relative))
+        return None
+    if not os.path.isfile(source):
+        failure.cleanupErrors.append("%s: %s is not in the Maestro output" % (attribute, relative))
+        return None
+    if os.path.abspath(evidence_dir) == os.path.abspath(flow_dir):
+        return relative
+    destination = _contained(evidence_dir, relative)
+    if destination is None:
+        failure.cleanupErrors.append(
+            "%s: copy of %s would leave the evidence folder" % (attribute, relative)
+        )
+        return None
+    try:
+        os.makedirs(os.path.dirname(destination) or evidence_dir, exist_ok=True)
+        shutil.copyfile(source, destination)
+    except OSError as problem:
+        failure.cleanupErrors.append("%s: copy of %s failed: %s" % (attribute, relative, problem))
+        return None
+    return relative
+
+
+def build_failure(flow_dir, command, metadata, case, evidence_dir):
+    """The `Failure` for one failed Maestro step, with the evidence found in `flow_dir`."""
+    failure = Failure(
+        case=case or os.path.basename(os.path.abspath(flow_dir)),
+        step=describe_command(command),
+        error=error_text(metadata),
+        evidenceDir=evidence_dir,
+    )
+    found = step_artifacts(flow_dir, metadata)
+    for attribute, kind, reason in RECORD_ARTIFACTS:
+        relative = found.get(kind)
+        if relative is None:
+            failure.cleanupErrors.append("%s: %s" % (attribute, reason))
+            continue
+        setattr(failure, attribute, _store(flow_dir, evidence_dir, relative, failure, attribute))
+    return failure
+
+
+def convert(maestro_dir, case=None, evidence_dir=None, stream=None, incomplete=None):
+    """Write the failure record for the first failed Flow under `maestro_dir`.
+
+    Returns the record written, or the earlier record kept in `evidence_dir`, or None when
+    no Flow failed. Raises `MaestroOutputError` when no readable `commands.json` is there.
+    Flows whose commands.json cannot be read are noted on `incomplete` and on the written
+    record's `cleanupErrors`; the failure itself is still recorded unchanged.
+    """
+    stream = stream if stream is not None else sys.stdout
+    notes = incomplete if incomplete is not None else []
+    selected = first_failure(maestro_dir, notes)
+    if selected is None:
+        return None
+    flow_dir, command, metadata = selected
+    evidence = os.path.abspath(evidence_dir or flow_dir)
+    kept = read_record(evidence)
+    if kept is not None:
+        # The first failure owns this evidence folder; a later conversion only repeats it.
+        print(kept.native_failure_line(), file=stream, flush=True)
+        return kept
+    failure = build_failure(flow_dir, command, metadata, case, evidence)
+    failure.cleanupErrors.extend(notes)
+    failure = write_record(failure, evidence)
+    print(failure.native_failure_line(), file=stream, flush=True)
+    return failure
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(
+        description="Convert a Maestro test-output session or Flow folder into the device failure record."
+    )
+    parser.add_argument(
+        "--maestro-dir",
+        required=True,
+        help="Maestro --test-output-dir session folder, or one Flow folder holding commands.json",
+    )
+    parser.add_argument("--case", help="case name for the record; defaults to the Flow folder name")
+    parser.add_argument(
+        "--evidence-dir",
+        help="run evidence folder that holds failure.json; defaults to the Flow folder",
+    )
+    options = parser.parse_args(argv)
+    if not os.path.isdir(options.maestro_dir):
+        print("maestro folder not found: %s" % options.maestro_dir, file=sys.stderr)
+        return 2
+    incomplete = []
+    try:
+        convert(
+            options.maestro_dir,
+            case=options.case,
+            evidence_dir=options.evidence_dir,
+            incomplete=incomplete,
+        )
+    except MaestroOutputError as problem:
+        print("not converted: %s" % problem, file=sys.stderr)
+        return 2
+    if incomplete:
+        for note in incomplete:
+            print("incomplete conversion: %s" % note, file=sys.stderr)
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())