@1aboveio/skills 1.0.15 → 1.0.17

package/skills/app-dev/android-development/scripts/maestro_failure.py

@@ -0,0 +1,401 @@
+"""Convert a Maestro test-output session or Flow folder into the device failure record.
+
+Maestro's ``--test-output-dir`` session holds one self-contained folder per Flow
+(`test reports and artifacts <https://docs.maestro.dev/maestro-flows/workspace-management/test-reports-and-artifacts>`_):
+
+    <output>/2026-07-31_173617/            # session folder
+    ├── maestro.log
+    └── checkout_flow/                     # one folder per Flow
+        ├── manifest.json
+        ├── commands.json                  # one entry per executed step
+        ├── screenshots/                   # step-<NNN>-<type>.png, failing step only
+        ├── screen-hierarchy/              # step-<NNN>-<type>.json
+        └── logs/device-logcat.txt
+
+``--maestro-dir`` takes the session folder or one Flow folder. The converter picks the
+first failed step, reuses ``failure_capture`` to write ``failure.json`` and prints the one
+``NATIVE_FAILURE {...}`` line, so the record shape cannot drift from the other layers:
+
+    python3 maestro_failure.py --maestro-dir "$LOG_DIR/2026-07-31_173617" --case checkout
+
+A passing Flow writes nothing. An existing ``failure.json`` is never replaced, so the first
+failure keeps its evidence. ``screenshot``, ``uiDump`` and ``logcatTail`` are paths relative
+to ``evidenceDir``: the Flow folder itself, unless ``--evidence-dir`` names another run
+folder, in which case the artifacts are copied there. A missing artifact is ``null`` and the
+reason goes to ``cleanupErrors``; it never replaces ``error``.
+
+Field mapping, with what could not be confirmed from the docs
+-------------------------------------------------------------
+
+Confirmed from Maestro's source (``TestOutputWriter.saveCommands`` and
+``CommandDebugMetadata``), because the docs name the fields but show no example:
+
+- ``commands.json`` is an array of ``{"command": <command object>, "metadata": {...}}``.
+- ``metadata.status`` is one of PENDING, RUNNING, COMPLETED, FAILED, WARNED, SKIPPED;
+  ``metadata.timestamp`` is epoch milliseconds; ``sequenceNumber`` starts at 0.
+- ``metadata.error`` is the exception serialized as ``{"message", "debugMessage"}``; keys
+  are omitted when empty.
+- ``metadata.artifacts`` is a list of ``{"type": "SCREENSHOT"|"SCREEN_HIERARCHY"|...,
+  "path": "<flow-folder-relative>"}``.
+
+Confirmed from the docs: ``screenshots/``, ``screen-hierarchy/``, ``logs/maestro.log``,
+``logs/device-logcat.txt`` (Android), ``manifest.json``, and that only the failing step is
+captured outside ``--analyze``.
+
+Assumed, and not confirmed from the docs:
+
+- The *step description*: ``commands.json`` stores the command object, not a rendered
+  description (Maestro's ``originalDescription`` is not serialized). ``step`` is the
+  command's own ``label`` when it has one, otherwise a mechanical reading of the command
+  type plus its first target (``text``/``textRegex``/``id``/``point``/``appId``/``link``/
+  ``path``). It is deliberately not a re-implementation of Maestro's phrasing.
+- Artifact file names ``step-<NNN>-<type>`` with ``NNN`` = ``sequenceNumber`` + 1. This was
+  observed in Maestro's own tests, not in the docs, so it is only the fallback when
+  ``metadata.artifacts`` does not name the file.
+- ``logs/device-simulator.log`` as the iOS fallback for the device log; the Android target
+  is ``logs/device-logcat.txt``.
+- Which Flow failed first in a multi-Flow session: the earliest failing step's
+  ``timestamp`` decides, with the folder name as the tie-break.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import shutil
+import sys
+
+from failure_capture import Failure, read_record, write_record
+
+COMMANDS_JSON = "commands.json"
+SCREENSHOT_DIR = "screenshots"
+HIERARCHY_DIR = "screen-hierarchy"
+DEVICE_LOGS = ("logs/device-logcat.txt", "logs/device-simulator.log")
+FAILED_STATUS = "FAILED"
+NO_ERROR_MESSAGE = "Maestro marked the step FAILED and gave no error message"
+TARGET_KEYS = ("text", "textRegex", "id", "point", "appId", "link", "path")
+# The record attribute, the Maestro artifact kind, and the reason when it is absent.
+RECORD_ARTIFACTS = (
+    ("screenshot", "SCREENSHOT", "no failing-step screenshot in the Maestro output"),
+    ("uiDump", "SCREEN_HIERARCHY", "no screen hierarchy in the Maestro output"),
+    ("logcatTail", "LOGCAT", "no device log in the Maestro output"),
+)
+
+_CAMEL_WORD = re.compile(r"(?<=[a-z0-9])([A-Z])")
+
+
+class MaestroOutputError(RuntimeError):
+    """The given folder holds no usable Maestro commands.json."""
+
+
+def _text(value):
+    if value is None:
+        return ""
+    return str(value).strip()
+
+
+def load_commands(flow_dir):
+    """The executed steps of `flow_dir`, or None when its commands.json is unusable."""
+    try:
+        with open(os.path.join(flow_dir, COMMANDS_JSON), "r", encoding="utf-8") as stream:
+            payload = json.load(stream)
+    except (OSError, ValueError):
+        return None
+    if not isinstance(payload, list):
+        return None
+    return [entry for entry in payload if isinstance(entry, dict)]
+
+
+def _pieces(entry):
+    """(command, metadata) for one entry, in the current or the documented flat shape."""
+    metadata = entry.get("metadata")
+    if isinstance(metadata, dict):
+        return entry.get("command"), metadata
+    return entry.get("command"), entry
+
+
+def _status(metadata):
+    return _text(metadata.get("status")).upper()
+
+
+def failed_steps(entries):
+    """(command, metadata) for each step Maestro marked FAILED, in execution order."""
+    steps = []
+    for entry in entries:
+        command, metadata = _pieces(entry)
+        if _status(metadata) == FAILED_STATUS:
+            steps.append((command, metadata))
+    return steps
+
+
+def _timestamp(metadata):
+    """The step's epoch-millisecond timestamp, or infinity when Maestro did not record one."""
+    value = metadata.get("timestamp")
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return float("inf")
+    return float(value)
+
+
+def flow_dirs(root, max_depth=3):
+    """Folders holding a commands.json, `root` included, at most `max_depth` levels down."""
+    if os.path.isfile(os.path.join(root, COMMANDS_JSON)):
+        return [root]
+    found = []
+    base_depth = os.path.abspath(root).rstrip(os.sep).count(os.sep)
+    for current, directories, files in os.walk(root):
+        depth = os.path.abspath(current).rstrip(os.sep).count(os.sep) - base_depth
+        directories.sort()
+        if COMMANDS_JSON in files:
+            found.append(current)
+            # A Flow folder's own subfolders hold artifacts, never another Flow.
+            directories[:] = []
+        elif depth >= max_depth:
+            directories[:] = []
+    return found
+
+
+def first_failure(root):
+    """(flow dir, command, metadata) for the earliest failing step under `root`, or None."""
+    directories = flow_dirs(root)
+    if not directories:
+        raise MaestroOutputError("no %s under %s" % (COMMANDS_JSON, root))
+    unreadable = []
+    failures = []
+    for directory in directories:
+        entries = load_commands(directory)
+        if entries is None:
+            unreadable.append(directory)
+            continue
+        steps = failed_steps(entries)
+        if steps:
+            command, metadata = steps[0]
+            failures.append((_timestamp(metadata), directory, command, metadata))
+    if not failures:
+        if unreadable:
+            raise MaestroOutputError(
+                "no readable %s in %s" % (COMMANDS_JSON, ", ".join(sorted(unreadable)))
+            )
+        return None
+    failures.sort(key=lambda failure: (failure[0], failure[1]))
+    _, directory, command, metadata = failures[0]
+    return directory, command, metadata
+
+
+def humanize(command_type):
+    """`tapOnElement` -> `Tap on element`, `assertScreenshotCommand` -> `Assert screenshot`."""
+    name = command_type[: -len("Command")] if command_type.endswith("Command") else command_type
+    spaced = _CAMEL_WORD.sub(r" \1", name).lower()
+    return spaced[:1].upper() + spaced[1:]
+
+
+def _target(value, depth=5):
+    """The first selector or target text inside a command, for a readable step."""
+    if depth <= 0:
+        return None
+    if isinstance(value, dict):
+        for key in TARGET_KEYS:
+            candidate = value.get(key)
+            if isinstance(candidate, bool):
+                continue
+            if isinstance(candidate, (str, int, float)) and _text(candidate):
+                return candidate
+        children = list(value.values())
+    elif isinstance(value, list):
+        children = value
+    else:
+        return None
+    for child in children:
+        found = _target(child, depth - 1)
+        if found is not None:
+            return found
+    return None
+
+
+def describe_command(command):
+    """The step recorded for the failing command, or None when Maestro stored none."""
+    if isinstance(command, str):
+        return _text(command) or None
+    if not isinstance(command, dict):
+        return None
+    for command_type, value in command.items():
+        if value is None:
+            continue
+        label = value.get("label") if isinstance(value, dict) else None
+        if _text(label):
+            return _text(label)
+        reading = humanize(command_type)
+        target = _target(value)
+        return reading if target is None else "%s %s" % (reading, json.dumps(target, ensure_ascii=False))
+    return None
+
+
+def error_text(metadata):
+    """The step's error message, keeping Maestro's debug detail when it adds anything."""
+    error = metadata.get("error")
+    if isinstance(error, dict):
+        message = _text(error.get("message"))
+        detail = _text(error.get("debugMessage"))
+        if message and detail and detail not in message:
+            return "%s: %s" % (message, detail)
+        return message or detail or NO_ERROR_MESSAGE
+    text = _text(error)
+    return text or NO_ERROR_MESSAGE
+
+
+def _listed_artifacts(metadata):
+    """{kind: relative path} from the step's artifacts list, in either shape."""
+    artifacts = metadata.get("artifacts")
+    if not isinstance(artifacts, list):
+        return {}
+    found = {}
+    for artifact in artifacts:
+        kind = ""
+        if isinstance(artifact, dict):
+            kind = _text(artifact.get("type")).upper()
+            path = artifact.get("path")
+        else:
+            path = artifact
+        if not isinstance(path, str) or not path:
+            continue
+        if not kind:
+            # The documented flat shape names the file without a kind.
+            lowered = path.lower()
+            if lowered.endswith(".png"):
+                kind = "SCREENSHOT"
+            elif lowered.endswith(".json") and "hierarch" in lowered:
+                kind = "SCREEN_HIERARCHY"
+        if kind:
+            found.setdefault(kind, path)
+    return found
+
+
+def _step_named_artifacts(flow_dir, metadata):
+    """Fallback: the files Maestro names after the step, step-<NNN>-<type>.(png|json)."""
+    number = metadata.get("sequenceNumber")
+    prefix = "step-%03d-" % (number + 1) if isinstance(number, int) and not isinstance(number, bool) else "step-"
+    found = {}
+    for kind, directory, suffix in (
+        ("SCREENSHOT", SCREENSHOT_DIR, ".png"),
+        ("SCREEN_HIERARCHY", HIERARCHY_DIR, ".json"),
+    ):
+        try:
+            names = sorted(os.listdir(os.path.join(flow_dir, directory)))
+        except OSError:
+            continue
+        for name in names:
+            if name.startswith(prefix) and name.endswith(suffix):
+                found[kind] = "%s/%s" % (directory, name)
+                break
+    return found
+
+
+def step_artifacts(flow_dir, metadata):
+    """{kind: relative path} for the failing step, as `commands.json` names it.
+
+    A path Maestro listed is kept even when the file is absent, so the record names the
+    artifact it expected instead of a generic gap. The step-named fallback only fills a
+    kind Maestro did not list, and only with a file that is actually there.
+    """
+    found = {}
+    listed = _listed_artifacts(metadata)
+    named = _step_named_artifacts(flow_dir, metadata)
+    for kind in ("SCREENSHOT", "SCREEN_HIERARCHY"):
+        relative = listed.get(kind) or named.get(kind)
+        if relative:
+            found[kind] = relative
+    for relative in DEVICE_LOGS:
+        if os.path.isfile(os.path.join(flow_dir, relative)):
+            found["LOGCAT"] = relative
+            break
+    return found
+
+
+def _store(flow_dir, evidence_dir, relative, failure, attribute):
+    """Copy `relative` into `evidence_dir` when they differ; record the failure otherwise."""
+    source = os.path.join(flow_dir, relative)
+    if not os.path.isfile(source):
+        failure.cleanupErrors.append("%s: %s is not in the Maestro output" % (attribute, relative))
+        return None
+    if os.path.abspath(evidence_dir) == os.path.abspath(flow_dir):
+        return relative
+    destination = os.path.join(evidence_dir, relative)
+    try:
+        os.makedirs(os.path.dirname(destination) or evidence_dir, exist_ok=True)
+        shutil.copyfile(source, destination)
+    except OSError as problem:
+        failure.cleanupErrors.append("%s: copy of %s failed: %s" % (attribute, relative, problem))
+        return None
+    return relative
+
+
+def build_failure(flow_dir, command, metadata, case, evidence_dir):
+    """The `Failure` for one failed Maestro step, with the evidence found in `flow_dir`."""
+    failure = Failure(
+        case=case or os.path.basename(os.path.abspath(flow_dir)),
+        step=describe_command(command),
+        error=error_text(metadata),
+        evidenceDir=evidence_dir,
+    )
+    found = step_artifacts(flow_dir, metadata)
+    for attribute, kind, reason in RECORD_ARTIFACTS:
+        relative = found.get(kind)
+        if relative is None:
+            failure.cleanupErrors.append("%s: %s" % (attribute, reason))
+            continue
+        setattr(failure, attribute, _store(flow_dir, evidence_dir, relative, failure, attribute))
+    return failure
+
+
+def convert(maestro_dir, case=None, evidence_dir=None, stream=None):
+    """Write the failure record for the first failed Flow under `maestro_dir`.
+
+    Returns the record written, or the earlier record kept in `evidence_dir`, or None when
+    no Flow failed. Raises `MaestroOutputError` when no readable `commands.json` is there.
+    """
+    stream = stream if stream is not None else sys.stdout
+    selected = first_failure(maestro_dir)
+    if selected is None:
+        return None
+    flow_dir, command, metadata = selected
+    evidence = os.path.abspath(evidence_dir or flow_dir)
+    kept = read_record(evidence)
+    if kept is not None:
+        # The first failure owns this evidence folder; a later conversion only repeats it.
+        print(kept.native_failure_line(), file=stream, flush=True)
+        return kept
+    failure = build_failure(flow_dir, command, metadata, case, evidence)
+    failure = write_record(failure, evidence)
+    print(failure.native_failure_line(), file=stream, flush=True)
+    return failure
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(
+        description="Convert a Maestro test-output session or Flow folder into the device failure record."
+    )
+    parser.add_argument(
+        "--maestro-dir",
+        required=True,
+        help="Maestro --test-output-dir session folder, or one Flow folder holding commands.json",
+    )
+    parser.add_argument("--case", help="case name for the record; defaults to the Flow folder name")
+    parser.add_argument(
+        "--evidence-dir",
+        help="run evidence folder that holds failure.json; defaults to the Flow folder",
+    )
+    options = parser.parse_args(argv)
+    if not os.path.isdir(options.maestro_dir):
+        print("maestro folder not found: %s" % options.maestro_dir, file=sys.stderr)
+        return 2
+    try:
+        convert(options.maestro_dir, case=options.case, evidence_dir=options.evidence_dir)
+    except MaestroOutputError as problem:
+        print("not converted: %s" % problem, file=sys.stderr)
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())