@mmerterden/multi-agent-toolkit-mcp 3.4.0 → 3.5.0

package/CHANGELOG.md

@@ -15,6 +15,44 @@ Releases before this file exists are recorded in the git tags and commit history
 
 ---
 
+## 3.5.0
+
+### Added
+
+- **`ios_accessibility_audit` checks the reading order.** A screen with perfect
+  labels is still unusable if VoiceOver walks it in the wrong order, and nothing
+  was looking. The audit compares the reading sequence against the visual layout
+  and reports the first element that arrives out of place.
+
+  The comparison is rows-then-columns, not a naive top-to-bottom sort: a home
+  screen is a grid, a toolbar is a row, and sorting by `y` alone calls every
+  correct grid broken. Elements share a row when their vertical extents overlap
+  by more than half the shorter one, which absorbs the few points of jitter real
+  layouts carry. `rtl: true` expects right-to-left within a row.
+
+  It is an `important` finding and never a `critical` one: a deliberate reading
+  order that differs from the visual one is legitimate, so this is a question
+  worth asking rather than a proven defect. Verified on a live home screen grid:
+  13 elements, reading order correct, no false positive.
+
+- **Traits and hints.** The dumper now reads `AXRoleDescription` (what a screen
+  reader announces the element AS) and `AXHelp` (where the iOS
+  `accessibilityHint` bridges to). A control with no role announcement is read
+  out as bare text and the user cannot tell it is actionable, so that is an
+  `important` finding. A missing hint is not: hints are not required on every
+  element, so the result reports `hints_present` as a count and leaves the
+  judgement to the reader.
+
+  `AXSubrole` was probed first and carries nothing at all through this bridge,
+  so it is not read. Guessing at it would have produced a check that never fires.
+
+### Note
+
+Stage 2 of 4. Contrast, Dynamic Type and clipped text still cannot be reached
+from a tree dump; they need Apple's own `XCUIAccessibilityAudit`, which is an
+XCUITest run rather than a simctl call. The Android side is stage 4 and is
+unverifiable on a machine with no adb.
+
 ## 3.4.0
 
 ### Fixed

package/index.js

@@ -351,7 +351,7 @@ const IOS_TOOLS = [
   { name: "ios_get_app_container", description: "Get iOS app container path", inputSchema: { type: "object", properties: { bundle_id: { type: "string" }, container: { type: "string", enum: ["app", "data", "groups"] }, device_id: { type: "string" } }, required: ["bundle_id"] } },
   { name: "ios_erase_device", description: "Factory reset iOS simulator", inputSchema: { type: "object", properties: { device_id: { type: "string" } } } },
   { name: "ios_get_ui_tree", description: "Get iOS accessibility UI tree via macOS AX APIs. Pass `path` to write the raw dump to a file and return only its location - the full tree can be tens of KB.", inputSchema: { type: "object", properties: { max_depth: { type: "number" }, path: { type: "string", description: "Absolute file path to write the raw tree JSON to. The parent directory must already exist." } } } },
-  { name: "ios_accessibility_audit", description: "Audit iOS app accessibility: missing labels, small tap targets (<44pt), missing identifiers. Use scope to filter by identifier prefix (e.g. 'login_' only checks login screen elements).", inputSchema: { type: "object", properties: { max_depth: { type: "number" }, scope: { type: "string", description: "Filter: only audit elements whose identifier starts with this prefix (e.g. 'login_', 'settings_'). Omit to audit all." } } } },
+  { name: "ios_accessibility_audit", description: "Audit iOS app accessibility on the booted simulator: missing labels, controls a screen reader cannot name, tap targets under 44pt, missing identifiers, and whether the reading order follows the visual layout. Reports measurable:false with a reason rather than a clean result when the tree could not be read - booting a device is not enough, Simulator.app must be running. Scopes to the device screen, never the simulator own chrome.", inputSchema: { type: "object", properties: { max_depth: { type: "number" }, scope: { type: "string", description: "Filter: only audit elements whose identifier starts with this prefix (e.g. 'login_', 'settings_'). Omit to audit all." }, rtl: { type: "boolean", description: "Expect right-to-left reading within a row. Default false." } } } },
   { name: "ios_biometric", description: "Simulate Face ID / Touch ID on iOS simulator (match or nomatch)", inputSchema: { type: "object", properties: { match: { type: "boolean", description: "true=success, false=failure" }, device_id: { type: "string" } }, required: ["match"] } },
   { name: "ios_archive_audit", description: "DEPRECATED - use ios_app_store_audit (18-rule deep scan). Lighter 6-check audit kept for backward compatibility; will be removed in the next major.", inputSchema: { type: "object", properties: { archive_path: { type: "string", description: "Path to .xcarchive" } }, required: ["archive_path"] } },
   { name: "ios_export_ipa", description: "Export a .xcarchive to a signed .ipa via xcodebuild -exportArchive. Generates the exportOptions.plist from the arguments (method defaults to app-store-connect), so callers do not have to hand-maintain one. Returns the .ipa path plus parsed errors; a run that exits 0 without producing an .ipa is reported as a failure. Pair with ios_testflight_validate for the pre-submission gate.", inputSchema: { type: "object", properties: { archive_path: { type: "string", description: "Absolute path to the .xcarchive" }, output_dir: { type: "string", description: "Directory to write the .ipa into" }, method: { type: "string", description: "Export method: app-store-connect (default) | release-testing | enterprise | development" }, team_id: { type: "string", description: "Apple Developer team ID" }, provisioning_profiles: { type: "object", description: "Map of bundleId -> provisioning profile name (manual signing)" }, signing_style: { type: "string", description: "automatic | manual" }, upload_symbols: { type: "boolean", description: "Include symbols (default true)" }, allow_provisioning_updates: { type: "boolean", description: "Off by default. Lets xcodebuild register devices and create/modify provisioning profiles in the developer account - a change on Apple's side, so it is opt-in" }, timeout_sec: { type: "number", description: "Default 900" } }, required: ["archive_path", "output_dir"] } },
@@ -508,13 +508,15 @@ async function handleIOS(name, args, ctx = {}) {
       } catch {
         tree = null;
       }
-      const r = auditIosTree({ tree, scope: args.scope || null });
+      const r = auditIosTree({ tree, scope: args.scope || null, rtl: args.rtl === true });
       return JSON.stringify({
         scope: r.scope,
         measurable: r.measurable,
         reason: r.reason,
         elements_scanned: r.elementsScanned,
         elements_skipped: r.elementsSkipped,
+        reading_order_ok: r.readingOrderOk ?? null,
+        hints_present: r.hintsPresent ?? null,
         total_issues: r.totalIssues,
         critical: r.critical,
         important: r.important,
@@ -1546,6 +1548,8 @@ const ACCESSIBILITY_AUDIT_SCHEMA = {
     reason: { type: ["string", "null"] },
     elements_scanned: { type: "integer" },
     elements_skipped: { type: "integer" },
+    reading_order_ok: { type: ["boolean", "null"] },
+    hints_present: { type: ["integer", "null"] },
     total_issues: { type: ["integer", "null"] },
     critical: { type: ["integer", "null"] },
     important: { type: ["integer", "null"] },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mmerterden/multi-agent-toolkit-mcp",
-  "version": "3.4.0",
+  "version": "3.5.0",
   "description": "MCP server for iOS Simulator, Android Emulator and headless web control. 86 tools: device automation (tap/swipe/type), accessibility audits, visual diff, crash logs, App Store / Play Store pre-submission compliance. Runs standalone over stdio with any MCP client.",
   "type": "module",
   "main": "index.js",

package/tools/a11y/index.js

@@ -65,6 +65,84 @@ export function deviceScreenSubtree(tree) {
   return groups.reduce((best, c) => (area(c) > area(best) ? c : best), groups[0]);
 }
 
+/**
+ * Flatten the interactive elements in the order a screen reader would visit
+ * them, which is the order the accessibility tree presents them in.
+ *
+ * @param {object} node
+ * @param {Set<string>} roles
+ * @returns {object[]}
+ */
+function readingSequence(node, roles) {
+  const out = [];
+  const visit = (n) => {
+    if (!n || typeof n !== "object") return;
+    if (roles.has(n.role)) out.push(n);
+    (n.children || []).forEach(visit);
+  };
+  visit(node);
+  return out;
+}
+
+/**
+ * Group elements into visual rows.
+ *
+ * A screen is not a single top-to-bottom column. A home screen is a grid, a
+ * toolbar is a row, and comparing the reading order against a naive
+ * top-to-bottom sort reports every grid as broken. Two elements share a row
+ * when their vertical extents overlap by more than half the shorter one.
+ *
+ * @param {object[]} els
+ * @returns {object[][]} rows, top to bottom, each sorted left to right
+ */
+export function visualRows(els) {
+  const sorted = [...els].sort((a, b) => (a.frame?.y || 0) - (b.frame?.y || 0));
+  const rows = [];
+  for (const el of sorted) {
+    const y = el.frame?.y || 0;
+    const h = el.frame?.h || 0;
+    const row = rows.find((r) => {
+      const ry = r.top;
+      const rh = r.height;
+      const overlap = Math.min(y + h, ry + rh) - Math.max(y, ry);
+      return overlap > Math.min(h, rh) / 2;
+    });
+    if (row) {
+      row.items.push(el);
+      row.top = Math.min(row.top, y);
+      row.height = Math.max(row.height, h);
+    } else {
+      rows.push({ top: y, height: h, items: [el] });
+    }
+  }
+  return rows.map((r) => r.items.sort((a, b) => (a.frame?.x || 0) - (b.frame?.x || 0)));
+}
+
+/**
+ * Does the reading order follow the visual layout?
+ *
+ * The check is deliberately coarse: it compares the reading sequence against
+ * rows-then-columns, and reports the first element that arrives out of place.
+ * A screen can carry a deliberate reading order that differs from the visual
+ * one, so this is an `important` finding and never a `critical` one: it is a
+ * question worth asking, not a proven defect.
+ *
+ * @param {object[]} sequence - elements in reading order
+ * @param {boolean} rtl - right-to-left layout
+ * @returns {{ok: boolean, firstOutOfPlace: object|null, expectedIndex: number|null}}
+ */
+export function checkReadingOrder(sequence, rtl = false) {
+  if (sequence.length < 2) return { ok: true, firstOutOfPlace: null, expectedIndex: null };
+  const rows = visualRows(sequence);
+  const expected = rows.flatMap((r) => (rtl ? [...r].reverse() : r));
+  for (let i = 0; i < sequence.length; i++) {
+    if (sequence[i] !== expected[i]) {
+      return { ok: false, firstOutOfPlace: sequence[i], expectedIndex: expected.indexOf(sequence[i]) };
+    }
+  }
+  return { ok: true, firstOutOfPlace: null, expectedIndex: null };
+}
+
 /**
  * @param {object} params
  * @param {object|null} params.tree - parsed ui-tree-dumper output
@@ -74,7 +152,7 @@ export function deviceScreenSubtree(tree) {
  *            totalIssues: number|null, critical: number|null,
  *            important: number|null, warning: number|null, issues: object[]}}
  */
-export function auditIosTree({ tree, scope = null }) {
+export function auditIosTree({ tree, scope = null, rtl = false }) {
   if (isDegenerateTree(tree)) {
     return unmeasurable(
       scope,
@@ -95,6 +173,7 @@ export function auditIosTree({ tree, scope = null }) {
   const issues = [];
   let elementsScanned = 0;
   let elementsSkipped = 0;
+  const sequence = [];
 
   const visit = (node, path = "") => {
     if (!node || typeof node !== "object") return;
@@ -109,6 +188,12 @@ export function auditIosTree({ tree, scope = null }) {
         return;
       }
       elementsScanned++;
+      sequence.push(node);
+      // What a screen reader announces the element AS. Without it a control is
+      // read out as bare text and the user cannot tell it is actionable.
+      if (!node.roleDescription) {
+        issues.push({ severity: "important", issue: "No role announced (VoiceOver cannot say what this control is)", element: loc, identifier: node.identifier || null });
+      }
       if (!node.title && !node.description && !node.value) {
         issues.push({ severity: "critical", issue: "Missing accessibility label", element: loc, identifier: node.identifier || null });
       }
@@ -135,12 +220,26 @@ export function auditIosTree({ tree, scope = null }) {
     );
   }
 
+  const order = checkReadingOrder(sequence, rtl);
+  if (!order.ok) {
+    const el = order.firstOutOfPlace;
+    issues.push({
+      severity: "important",
+      issue: `Reading order does not follow the visual layout: this element is read at position ${sequence.indexOf(el) + 1} but sits at position ${order.expectedIndex + 1} on screen`,
+      element: el.role,
+      identifier: el.identifier || null,
+      label: el.title || el.description || el.value || null,
+    });
+  }
+
   return {
     measurable: true,
     reason: null,
     scope: scope || "all",
     elementsScanned,
     elementsSkipped,
+    readingOrderOk: order.ok,
+    hintsPresent: sequence.filter((n) => n.help).length,
     ...tally(issues),
   };
 }

package/ui-tree-dumper.swift

@@ -9,6 +9,12 @@ import Foundation
 
 struct AXNode: Codable {
     let role: String
+    // What a screen reader announces the element AS. Probed on a live simulator:
+    // AXRoleDescription carries "button" and friends, AXSubrole carries nothing
+    // at all through this bridge, so only the former is read.
+    let roleDescription: String?
+    // The iOS accessibilityHint bridges to AXHelp ("Double tap to open").
+    let help: String?
     let title: String?
     let value: String?
     let description: String?
@@ -53,6 +59,8 @@ func dumpElement(_ element: AXUIElement, depth: Int, maxDepth: Int) -> AXNode? {
     let value = getString(element, kAXValueAttribute)
     let desc = getString(element, kAXDescriptionAttribute)
     let identifier = getString(element, kAXIdentifierAttribute)
+    let roleDescription = getString(element, "AXRoleDescription")
+    let help = getString(element, "AXHelp")
     let enabled = getBool(element, kAXEnabledAttribute)
     let focused = getBool(element, kAXFocusedAttribute)
     let frame = getFrame(element)
@@ -75,6 +83,8 @@ func dumpElement(_ element: AXUIElement, depth: Int, maxDepth: Int) -> AXNode? {
 
     return AXNode(
         role: role,
+        roleDescription: roleDescription,
+        help: help,
         title: title,
         value: value,
         description: desc,