agent-scenario-loop 0.1.0

package/schemas/scenario.schema.json

@@ -0,0 +1,400 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agent-scenario-loop.dev/schemas/scenario.schema.json",
+  "title": "Agent Scenario Loop Scenario",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemaVersion",
+    "id",
+    "flowId",
+    "platforms",
+    "requiredCapabilities",
+    "truthEvents",
+    "steps"
+  ],
+  "properties": {
+    "schemaVersion": {
+      "type": "string",
+      "pattern": "^1\\.0\\.0$"
+    },
+    "id": {
+      "type": "string",
+      "pattern": "^[a-z0-9][a-z0-9-]*$"
+    },
+    "flowId": {
+      "type": "string",
+      "pattern": "^[a-z0-9][a-z0-9-]*$"
+    },
+    "comparisonLane": {
+      "type": "string"
+    },
+    "description": {
+      "type": "string"
+    },
+    "journey": {
+      "$ref": "#/$defs/journey"
+    },
+    "platforms": {
+      "type": "array",
+      "minItems": 1,
+      "uniqueItems": true,
+      "items": {
+        "type": "string",
+        "enum": ["ios", "android"]
+      }
+    },
+    "requiredCapabilities": {
+      "type": "array",
+      "minItems": 1,
+      "uniqueItems": true,
+      "items": {
+        "$ref": "#/$defs/capability"
+      }
+    },
+    "optionalCapabilities": {
+      "type": "array",
+      "uniqueItems": true,
+      "items": {
+        "$ref": "#/$defs/capability"
+      }
+    },
+    "truthEvents": {
+      "type": "object",
+      "minProperties": 1,
+      "additionalProperties": {
+        "$ref": "#/$defs/truthMilestone"
+      }
+    },
+    "milestones": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "$ref": "#/$defs/milestone"
+      }
+    },
+    "expectedEvents": {
+      "type": "array",
+      "minItems": 1,
+      "uniqueItems": true,
+      "items": {
+        "type": "string"
+      }
+    },
+    "cycles": {
+      "$ref": "#/$defs/cycles"
+    },
+    "budgets": {
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/budget"
+      }
+    },
+    "steps": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "$ref": "#/$defs/step"
+      }
+    },
+    "artifacts": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "required": {
+          "type": "array",
+          "uniqueItems": true,
+          "items": {
+            "$ref": "#/$defs/artifactKind"
+          }
+        },
+        "optional": {
+          "type": "array",
+          "uniqueItems": true,
+          "items": {
+            "$ref": "#/$defs/artifactKind"
+          }
+        }
+      }
+    },
+    "adapterOptions": {
+      "$ref": "#/$defs/adapterOptions"
+    }
+  },
+  "$defs": {
+    "journey": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "intent"],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "intent": {
+          "type": "string"
+        },
+        "actor": {
+          "type": "string"
+        },
+        "startState": {
+          "type": "string"
+        },
+        "endState": {
+          "type": "string"
+        }
+      }
+    },
+    "cycles": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "iterations": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "warmupIterations": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "stopOnFailure": {
+          "type": "boolean",
+          "default": true
+        }
+      }
+    },
+    "capability": {
+      "type": "string",
+      "enum": [
+        "launch",
+        "sessionControl",
+        "command",
+        "logCapture",
+        "artifactWrite",
+        "screenshot",
+        "video",
+        "uiTree",
+        "memory",
+        "network",
+        "profiler",
+        "accessibility"
+      ]
+    },
+    "artifactKind": {
+      "type": "string",
+      "enum": [
+        "logs",
+        "screenshot",
+        "video",
+        "uiTree",
+        "memory",
+        "network",
+        "profiler",
+        "accessibility",
+        "signals"
+      ]
+    },
+    "truthMilestone": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["event"],
+      "properties": {
+        "event": {
+          "type": "string"
+        },
+        "required": {
+          "type": "boolean",
+          "default": true
+        },
+        "timeoutMs": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "phase": {
+          "type": "string",
+          "enum": [
+            "intent",
+            "navigation",
+            "domain",
+            "query",
+            "network",
+            "render",
+            "native",
+            "visual",
+            "completion"
+          ]
+        }
+      }
+    },
+    "milestone": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "event"],
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^[a-zA-Z0-9][a-zA-Z0-9_-]*$"
+        },
+        "event": {
+          "type": "string"
+        },
+        "description": {
+          "type": "string"
+        },
+        "required": {
+          "type": "boolean",
+          "default": true
+        },
+        "timeoutMs": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "phase": {
+          "type": "string",
+          "enum": [
+            "intent",
+            "navigation",
+            "domain",
+            "query",
+            "network",
+            "render",
+            "native",
+            "visual",
+            "completion"
+          ]
+        }
+      }
+    },
+    "budget": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "source", "metric", "unit", "limit"],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "source": {
+          "type": "string",
+          "enum": ["milestone", "artifact"]
+        },
+        "metric": {
+          "type": "string"
+        },
+        "unit": {
+          "type": "string",
+          "enum": ["ms", "count", "bytes", "percent", "boolean"]
+        },
+        "limit": {
+          "type": ["number", "integer", "boolean"]
+        },
+        "fromMilestone": {
+          "type": "string"
+        },
+        "toMilestone": {
+          "type": "string"
+        },
+        "artifact": {
+          "$ref": "#/$defs/artifactKind"
+        }
+      }
+    },
+    "step": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind"],
+      "properties": {
+        "id": {
+          "type": "string"
+        },
+        "kind": {
+          "type": "string",
+          "enum": [
+            "launch",
+            "command",
+            "waitForMilestone",
+            "captureEvidence",
+            "gesture",
+            "assertUi"
+          ]
+        },
+        "command": {
+          "type": "string"
+        },
+        "milestone": {
+          "type": "string"
+        },
+        "artifact": {
+          "$ref": "#/$defs/artifactKind"
+        },
+        "driverAction": {
+          "$ref": "#/$defs/driverAction"
+        },
+        "selector": {
+          "$ref": "#/$defs/selector"
+        },
+        "timeoutMs": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "required": {
+          "type": "boolean",
+          "default": true
+        },
+        "adapterOptions": {
+          "$ref": "#/$defs/adapterOptions"
+        }
+      }
+    },
+    "adapterOptions": {
+      "type": "object",
+      "additionalProperties": true
+    },
+    "driverAction": {
+      "type": "string",
+      "enum": [
+        "tap",
+        "scroll",
+        "assertVisible",
+        "inspectTree",
+        "screenshot",
+        "record",
+        "readLogs",
+        "collectPerfSignals"
+      ]
+    },
+    "selector": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "value"],
+      "properties": {
+        "kind": {
+          "type": "string",
+          "enum": [
+            "testId",
+            "accessibilityId",
+            "accessibilityLabel",
+            "text",
+            "resourceId",
+            "xpath"
+          ]
+        },
+        "value": {
+          "type": "string",
+          "minLength": 1
+        },
+        "match": {
+          "type": "string",
+          "enum": ["exact", "contains", "regex"]
+        },
+        "platforms": {
+          "type": "array",
+          "minItems": 1,
+          "uniqueItems": true,
+          "items": {
+            "type": "string",
+            "enum": ["ios", "android"]
+          }
+        }
+      }
+    }
+  }
+}

package/schemas/verdict.schema.json

@@ -0,0 +1,88 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agent-scenario-loop.dev/schemas/verdict.schema.json",
+  "title": "Agent Scenario Loop Verdict",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schemaVersion", "scenarioId", "runId", "healthStatus", "verdictStatus"],
+  "properties": {
+    "schemaVersion": {
+      "type": "string",
+      "pattern": "^1\\.0\\.0$"
+    },
+    "scenarioId": {
+      "type": "string"
+    },
+    "flowId": {
+      "type": "string"
+    },
+    "runId": {
+      "type": "string"
+    },
+    "healthStatus": {
+      "type": "string",
+      "enum": ["passed", "failed", "partial"]
+    },
+    "verdictStatus": {
+      "type": "string",
+      "enum": ["passed", "failed", "inconclusive", "not_evaluated"]
+    },
+    "budgetChecks": {
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/budgetCheck"
+      }
+    },
+    "comparison": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "artifact": {
+          "type": "string"
+        },
+        "status": {
+          "type": "string",
+          "enum": ["better", "worse", "unchanged", "inconclusive"]
+        }
+      }
+    },
+    "summary": {
+      "type": "string"
+    }
+  },
+  "$defs": {
+    "budgetCheck": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "source", "metric", "unit", "expected", "actual", "pass"],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "source": {
+          "type": "string",
+          "enum": ["milestone", "artifact"]
+        },
+        "metric": {
+          "type": "string"
+        },
+        "unit": {
+          "type": "string",
+          "enum": ["ms", "count", "bytes", "percent", "boolean"]
+        },
+        "expected": {
+          "type": ["number", "integer", "boolean"]
+        },
+        "actual": {
+          "type": ["number", "integer", "boolean", "null"]
+        },
+        "pass": {
+          "type": "boolean"
+        },
+        "notes": {
+          "type": "string"
+        }
+      }
+    }
+  }
+}

package/templates/evidence-provider.json

@@ -0,0 +1,83 @@
+{
+  "schemaVersion": "1.0.0",
+  "runnerId": "example-evidence-provider",
+  "kind": "evidenceProvider",
+  "platforms": ["ios", "android"],
+  "capabilities": ["accessibility", "memory", "network", "profiler"],
+  "driverActions": ["collectPerfSignals"],
+  "artifactOutputs": ["accessibility", "memory", "network", "profiler", "signals"],
+  "lifecycle": ["prepare", "startWindow", "capture", "stopWindow", "finalize"],
+  "providerCommands": [
+    {
+      "id": "capture-accessibility",
+      "phase": "capture",
+      "command": "node",
+      "cwd": "..",
+      "args": [
+        "scripts/asl-capture-accessibility-provider.mjs",
+        "--platform",
+        "{platform}",
+        "--scenario",
+        "{scenarioId}",
+        "--run-id",
+        "{runId}",
+        "--out",
+        "{providerDir}/accessibility.json"
+      ],
+      "outputs": [
+        {
+          "channel": "provider",
+          "kind": "accessibility",
+          "path": "{providerDir}/accessibility.json"
+        }
+      ]
+    },
+    {
+      "id": "capture-profiler",
+      "phase": "capture",
+      "command": "node",
+      "cwd": "..",
+      "args": [
+        "scripts/asl-capture-profiler-provider.mjs",
+        "--platform",
+        "{platform}",
+        "--scenario",
+        "{scenarioId}",
+        "--run-id",
+        "{runId}",
+        "--out",
+        "{providerDir}/profiler.json",
+        "--memory-out",
+        "{providerDir}/memory.json",
+        "--network-out",
+        "{providerDir}/network.har"
+      ],
+      "outputs": [
+        {
+          "channel": "provider",
+          "kind": "profiler",
+          "path": "{providerDir}/profiler.json"
+        },
+        {
+          "channel": "signal",
+          "kind": "js",
+          "path": "{providerDir}/profiler.json"
+        },
+        {
+          "channel": "signal",
+          "kind": "memory",
+          "path": "{providerDir}/memory.json"
+        },
+        {
+          "channel": "signal",
+          "kind": "network",
+          "path": "{providerDir}/network.har"
+        }
+      ]
+    }
+  ],
+  "adapterOptionsSchema": {
+    "signalPath": "signals/js",
+    "window": "scenario"
+  }
+}

package/templates/gitignore-snippet

@@ -0,0 +1,9 @@
+# Agent Scenario Loop runtime artifacts
+.asl.local.env
+artifacts/asl/
+artifacts/example-mobile-app/
+
+# Device/simulator evidence captured during local proof runs
+*.memgraph
+*.trace
+*.xcresult