archal 0.9.18 → 0.9.20

package/dist/vitest/runtime/hosted-session-reaper.js

@@ -4,7 +4,7 @@ import {
   createHostedAuthLease,
   parsePositiveInteger,
   runHostedSessionReaper
-} from "../chunk-CTSN67QR.js";
+} from "../chunk-ARVS45PP.js";
 
 // src/runtime/hosted-session-reaper.ts
 var VITEST_AUTH_LEASE_OPTIONS = {

package/dist/vitest/runtime/setup-files.js

@@ -1,7 +1,7 @@
 import {
   bootstrapArchalVitestRouting
-} from "../chunk-IVXSSEYS.js";
-import "../chunk-CTSN67QR.js";
+} from "../chunk-6CBYFCFK.js";
+import "../chunk-ARVS45PP.js";
 
 // src/runtime/setup-files.ts
 import { existsSync, rmSync } from "fs";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archal",
-  "version": "0.9.18",
+  "version": "0.9.20",
   "description": "Test your agents & integrations against service clones",
   "type": "module",
   "main": "dist/index.cjs",
@@ -17,6 +17,10 @@
       "types": "./dist/harness.d.cts",
       "default": "./dist/harness.cjs"
     },
+    "./scenarios": {
+      "types": "./dist/scenarios.d.cts",
+      "default": "./dist/scenarios.cjs"
+    },
     "./vitest": {
       "types": "./dist/vitest/index.d.ts",
       "import": "./dist/vitest/index.js",
@@ -47,15 +51,9 @@
     "dist",
     "skills",
     "clone-assets",
+    "agents",
     "LICENSE"
   ],
-  "scripts": {
-    "verify:artifacts": "node scripts/assert-artifacts.mjs",
-    "prepack": "pnpm run verify:artifacts",
-    "prepare": "node scripts/prepare.cjs",
-    "typecheck:raw": "node --check bin/archal.cjs && node --check scripts/assert-artifacts.mjs && node --check scripts/prepare.cjs",
-    "typecheck": "pnpm run typecheck:raw"
-  },
   "peerDependencies": {
     "vitest": ">=2.1.0"
   },
@@ -65,6 +63,13 @@
     }
   },
   "dependencies": {
+    "@aws-sdk/client-secrets-manager": "^3.1065.0",
+    "e2b": "^2.28.2",
     "picomatch": "^4.0.4"
+  },
+  "scripts": {
+    "verify:artifacts": "node scripts/assert-artifacts.mjs",
+    "typecheck:raw": "node --check bin/archal.cjs && node --check scripts/assert-artifacts.mjs && node --check scripts/prepare.cjs",
+    "typecheck": "pnpm run typecheck:raw"
   }
-}
+}

package/skills/archal-agent/SKILL.md

@@ -0,0 +1,87 @@
+---
+name: archal-agent
+description: The front door for using Archal to test, debug, and fix an AI agent. START HERE, then route to the right sub-skill instead of guessing. Use when the user says "use Archal", "test my agent", "set up Archal", "my agent is failing", "reproduce this production failure", "grade my traces", or otherwise wants to evaluate, debug, or auto-fix an agent against service clones. Pick this whenever the request is Archal-shaped but the specific workflow is unclear.
+user-invocable: true
+argument-hint: "[what you want to do with your agent]"
+---
+
+# Archal
+
+You are the entry point for Archal. Archal is the QA layer for AI agents: it
+runs an agent against stateful behavioral clones of real services (GitHub,
+Slack, Stripe, Linear, Jira, Supabase, and more), scores how well it satisfies
+each scenario, and turns failures into reproductions and PR fixes. Your job here
+is to orient the operator and route to the sub-skill that owns their workflow.
+Do not inline those flows; hand off by exact name and let the sub-skill drive.
+
+## Product mental model
+
+Archal tests AI agents against service clones instead of real services, so every
+run is deterministic and replayable. You describe a task and success criteria,
+the agent runs against clones, and an evaluator scores satisfaction (a
+probability, not pass/fail). When a real production trace shows a failure, Archal
+reproduces it on clones from trace evidence and ships the fix as a GitHub PR.
+Everything aims at the same thing: deterministic, replayable evals you can trust.
+
+## Decision guide
+
+| I want to... | Route to |
+|--------------|----------|
+| Set up Archal in a repo from scratch (install, auth, detect clones) | `onboard` |
+| Write or edit a scenario test file | `scenario` |
+| Load explicit JSON/SQL/catalog state into a clone (deterministic, no LLM) | `seed` |
+| Run scenarios or tasks and interpret satisfaction scores and failures | `eval` |
+| Wire clones into an existing Vitest suite | `vitest` |
+| Connect a repo's production observability so traces get captured | `install-agent` |
+| Run the autoloop (ingest -> grade -> find-failed -> reproduce-on-clones) and ship the fix as a PR (autofix) over real trace sources | `autoloop` |
+| Turn autofix or autoloop on/off for an agent | `autoloop`; the copilot can toggle either |
+
+If the user is brand new and has none of this set up, start with `onboard`; it
+detects clones and routes onward from there.
+
+## The sub-skills
+
+Each lives in `packages/archal/skills/` and owns its own commands, contracts,
+and mental model. Route by exact name:
+
+- `onboard` — set up Archal in a repo from scratch: install the CLI, handle auth,
+  detect which clones the agent needs, and hand off to the right workflow.
+- `scenario` — author and edit scenario test files (Setup / Prompt / Expected
+  Behavior / Success Criteria) that `archal run` executes against clones.
+- `seed` — load explicit JSON, SQL, or catalog state into a clone deterministically,
+  with no LLM in the loop, so runs start from a known fixture state.
+- `eval` — run scenarios or inline tasks against clones and interpret the results:
+  satisfaction scores, `[D]` vs `[P]` criteria, trace inspection, failure diagnosis.
+- `vitest` — wire clones into an existing Vitest suite using the right composition
+  pattern, so the agent's own tests route through clones.
+- `install-agent` — connect a repo's production observability (OTLP, Langfuse,
+  Braintrust, database trace tables) so real agent traces are captured for Archal.
+- `autoloop` — the loop over real trace sources: ingest a trace, grade it for a
+  real failure, find the failed trace, and reproduce it on clones. Autofix (the
+  fix/PR step) is a separate toggle on top of this: when turned on, autoloop
+  reproduces a failure and ships the fix as a PR.
+
+## Autoloop and autofix toggles
+
+Autoloop (ingest -> grade -> find-failed -> reproduce-on-clones) and autofix (the
+fix/PR step) are **separate per-agent toggles**. Autofix is opt-in: it is not part
+of autoloop until it is turned on. Either can be switched on or off per agent from
+the agents tab, the CLI (`--execution-policy reproduce` is autoloop with autofix
+off; `fix` turns autofix on), or by asking the Archal copilot in chat — the
+copilot can toggle either for an agent. When the user asks to turn autofix or
+autoloop on/off for an agent, handle the toggle, then route to `autoloop` for the
+deeper flow.
+
+## Provider-switchable remediation
+
+The Archal copilot is not locked to one model. When autofix reproduces a failure
+and writes a fix, the user can drive that remediation with their own agent —
+`archal preprod` exposes `--remediation-agent auto|codex|claude|cursor` so the
+fix is written by their Claude Code, Cursor, or Codex — or let Archal's managed
+agent do it. Mention this when the user asks who writes the fix or wants to use
+their own coding agent, then route to `autoloop`.
+
+## Docs
+
+- Quickstart: https://docs.archal.ai/quickstart
+- Full docs: https://docs.archal.ai

package/skills/autoloop/SKILL.md

@@ -0,0 +1,376 @@
+---
+name: autoloop
+description: Wire Archal Autoloop to a repo plus a real agent-trace source, then drive the import -> grade -> reproduce -> PR-fix loop. USE THIS whenever the user wants to turn production agent traces into reproducible failures and fixes: "I have prod agent traces and want to reproduce a failure", "import my Langfuse / Braintrust / OTel / Supabase traces", "connect a trace source", "grade my prod traces", "turn a failed trace into a PR", "set up / configure the autoloop", or any mention of replaying, grading, or auto-fixing real traces. Also fires when diagnosing a stuck import, grade, reproduction, or PR-fix run, or configuring trace schema mapping.
+user-invocable: true
+argument-hint: "[repo, trace source, or failure description]"
+---
+
+# Archal Autoloop
+
+You help users connect real agent traces to Archal. Your job is to wire the repo,
+trace source, harness contract, scenario contract, and GitHub PR path without
+guessing or leaking secrets.
+
+Autoloop is not a replacement for `archal run`. It uses the same harness and clone
+routing ideas, but the trigger is a trace that already happened.
+
+Autoloop is also not arbitrary production trace replay. It can reproduce a
+failure only when the trace, scenario contract, and seed templates contain
+enough evidence to reconstruct the service state that matters. If the evidence
+is thin, block and name the missing data instead of claiming reproduction.
+
+## Product mental model
+
+Autoloop does this loop:
+
+1. Import a trace and its child spans from a read-only source.
+2. Grade whether the trace contains a real failure.
+3. Build a reproduction scenario and clone seed from trace evidence plus
+   repo-owned seed templates.
+4. Run the reproduction against service clones through the customer harness.
+5. If reproduced, patch the repo and open a GitHub issue or PR.
+
+Steps 1-4 are **autoloop**: ingest -> grade -> find the failed trace ->
+reproduce on clones. Step 5 (writing the fix and opening the PR) is **autofix**,
+a separate opt-in step that is *not* part of autoloop until it is turned on.
+Both are per-agent toggles, switchable from the agents tab, the CLI, or by asking
+the Archal copilot in chat.
+
+The CLI maps these toggles to `--execution-policy`: `reproduce` runs autoloop
+only with autofix off, while `fix` turns autofix on (autoloop plus the fix/PR
+step). Narrower policies stop earlier:
+
+| Policy | Stops after | Autofix |
+|--------|-------------|---------|
+| `observe` | import | off |
+| `grade` | grading | off |
+| `reproduce` | reproduction | off |
+| `fix` | PR or blocked fix status | on |
+
+Do not invent or promote separate top-level judge, reproduce, fix, or
+trace-replay commands. The public command is `archal autoloop`. Local stop
+command is `archal detach` for file-backed trace directories.
+
+## Discover first
+
+Before changing anything, inspect the repo:
+
+1. `package.json` and scripts: how is the agent run? What tests should a fix PR
+   pass?
+2. Existing Archal files:
+   - `.archal.json`
+   - `.archal/harness.*`
+   - `archal/harness.json`
+   - `archal/scenario.md`
+   - `archal/run-input.yaml`
+   - `archal/seeds/*.json`
+   - `scenarios/*.md`
+3. Service SDKs and likely clones:
+   - `stripe` -> `stripe`
+   - `@octokit/rest`, `octokit` -> `github`
+   - `@slack/web-api`, `@slack/bolt` -> `slack`
+   - `jira`, `jira-client`, `jira.js` -> `jira`
+   - `@linear/sdk` -> `linear`
+   - `@supabase/supabase-js`, `pg` -> `supabase`
+   - `googleapis` -> `google-workspace`
+4. GitHub remote:
+   ```bash
+   git remote get-url origin
+   ```
+   Hosted sources and `--execution-policy fix` need a GitHub remote.
+5. Trace source shape, if available:
+   - provider: local files, Postgres, Supabase
+   - or local/client-ingested sources normalized through `archal trace-source`
+     such as file, HTTP/OTel, Langfuse, Braintrust, S3/GCS, or custom JSON
+   - trace table and span table names
+   - id columns
+   - parent span column
+   - timestamp/cursor columns
+   - status, workspace, trace group, and agent filters
+
+Never print secrets while inspecting. If you need to show a database URL, show
+only the env var name or secret reference.
+
+## Preconditions
+
+You need these before a full hosted Autoloop setup:
+
+- Archal CLI installed in the repo or reachable with `npx archal`
+- authenticated user (`archal login`) or `ARCHAL_TOKEN=archal_ws_...`
+- GitHub App installed on the target repo
+- repo origin that resolves to `github.com/<owner>/<repo>`
+- read-only trace source credential
+- headless harness command
+- `archal/scenario.md` for the trace family
+- optional but strongly recommended `archal/seeds/*.json`
+- model/provider keys required by the user's agent and tests
+
+If any prerequisite is missing, make the smallest safe change and explain what
+is still required. Do not continue into a fake reproduction.
+
+## Safe setup flow
+
+### 1. Install and authenticate
+
+Prefer project-local install:
+
+```bash
+npm install -D archal
+npx archal login
+npx archal usage
+```
+
+In CI or a customer repo, use:
+
+```bash
+export ARCHAL_TOKEN=archal_ws_...
+npx archal usage
+```
+
+Use a workspace API key for automated runs. Do not commit it.
+
+### 2. Add or verify `archal/harness.json`
+
+Minimal shape:
+
+```json
+{
+  "version": 1,
+  "local": {
+    "command": "node",
+    "args": ["agent.mjs"]
+  }
+}
+```
+
+The command must be headless and repeatable. It should run the real agent path,
+not a hand-authored mock. During reproduction, Archal invokes this command
+through `archal run`, so the agent should read the task from `AGENT_TASK` and
+print its final answer to stdout. If the project already has `.archal.json` for
+`archal run`, align the Autoloop harness with that command.
+
+### 3. Add or verify `archal/scenario.md`
+
+The scenario describes the standing task and checks for this trace family.
+
+Required sections:
+
+```md
+# Scenario title
+
+## Setup
+Trace-family context and the starting state Archal should reconstruct.
+
+## Prompt
+The task the agent should complete.
+
+## Expected Behavior
+The answer key for grading and reproduction.
+
+## Success Criteria
+- [D] Deterministic clone-state check
+- [P] Probabilistic trace/output check
+
+## Config
+clones: stripe, slack
+timeout: 120
+```
+
+Keep model-visible instructions realistic. Do not tell the tested agent that it
+is in Archal, a clone-backed environment, or a special replay.
+
+### 4. Add seed templates when trace evidence is thin
+
+Recommended:
+
+```text
+archal/seeds/
+  stripe-billing-support.json
+  jira-escalations.json
+```
+
+Seed templates should contain stable service state for the task family. Autoloop
+can then fill in trace-specific identifiers. This is much safer than expecting
+weak traces to reconstruct full service state.
+
+## Hosted database source
+
+Use this when traces live in Postgres or Supabase. Create a read-only DB user,
+keep the URL in `TRACE_DATABASE_URL` (or a secret ref in hosted production), then
+`--check` the source and re-run without `--check` to register it. Registration
+posts the source config to Archal and returns; hosted workers own polling after
+that, so local `archal detach` does not disable it.
+
+See `references/hosted-sources.md` for the full check, register, and
+`--database-url-secret-ref` flag blocks.
+
+## Client-side trace ingestion
+
+Use `archal trace-source` when traces are not already in a hosted Postgres or
+Supabase table. This command normalizes source-specific payloads into Archal
+trace upload envelopes, writes them to a trace directory, and can upload them to
+hosted Autoloop when workspace auth is available.
+
+Common paths:
+
+```bash
+npx archal trace-source import ./exports --preview --json
+npx archal trace-source import ./exports --upload --repository owner/repo
+
+npx archal trace-source connect langfuse \
+  --base-url https://cloud.langfuse.com \
+  --api-key-env LANGFUSE_READ_KEY \
+  --out .archal/traces/inbox
+npx archal trace-source test langfuse
+npx archal trace-source sync langfuse --upload --repository owner/repo
+npx archal trace-source watch langfuse --upload --repository owner/repo
+
+npx archal trace-source connect custom --name "prod exporter" --out .archal/traces/inbox
+npx archal trace-source serve "prod exporter" --port 4319
+```
+
+Use `archal trace-source status [source]` to inspect registry validation,
+cursor, and last-sync state. `watch` is for pull-style sources; push sources
+stay continuous through `serve`.
+
+## Trace schema mapping
+
+Hosted sources default to `ai_traces` / `ai_spans` with `id` / `trace_id`
+columns and `updated_at_id` cursor mode. When the customer's tables differ, pass
+mapping flags to override table names, id columns, parent-span column, and
+cursor columns; switch to `created_at_id` cursor mode for append-only sources;
+and use `--source-*` filters to scope noisy sources by workspace, agent, status,
+trace group, or limit.
+
+See `references/trace-schema-mapping.md` for the full defaults table plus the
+custom-schema, append-only, and filter flag blocks.
+
+## Local trace directory
+
+Use this for a local pilot or exported trace files:
+
+```bash
+npx archal autoloop ./prod-traces --repo . --execution-policy reproduce
+```
+
+Artifacts are written under:
+
+```text
+.archal/autoloop/
+  autoloops.json
+  runs.jsonl
+  raw/
+  grades/
+  seeds/
+  runs/
+  fixes/
+  failed/
+  logs/
+```
+
+Stop the local file-backed loop:
+
+```bash
+npx archal detach ./prod-traces --repo .
+```
+
+Do not describe `archal detach` as a hosted source disable command.
+
+## CLI-first operation
+
+Prefer CLI and artifact evidence for handoffs unless the user explicitly asks
+for a workspace page.
+
+- Local file-backed loops: `archal autoloop <trace-dir> --repo ...` starts the
+  watcher, `archal detach <trace-dir> --repo ...` stops it, `archal
+  autoloop-status --repo ...` summarizes trace jobs, and `archal
+  autoloop-reprocess --repo ... <trace-id>` retries terminal jobs after a
+  blocker is fixed.
+- Hosted database sources: `archal autoloop --source postgres|supabase ...`
+  registers the source and returns. Local `archal detach` does not disable a
+  hosted source because hosted workers own polling after registration.
+- Safe resume means re-running the same `archal autoloop` registration or
+  reprocessing a terminal local trace only after the missing evidence,
+  credential, mapping, harness, or GitHub blocker is corrected.
+
+Report exact artifact paths and statuses. Do not make dashboard pages the only
+place a user can understand what happened.
+
+## How to diagnose failures
+
+Classify failures precisely:
+
+- Trace import failure: database/source auth, mapping, cursor, filters, bad
+  trace shape.
+- Trace ingestion failure: `trace-source` adapter mismatch, rejected hosted
+  upload, missing workspace auth, bad idempotency key, or receiver auth failure.
+- Grade failure: judge could not determine expected outcome, missing evaluator
+  contract, trace lacks task context.
+- Missing evidence: trace does not contain enough state to seed. Add spans,
+  state snapshots, or repo-owned seed templates.
+- Reproduction failure: scenario or seed could not replay the failure against
+  clones. Inspect generated `scenario.md`, `seed.json`, and run manifest.
+- Agent behavior: the reproduced run shows the agent making the same wrong
+  service action it made in the original trace.
+- Harness issue: the agent command crashes, hangs, needs UI auth, or does not
+  reach clone-routed services.
+- Fix generation issue: patch does not apply, tests fail, no changes produced,
+  or generated PR metadata is incomplete.
+- GitHub issue: GitHub App missing, branch protection, permission denied, PR
+  checks unavailable.
+
+When evidence is insufficient, say so directly. Do not manufacture a seed or
+claim reproduction succeeded.
+
+## Artifact reading guide
+
+Local file-backed Autoloop uses repo-local artifacts. Hosted Autoloop exposes the
+same phase information in the dashboard.
+
+| Artifact | What to inspect |
+|----------|-----------------|
+| `grades/<trace>/routing.json` | trace import route and selected phase |
+| `grades/<trace>/grade.json` | verdict, summary, and reproduction decision |
+| `seeds/<trace>/scenario.md` | generated reproduction scenario |
+| `seeds/<trace>/seed.json` | generated seed request or materialized seed metadata |
+| `runs/<trace>/manifest.json` | reproduction status, command, attempts, evidence |
+| `runs/<trace>/stdout.json` | machine-readable run output |
+| `runs/<trace>/stderr.log` | reproduction stderr |
+| `fixes/<trace>/status.json` | blocked fix status |
+| `fixes/<trace>/pr-details.md` | PR reviewer summary |
+| `fixes/<trace>/repo.patch` | patch captured when PR creation cannot complete |
+
+## Security rules
+
+- Use read-only trace database credentials.
+- Never commit database URLs, API keys, model keys, or GitHub tokens.
+- Prefer `--database-url-env` locally and `--database-url-secret-ref` in hosted
+  production setup.
+- Do not pass production write credentials to a clone-routed reproduction.
+- Do not add model-visible copy that reveals Archal or clone routing to the
+  tested agent.
+- Do not bypass the GitHub App PR workflow with direct pushes.
+- Redact raw trace payloads before sharing artifacts outside the workspace.
+
+## What to report back
+
+After setup or debugging, give the user:
+
+- command run
+- source provider and source id
+- repo full name
+- execution policy
+- CLI status command to run next
+- artifacts produced, if local
+- whether import, grade, seed, reproduce, and fix phases are ready
+- exact blocker if any
+- next command or next owner
+
+## Docs
+
+- Autoloop production traces: https://docs.archal.ai/guides/autoloop-production-traces
+- Autonomous loops: https://docs.archal.ai/guides/autoloop-production-traces
+- CLI reference: https://docs.archal.ai/cli/autoloop
+- Running with an agent: https://docs.archal.ai/guides/run-with-agent
+- Writing scenarios: https://docs.archal.ai/guides/writing-scenarios
+- Seeds: https://docs.archal.ai/guides/seeds

package/skills/autoloop/references/hosted-sources.md

@@ -0,0 +1,62 @@
+# Hosted database source
+
+Full flag reference for registering a hosted Postgres or Supabase trace source.
+Use this when traces live in Postgres or Supabase. Registration posts the source
+config to Archal and returns; hosted workers own polling after that.
+
+## Contents
+
+- Read-only credential
+- Check the source
+- Register the source
+- Secret reference (hosted production)
+
+## Read-only credential
+
+First, create or request a read-only database user. Then keep the URL in an env
+var:
+
+```bash
+export TRACE_DATABASE_URL='postgres://readonly:...'
+```
+
+## Check the source
+
+Run a check:
+
+```bash
+npx archal autoloop \
+  --repo . \
+  --source supabase \
+  --database-url-env TRACE_DATABASE_URL \
+  --source-id prod-agent-traces \
+  --check
+```
+
+## Register the source
+
+Register:
+
+```bash
+npx archal autoloop \
+  --repo . \
+  --source supabase \
+  --database-url-env TRACE_DATABASE_URL \
+  --source-id prod-agent-traces
+```
+
+This posts the source config to Archal and returns. It does not start a local
+watcher and does not write local source state.
+
+## Secret reference (hosted production)
+
+Use a secret reference when the customer already has one:
+
+```bash
+npx archal autoloop \
+  --repo . \
+  --source postgres \
+  --database-url-secret-ref aws-secretsmanager://customer/prod-agent-traces
+```
+
+`--database-url-secret-ref` must not contain a plaintext credential.

package/skills/autoloop/references/trace-schema-mapping.md

@@ -0,0 +1,73 @@
+# Trace schema mapping
+
+Full mapping-flag reference for hosted Postgres/Supabase trace sources. Use this
+when the customer's trace and span tables do not match the defaults below.
+
+## Contents
+
+- Defaults
+- Custom schema flags
+- Append-only sources
+- Filters for noisy sources
+
+## Defaults
+
+| Concept | Default |
+|---------|---------|
+| trace table | `ai_traces` |
+| span table | `ai_spans` |
+| trace id | `id` |
+| span id | `id` |
+| span trace id | `trace_id` |
+| trace updated cursor | `updated_at` |
+| span updated cursor | `updated_at` |
+| cursor mode | `updated_at_id` |
+
+## Custom schema flags
+
+For a custom schema, pass mapping flags:
+
+```bash
+npx archal autoloop \
+  --repo . \
+  --source postgres \
+  --database-url-env TRACE_DATABASE_URL \
+  --trace-table public.agent_traces \
+  --span-table public.agent_spans \
+  --trace-id-column trace_id \
+  --span-id-column span_id \
+  --span-trace-id-column trace_id \
+  --parent-span-id-column parent_span_id \
+  --trace-updated-at-column updated_at \
+  --span-updated-at-column updated_at
+```
+
+## Append-only sources
+
+For append-only sources:
+
+```bash
+npx archal autoloop \
+  --repo . \
+  --source supabase \
+  --database-url-env TRACE_DATABASE_URL \
+  --cursor-mode created_at_id \
+  --trace-created-at-column created_at \
+  --span-created-at-column created_at
+```
+
+## Filters for noisy sources
+
+Use filters for noisy sources:
+
+```bash
+npx archal autoloop \
+  --repo . \
+  --source supabase \
+  --database-url-env TRACE_DATABASE_URL \
+  --source-workspace-id workspace_123 \
+  --source-agent-id support-agent \
+  --source-status failed error \
+  --source-trace-group billing-support \
+  --source-limit 250
+```