@deeeed/metamask-harness 0.71.0 → 0.73.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/adapters/mobile/verify.js +8 -3
- package/dist/commands/run-engine.js +3 -66
- package/dist/discovery.js +15 -0
- package/dist/recipe-assistance-context.js +2 -1
- package/docs/CONTRIBUTING.md +2 -0
- package/docs/cli-contract.md +163 -0
- package/package.json +7 -5
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,20 @@
|
|
|
2
2
|
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
## 0.73.0 - 2026-10-03
|
|
6
|
+
|
|
7
|
+
### Changed
|
|
8
|
+
|
|
9
|
+
- Recipe discovery runs on `@farmslot/recipe-cli`: `run --list` and recipe completion candidates use its readiness rule, and `run --describe` and recipe assistance use its call-graph composition. mm-harness still supplies the view and the policy: only the requested adapter's libraries are loaded, and readiness is judged against the MetaMask manifest `run` executes, so a broken recipe for another platform no longer matters and team manifests follow the same rules as `run --plan`. Flags and output are unchanged; the discovery goldens prove it.
|
|
10
|
+
- Move to the Farmslot release that ships `@farmslot/recipe-cli`: `@farmslot/protocol` 0.34.0, `@farmslot/recipe-harness` 0.22.1, `@farmslot/recipe-cli` 0.1.1, `@farmslot/agent-runtime` 0.16.0 and `@farmslot/expo-recipe` 0.13.1, all on one protocol and one recipe-harness copy. With recipe-harness 0.22, a library's `recipes/terminal/` folder counts as the terminal variant of its plain ids, so `run --list --adapter terminal` with the terminal library also lists `perps.manage-position` and its siblings; the `terminal.perps.*` ids are unchanged.
|
|
11
|
+
|
|
12
|
+
## 0.72.0 - 2026-10-03
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- Freeze the CLI contract the farms, gateway, and recipe libraries depend on: `docs/cli-contract.md` inventories each caller's commands, flags, and parsed outputs, and `yarn test:goldens [--update]` compares `--json` envelopes, exit codes, and runtime layouts against checked-in goldens (also run in the contract CI shards).
|
|
17
|
+
- `MM_HARNESS_MOBILE_VERIFY_PROBE_TIMEOUT_MS` shortens Mobile verify's wait for the React Native debug target (default unchanged at 60 s), so hermetic tests with no Metro do not wait out the probe.
|
|
18
|
+
|
|
5
19
|
## 0.71.0 - 2026-10-02
|
|
6
20
|
|
|
7
21
|
### Added
|
|
@@ -12,7 +12,11 @@ const AUTO_START_TRUE = ["1", "true", "TRUE", "True", "yes", "YES", "Yes", "on",
|
|
|
12
12
|
const AUTO_START_FALSE = ["0", "false", "FALSE", "False", "no", "NO", "No", "off", "OFF", "Off", ""];
|
|
13
13
|
const AUTO_START_REFUSAL = "Mobile auto-start is not allowed from product-local scripts. Start or prepare the app through the runner/slot runtime, then rerun verify with --no-auto-start.";
|
|
14
14
|
const EVM_READINESS_EXPRESSION = "(async function(){try{var engine=globalThis.Engine;var controller=engine&&engine.context&&engine.context.NetworkController;if(!controller||typeof controller.getSelectedNetworkClient!=='function')throw new Error('NetworkController selected client is unavailable');var client=controller.getSelectedNetworkClient();if(!client||!client.provider||typeof client.provider.request!=='function')throw new Error('selected EVM provider is unavailable');var result=await client.provider.request({method:'eth_blockNumber'});if(typeof result!=='string'||!/^0x[0-9a-f]+$/i.test(result))throw new Error('eth_blockNumber returned an invalid result');return {ok:true,result:result};}catch(error){return {ok:false,error:error&&(error.message||String(error))};}})()";
|
|
15
|
-
|
|
15
|
+
function liveSmokeProbeTimeoutMs(env = process.env) {
|
|
16
|
+
const value = Number(env.MM_HARNESS_MOBILE_VERIFY_PROBE_TIMEOUT_MS);
|
|
17
|
+
return Number.isInteger(value) && value > 0 ? value : 6e4;
|
|
18
|
+
}
|
|
19
|
+
const liveSmokeRecipeText = (timeoutMs) => `{
|
|
16
20
|
"$schema": "https://farmslot.io/schemas/recipe-v1.schema.json",
|
|
17
21
|
"title": "Mobile v1 runner live bridge smoke",
|
|
18
22
|
"description": "Verifies the installed MetaMask runner can read the React Native debug bridge without mutating wallet state.",
|
|
@@ -20,7 +24,7 @@ const LIVE_SMOKE_RECIPE = `{
|
|
|
20
24
|
"entry": "status",
|
|
21
25
|
"nodes": {
|
|
22
26
|
"status": { "action": "app.status", "intent": "Read Mobile app status through the v1 runner", "next": "cdp-probe" },
|
|
23
|
-
"cdp-probe": { "action": "cdp.target", "intent": "Verify the React Native debug bridge target is reachable", "require_reachable": true, "timeout_ms":
|
|
27
|
+
"cdp-probe": { "action": "cdp.target", "intent": "Verify the React Native debug bridge target is reachable", "require_reachable": true, "timeout_ms": ${timeoutMs}, "cdp_timeout_ms": ${timeoutMs}, "next": "done" },
|
|
24
28
|
"done": { "action": "end", "status": "pass" }
|
|
25
29
|
}
|
|
26
30
|
}
|
|
@@ -386,7 +390,7 @@ async function runMobileVerify(argv, io = defaultIo) {
|
|
|
386
390
|
}
|
|
387
391
|
}
|
|
388
392
|
const liveSmokeRecipe = path.join(artifacts, "mobile-v1-live-smoke.recipe.json");
|
|
389
|
-
fs.writeFileSync(liveSmokeRecipe,
|
|
393
|
+
fs.writeFileSync(liveSmokeRecipe, liveSmokeRecipeText(liveSmokeProbeTimeoutMs()));
|
|
390
394
|
const iosSimulatorResolved = resolveJsEnvValue(target, "IOS_SIMULATOR");
|
|
391
395
|
const adbSerialResolved = resolveJsEnvValue(target, "ADB_SERIAL");
|
|
392
396
|
let liveSmokeCode = 1;
|
|
@@ -495,6 +499,7 @@ export {
|
|
|
495
499
|
buildMobileVerifySummary,
|
|
496
500
|
classifyMobileRuntimeOwner,
|
|
497
501
|
isNumericWatcherPort,
|
|
502
|
+
liveSmokeProbeTimeoutMs,
|
|
498
503
|
parseMobileVerifyArgs,
|
|
499
504
|
resolveJsEnvValue,
|
|
500
505
|
resolveWatcherPort,
|
|
@@ -30,6 +30,8 @@ import {
|
|
|
30
30
|
validateMetaMaskActionInputs
|
|
31
31
|
} from "../metamask-action-validation.js";
|
|
32
32
|
import { gitLibraryProvenance } from "../library-provenance.js";
|
|
33
|
+
import { runnableLibraryRecipes } from "../discovery.js";
|
|
34
|
+
import { recipeComposition } from "@farmslot/recipe-cli";
|
|
33
35
|
import {
|
|
34
36
|
captureExecutionProvenance,
|
|
35
37
|
executionProvenanceDrift,
|
|
@@ -710,35 +712,7 @@ function effectiveLibrarySources(librarySources) {
|
|
|
710
712
|
return librarySources && librarySources.length > 0 ? librarySources : [{ name: "metamask", root: path.resolve(runnerDir, "library") }];
|
|
711
713
|
}
|
|
712
714
|
async function listRunnableRecipes(adapter, librarySources) {
|
|
713
|
-
|
|
714
|
-
const sources = effectiveLibrarySources(librarySources);
|
|
715
|
-
const resolution = await harness.loadRecipeLibraries(
|
|
716
|
-
sources,
|
|
717
|
-
{
|
|
718
|
-
adapter
|
|
719
|
-
}
|
|
720
|
-
);
|
|
721
|
-
const protocol = await importRecipeProtocol();
|
|
722
|
-
const { manifest } = await resolveActionManifest(adapter, void 0, sources);
|
|
723
|
-
const externalRecipeIds = new Set(resolution.recipes.keys());
|
|
724
|
-
const validRefs = new Set([...resolution.recipes.values()].filter((recipe) => protocol.validateRecipeWithManifest(recipe.document, manifest, {
|
|
725
|
-
externalRecipeIds
|
|
726
|
-
}).status === "valid").map((recipe) => recipe.ref));
|
|
727
|
-
return [...resolution.recipes.values()].filter((recipe) => {
|
|
728
|
-
if (!validRefs.has(recipe.ref)) return false;
|
|
729
|
-
try {
|
|
730
|
-
const dependencies = harness.resolveRecipeDependencies({
|
|
731
|
-
rootRef: recipe.ref,
|
|
732
|
-
root: recipe.document,
|
|
733
|
-
rootSource: recipe.provenance,
|
|
734
|
-
recipes: resolution.recipes
|
|
735
|
-
});
|
|
736
|
-
return [...dependencies.recipes.keys()].every((ref) => validRefs.has(ref));
|
|
737
|
-
} catch (error) {
|
|
738
|
-
if (error instanceof harness.RecipeResolutionError) return false;
|
|
739
|
-
throw error;
|
|
740
|
-
}
|
|
741
|
-
}).map(
|
|
715
|
+
return (await runnableLibraryRecipes(adapter, effectiveLibrarySources(librarySources))).map(
|
|
742
716
|
(recipe) => recipeSummary(recipe.ref, adapter, recipe.document, recipe)
|
|
743
717
|
).sort((left, right) => left.name.localeCompare(right.name));
|
|
744
718
|
}
|
|
@@ -822,42 +796,6 @@ function recipeParameters(document) {
|
|
|
822
796
|
};
|
|
823
797
|
});
|
|
824
798
|
}
|
|
825
|
-
function recipeComposition(recipe, recipes) {
|
|
826
|
-
const actions = /* @__PURE__ */ new Set();
|
|
827
|
-
const nestedRecipes = /* @__PURE__ */ new Set();
|
|
828
|
-
const unresolvedRecipes = /* @__PURE__ */ new Set();
|
|
829
|
-
const visitedRecipes = /* @__PURE__ */ new Set();
|
|
830
|
-
const visitNode = (value) => {
|
|
831
|
-
if (!isRecord(value) || typeof value.action !== "string") return;
|
|
832
|
-
if (value.action !== "call") {
|
|
833
|
-
actions.add(value.action);
|
|
834
|
-
return;
|
|
835
|
-
}
|
|
836
|
-
if (typeof value.ref !== "string") return;
|
|
837
|
-
nestedRecipes.add(value.ref);
|
|
838
|
-
const dependency = recipes.get(value.ref);
|
|
839
|
-
if (!dependency) {
|
|
840
|
-
unresolvedRecipes.add(value.ref);
|
|
841
|
-
return;
|
|
842
|
-
}
|
|
843
|
-
if (visitedRecipes.has(value.ref)) return;
|
|
844
|
-
visitedRecipes.add(value.ref);
|
|
845
|
-
visitRecipe(dependency.document);
|
|
846
|
-
};
|
|
847
|
-
const visitWorkflow = (value) => {
|
|
848
|
-
if (!isRecord(value)) return;
|
|
849
|
-
if (isRecord(value.nodes)) Object.values(value.nodes).forEach(visitNode);
|
|
850
|
-
};
|
|
851
|
-
const visitRecipe = (document) => {
|
|
852
|
-
visitWorkflow(document.workflow);
|
|
853
|
-
};
|
|
854
|
-
visitRecipe(recipe);
|
|
855
|
-
return {
|
|
856
|
-
actions: [...actions].sort(),
|
|
857
|
-
nestedRecipes: [...nestedRecipes].sort(),
|
|
858
|
-
unresolvedRecipes: [...unresolvedRecipes].sort()
|
|
859
|
-
};
|
|
860
|
-
}
|
|
861
799
|
async function validateRunRecipeStatic(recipeArg, adapter, options, params = {}) {
|
|
862
800
|
let librarySources = await resolveMetaMaskLibrarySources(
|
|
863
801
|
optionStrings(options, "library")
|
|
@@ -1492,7 +1430,6 @@ export {
|
|
|
1492
1430
|
preflightRecipe,
|
|
1493
1431
|
prepareHeal,
|
|
1494
1432
|
prepareRuntimeIfNeeded,
|
|
1495
|
-
recipeComposition,
|
|
1496
1433
|
resolveMetaMaskLibrarySources,
|
|
1497
1434
|
resolveRunRecipeArg,
|
|
1498
1435
|
runOneNode,
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { assessRecipe } from "@farmslot/recipe-cli";
|
|
2
|
+
import {
|
|
3
|
+
loadRecipeLibraries
|
|
4
|
+
} from "@farmslot/recipe-harness";
|
|
5
|
+
import { resolveActionManifest } from "./manifest.js";
|
|
6
|
+
async function runnableLibraryRecipes(adapter, sources) {
|
|
7
|
+
const resolution = await loadRecipeLibraries(sources, { adapter });
|
|
8
|
+
const { manifest } = await resolveActionManifest(adapter, void 0, sources);
|
|
9
|
+
return [...resolution.recipes.values()].filter(
|
|
10
|
+
(recipe) => assessRecipe({ resolution, manifest }, recipe).length === 0
|
|
11
|
+
);
|
|
12
|
+
}
|
|
13
|
+
export {
|
|
14
|
+
runnableLibraryRecipes
|
|
15
|
+
};
|
|
@@ -4,7 +4,8 @@ import { loadRecipeLibraries, resolveRecipeDependencies, rootResolutionRef } fro
|
|
|
4
4
|
import { isSensitiveKey } from "./command-journal.js";
|
|
5
5
|
import { resolveActionManifest } from "./manifest.js";
|
|
6
6
|
import { describeManifestActions } from "./commands/manifest.js";
|
|
7
|
-
import { recipeComposition
|
|
7
|
+
import { recipeComposition } from "@farmslot/recipe-cli";
|
|
8
|
+
import { resolveMetaMaskLibrarySources, validateRunRecipeStatic } from "./commands/run-engine.js";
|
|
8
9
|
import { CliError, isRecord, optionStrings } from "./commands/parse-args.js";
|
|
9
10
|
import { EXIT } from "./commands/shared.js";
|
|
10
11
|
import { MAX_ADVICE_STATE_BYTES } from "./recipe-assistance-rubric.js";
|
package/docs/CONTRIBUTING.md
CHANGED
|
@@ -12,6 +12,7 @@ bin/mm-harness
|
|
|
12
12
|
-> src/ typed CLI and product decisions
|
|
13
13
|
-> @farmslot/agent-runtime checklist state
|
|
14
14
|
-> @farmslot/handoff scrubbed learning capture/share
|
|
15
|
+
-> @farmslot/recipe-cli recipe discovery: libraries, runnable checks, composition
|
|
15
16
|
-> @farmslot/recipe-harness generic execution, UI transports, evidence
|
|
16
17
|
-> @farmslot/protocol schemas
|
|
17
18
|
-> adapters/ focused host/browser/device leaves
|
|
@@ -24,6 +25,7 @@ bin/mm-harness
|
|
|
24
25
|
| `@farmslot/protocol` | recipe and evidence schemas |
|
|
25
26
|
| `@farmslot/agent-runtime` | task-local checklist state and terminal contracts |
|
|
26
27
|
| `@farmslot/handoff` | learning assembly, scrubbing, validation, approval, and publication |
|
|
28
|
+
| `@farmslot/recipe-cli` | recipe discovery: library resolution, runnable checks, composition |
|
|
27
29
|
| `@farmslot/recipe-harness` | generic execution, recovery, traces, artifacts, `ui.*` |
|
|
28
30
|
| `mm-harness` | MetaMask runtime control, diagnostics, durable domain actions |
|
|
29
31
|
| skills/checklists | task workflow and proof expectations |
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# mm-harness CLI contract
|
|
2
|
+
|
|
3
|
+
This page lists every `mm-harness` invocation that something outside this repo depends on, and what that caller reads back. The golden tests in `tests/contract/goldens-*.test.sh` freeze this surface. A refactor that changes any of it fails CI until someone accepts the change with `yarn test:goldens --update` and updates the callers in the same release.
|
|
4
|
+
|
|
5
|
+
Callers scanned (2026-10-02):
|
|
6
|
+
- Farmslot projects `metamask-{extension,mobile,core}-farm` and `va-mmcx-terminal-farm`: `project.json` hooks, `setup/*.sh`, `scripts/*.sh`, worker templates.
|
|
7
|
+
- Farmslot gateway: `services/gateway/src/methods/recipe.ts` and the modules it calls.
|
|
8
|
+
- Recipe libraries: `experimental-metamask-recipe-perps` and `experimental-metamask-recipe-terminal` (`checks/*.mjs`).
|
|
9
|
+
- This repo's README and `docs/`.
|
|
10
|
+
|
|
11
|
+
## Resolution and environment
|
|
12
|
+
|
|
13
|
+
- Farms resolve the binary with `HARNESS_BIN="${METAMASK_HARNESS_BIN:-${MM_HARNESS_BIN:-mm-harness}}"; command -v "$HARNESS_BIN"`. `MM_HARNESS_BIN` hands the whole invocation to another checkout's `bin/mm-harness`.
|
|
14
|
+
- Static review runs `{{support}}/bin/mm-harness` with entry `dist/mm-harness-cli.js`, so the package layout (`bin/`, `dist/`) is part of the contract.
|
|
15
|
+
- Hook environment: `RECIPE_SLOT_ID`, `RECIPE_RUNTIME_DIR` (default `temp/recipe/runtime`), `RECIPE_WALLET_FIXTURE`, `RECIPE_LIBRARY_PATH`, `MM_HARNESS_DOMAIN`, `IOS_SIMULATOR`/`ADB_SERIAL` (mobile slots), `RECIPE_HARNESS_ROOT=temp/recipe/harness` (Extension).
|
|
16
|
+
|
|
17
|
+
## Commands, callers, and what they consume
|
|
18
|
+
|
|
19
|
+
`G:` names the golden that freezes the row (`tests/contract/goldens/<family>/<case>.json`).
|
|
20
|
+
|
|
21
|
+
### Recipe execution (gateway `recipe.ts`, worker templates, library checks)
|
|
22
|
+
|
|
23
|
+
| Command | Caller | Consumes | G: |
|
|
24
|
+
|---|---|---|---|
|
|
25
|
+
| `run <recipe> --adapter <a> --artifacts-dir <d> --target <repo> [--slot <s>] --json [--cdp-port <p>] [--watcher-port <p>] [--launch-existing-dist] [--record-video=full-run]` | `recipe_run` hook, all four packs | exit code, then the artifact package (below) | `core/run-hook`, `extension/run-hook`, `extension/run-hook-plan` (all flags incl. `--record-video=full-run`), `mobile/run-hook`, `mobile/run-hook-plan`, `core/run-invalid`, `*/run-unknown-flag` |
|
|
26
|
+
| `run <recipe> --artifacts-dir <d> --json` from the checkout | Core templates | exit code; `trace.json` on failure | `core/run-autodetect` |
|
|
27
|
+
| `run <recipe> [k=v…] --plan [--adapter <a>] [--library ns=dir] [--json]`, with `RECIPE_LIBRARY_PATH=ns=dir` or cleared | templates; perps `checks/*.mjs` | exit code; `JSON.parse(stdout).status === 'pass'`; on rejection stdout+stderr matching `RECIPE_PARAMS_INVALID\|parameter\|required\|enum`. No harness code emits `RECIPE_PARAMS_INVALID`: plan rejections exit 5 with findings `recipe.missing_param` / `recipe.invalid_param_value_enum` / unknown-param, and a real run exits 5 with `RECIPE_VALIDATION_FAILED`. The check passes on the `parameter\|required\|enum` alternatives | `core/run-plan`, `mobile/run-plan`, `library/*` |
|
|
28
|
+
| `run --list [--adapter <a>] [--json]`, `run <r> --describe --json` | templates, docs | prose | `discovery/run-list-*`, `discovery/run-describe` |
|
|
29
|
+
| `call <action> [k=v…] --adapter <a> [--target] [--watcher-port] [--json]` | mobile `unlock` hook, templates | exit code (unlock output is ignored) | `core/call-command`, `mobile/call-unlock` |
|
|
30
|
+
| `actions --raw --adapter <a> --json` | `recipe_action_manifest` hook, all packs | stdout is a Recipe v1 action manifest (`validateRecipeActionManifestDocument`) | `discovery/actions-raw-*` |
|
|
31
|
+
| `actions --adapter <a> --json` | templates (Core runs `jq -r '.actions[].name'`) | `.actions[].name` | `discovery/actions-*` |
|
|
32
|
+
| `actions --action <name> --json`, `actions <term> --json` | templates, docs | prose | `discovery/actions-action`, `discovery/actions-search` |
|
|
33
|
+
|
|
34
|
+
### Readiness (preflight, health_check, recipe_doctor)
|
|
35
|
+
|
|
36
|
+
| Command | Caller | Consumes | G: |
|
|
37
|
+
|---|---|---|---|
|
|
38
|
+
| `doctor --adapter <a> --target <repo> [--cdp-port\|--watcher-port <p>] --json` | `recipe_doctor` hook | `runner_protocol_version === 1`, `status === 'pass'`, every `checks[].status === 'pass'` | `*/doctor-json` |
|
|
39
|
+
| `doctor --adapter <a> --target <repo> [--cdp-port\|--watcher-port <p>] [--runtime-dir <rd>] [--device <d>] --print-ready` | `health_check` (gateway `slot/check.ts`, `2>/dev/null`) | exit 0 and trimmed stdout equal to `OK` (Extension, Mobile) or `ready` (Core) | `*/doctor-print-ready`, `mobile/doctor-print-ready-device` |
|
|
40
|
+
| `doctor … --expect-live [--json]` | Extension `ensure-runtime-ready.sh`, `preflight.sh`; templates | exit code; JSON shown to the agent | `extension/doctor-expect-live-json`, `mobile/doctor-expect-live-json` |
|
|
41
|
+
| `prepare --target <repo> --platform <extension\|mobile\|core> --artifacts-dir <repo>/<rd> --json` | preflight (all packs, stdout to `/dev/null`) | exit code; gateway reads `<rd>/sandbox.json`: `schemaVersion === 1`, `steps[]` (`id`, `status`, `reason`), `ready`, `harness.name`, `harness.version` | `*/prepare` |
|
|
42
|
+
| `install --adapter <a> --target <repo>` | `recipe_harness_install`, Core preflight | exit code | `*/install` |
|
|
43
|
+
| `verify --adapter <a> --target <repo> [--json]` | `recipe_harness_verify`, Core preflight | exit code (JSON not parsed) | `core/verify`, `mobile/verify` |
|
|
44
|
+
| `cleanup --adapter <a> --target <repo>` | `recipe_harness_cleanup`, Extension recycle | exit code | `*/cleanup` |
|
|
45
|
+
|
|
46
|
+
### Runtime lifecycle
|
|
47
|
+
|
|
48
|
+
| Command | Caller | Consumes | G: |
|
|
49
|
+
|---|---|---|---|
|
|
50
|
+
| `launch --verify --adapter extension --target <repo> --watcher-port <p> --cdp-port <p> --surface <fullscreen\|sidepanel> [--build] [--url <u>]` | Extension `preflight.sh` | exit code | `extension/launch-verify` |
|
|
51
|
+
| `launch --adapter extension --target --cdp-port (--fullscreen \| --sidepanel --url <u>) [--verify] [--build] [--remote-flag K=V] [--json]` | Extension browser resource boot hook and slot actions | exit code | `extension/launch-boot-fullscreen`, `extension/launch-sidepanel-url`, `extension/launch-verify-remote-flag`, `extension/launch-build-verify` (cold, through the stubbed browser/build leaf; `live-calls.log` frozen) |
|
|
52
|
+
| `launch <ios\|android> --adapter mobile --target <repo> --verify --json --watcher-port <p> [--build] [--device <d>]` | Mobile `preflight.sh` | exit code; stdout saved to `<rd>/mobile-launch/summary.json` (not parsed) | `mobile-launch/launch-ios-verify`, `mobile-launch/launch-ios-device-verify`, `mobile-launch/launch-android-device-verify` (`leaf-calls.log` frozen) |
|
|
53
|
+
| `stop --adapter <extension\|mobile> --port <p> --target <repo>` | teardown, recycle, shutdown hooks | exit code | `extension/stop`, `mobile/stop` |
|
|
54
|
+
| `fixtures set --adapter extension --target <repo> --fixture <rd>/wallet-fixture.json` | Extension preflight | exit code | `extension/fixtures-set-no-browser` |
|
|
55
|
+
| `fixtures set --adapter mobile --target <repo> --json` | Mobile preflight | exit code; stdout saved to `<rd>/mobile-launch/wallet-setup.json` | `mobile/fixtures-set` |
|
|
56
|
+
| `fixtures generate --target <repo> --fixture <rd>/wallet-fixture.json --out <rd>/fixture-state.json` | Extension `launch-browser.sh` | reads `fixture-state.json` | `extension/fixtures-generate` |
|
|
57
|
+
| `<repo>/temp/recipe/harness/extension/runner/bin/mm-harness resolve-extension --adapter extension --target <repo>` (overlay path, not the public bin) | Extension `setup/launch-browser.sh:53,449` | stdout must match `/^[a-p]{32}$/`; failure falls back to scanning | `extension/resolve-extension-overlay` |
|
|
58
|
+
| `provision runway <ios\|android> --adapter mobile --target --slot --watcher-port --runtime-dir --run <id> [--device] [--force]` | Mobile `runway-preflight.sh` | exit code (non-zero falls back) | `mobile/provision-runway-android` (ios needs `gh` + network: not covered) |
|
|
59
|
+
|
|
60
|
+
### Task tooling (gateway and worker templates)
|
|
61
|
+
|
|
62
|
+
| Command | Caller | Consumes | G: |
|
|
63
|
+
|---|---|---|---|
|
|
64
|
+
| `checklist mark <taskDir> {start\|<n>\|complete [--mark-last]\|blocked --reason <s>\|no-change --reason <s>\|--help}` | `{{TASK_DIR}}/mark` shim (`mark_cmd`) | exit code; `SIGNAL.json`, `CHECKLIST.md` | `task/mark-*` |
|
|
65
|
+
| `pr-body render <taskDir> --json` (cwd = task dir) | gateway `pr-body-render.ts` (`pr_body_cmd`) | on failure `JSON.parse(stdout).error`, else stderr; then `artifacts/pr-body.md` | `task/pr-body-render-*` |
|
|
66
|
+
| `review checklist --out <file> [--domain <d>] [--since <sha>]` | domain fixtures, review templates | the written file | `task/review-checklist` |
|
|
67
|
+
| `check diff …`, `recipe-quality build …` | templates | exit code | not covered: run repo lint/test tooling |
|
|
68
|
+
|
|
69
|
+
### Discovery (moves to Farmslot in phase 2a)
|
|
70
|
+
|
|
71
|
+
`--help`, `--version`, `help [review] --json`, `actions` (above), `run --list`, `call --list`, `completion-candidates actions`, `execution-template {new,list,materialize} --json`, `completions {zsh,bash}`, and the unknown-command error. G: `discovery/*`.
|
|
72
|
+
|
|
73
|
+
## Files callers read
|
|
74
|
+
|
|
75
|
+
- **Recipe artifacts dir** (`--artifacts-dir`): `summary.json` (`.status`), `trace.json`, `artifact-manifest.json` (`.runStatus` equals `summary.status`; Recipe v1), optional `recipe.json`, `recipe-resolution.json` with `resolved-recipes/<sha256>.recipe.json`, and videos when `--record-video=full-run`. Frozen with full normalised values (`valueTrees`) for passing runs in `core/run-hook`, `core/run-autodetect`, `core/call-command` and `library/run-library-recipe`. `extension/run-hook` and `mobile/run-hook` run cold with no runtime, so they freeze the failure envelope and exit code, and their artifact tree is `null` (nothing is written). A passing Extension or Mobile artifact package needs a live runtime and is not covered.
|
|
76
|
+
- **`<rd>/sandbox.json`** from `prepare`: frozen in full by `*/prepare`.
|
|
77
|
+
- **Pid files and logs** under `<rd>`: `browser.pid`, `recipe-harness-webpack.pid`, `recipe-harness-webpack.log` (Extension); `metro.log` (Mobile); the gateway kills `launcher.pid`, `browser.pid`, `chromium.pid`, `webpack.pid` and deletes `extension.id` and `preflight.pgid`. Live-runtime only, so not covered by the cold-path goldens.
|
|
78
|
+
- **`<rd>/fixture-state.json`** from `fixtures generate`: `extension/fixtures-generate`.
|
|
79
|
+
- **Task artifacts**: `SIGNAL.json`, `artifacts/pr-body.md`, review checklist file.
|
|
80
|
+
|
|
81
|
+
## Callers that are out of contract today
|
|
82
|
+
|
|
83
|
+
These are real farm bugs. The goldens freeze the current rejections, so fixing either side shows up as a reviewed golden change. The farm fixes go in a separate Farmslot PR.
|
|
84
|
+
|
|
85
|
+
| Caller | Today | Correct call | G: |
|
|
86
|
+
|---|---|---|---|
|
|
87
|
+
| Mobile `scripts/cleanup-recipe-harness.sh:30` (`recipe_harness_cleanup`) | `cleanup … --allow-managed-changes` exits 2 (`CLI_UNKNOWN_OPTION`); its `No mobile harness backup found` fallback never matches, so the hook always fails | `cleanup --adapter mobile --target <repo>` | `mobile/cleanup-allow-managed-changes`, `mobile/cleanup` |
|
|
88
|
+
| Mobile `project.json` `unlock` hook | `call app.unlock` exits 2 (unknown action); the gateway ignores the result, so unlock is a silent no-op | `call metamask.wallet.ensure_unlocked --adapter mobile --target <repo> --watcher-port <p>` | `mobile/call-unlock` |
|
|
89
|
+
| Mobile `scripts/runway-preflight.sh:66` | `provision runway android` exits 2 (only `ios`), so the Android runway profile always falls back | add Android runway to the harness, or have the farm reject Android before calling | `mobile/provision-runway-android` |
|
|
90
|
+
| Mobile templates `dev.md`, `dev-interactive.md`, `fix-bug.md`, `review-pr.md` | `mark complete --status blocked --outcome partial --reason …` exits 2 | `mark blocked --reason "<why>"` | `task/mark-invalid-status-outcome`, `task/mark-blocked` |
|
|
91
|
+
| Docs | `launch --build-lavamoat` (only `runtime-launch` has it); `run --target <recipe-file>` | `runtime-launch --build-lavamoat`; `run <recipe-file> --target <repo>` | — |
|
|
92
|
+
|
|
93
|
+
## Terminal adapter: TODO
|
|
94
|
+
|
|
95
|
+
#298 merged the `terminal` adapter (`fd1c77e`). Discovery already reflects it (`discovery/actions-matrix`, the adapter hint in `help`). **Follow-up, not in this PR:** freeze its farm hooks once a stub browser replaces the LaunchServices launch of a visible Chrome for Testing (the goldens must never start a real browser). Add `tests/contract/goldens-terminal.test.sh` covering the `va-mmcx-terminal-farm` hooks:
|
|
96
|
+
|
|
97
|
+
- [ ] `run <recipe> --adapter terminal --artifacts-dir <d> --target <repo> --slot <s> --cdp-port <p> --watcher-port <p> --json`
|
|
98
|
+
- [ ] `actions --raw --adapter terminal --json`
|
|
99
|
+
- [ ] `doctor --adapter terminal --target <repo> --cdp-port <p> --watcher-port <p> --json`
|
|
100
|
+
- [ ] `install --adapter terminal --target <repo>`
|
|
101
|
+
- [ ] `verify --adapter terminal --target <repo> -- --cdp-port <p> --watcher-port <p> --account <a>`
|
|
102
|
+
- [ ] `cleanup --adapter terminal --target <repo> -- --cdp-port <p>`
|
|
103
|
+
- [ ] `launch --adapter terminal --target <repo> --cdp-port <p> --watcher-port <p> --signer extension --account <a>` (use the stub browser from #298's `tests/fixtures/terminal-stub-browser.mjs`)
|
|
104
|
+
- [ ] `stop --adapter terminal --target <repo>` with `RECIPE_CDP_PORT`
|
|
105
|
+
- [ ] `run terminal.perps.<name> k=v… --adapter terminal --target <t> --cdp-port <p> --watcher-port <p> --plan` (recipe-terminal `checks/plan-recipes.mjs`)
|
|
106
|
+
- [ ] runtime layout of `temp/recipe/runtime/terminal/` (`browser.pid` is watched by the farm)
|
|
107
|
+
- [ ] discovery: add `terminal` to the per-adapter loop in `goldens-discovery.test.sh` (`actions --raw`, `actions`, `run --list`, `call --list`)
|
|
108
|
+
- [ ] recipe-terminal `console-guard.mjs` imports the harness's `library/actions/shared/console-findings.mjs` and `tests/fixtures/console-allowlist-vectors.json`: freeze both paths and the exported surface
|
|
109
|
+
|
|
110
|
+
## Running the goldens
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
yarn test:goldens # compare all families in parallel (CI runs them in the contract shards)
|
|
114
|
+
yarn test:goldens mobile # families whose name contains "mobile"
|
|
115
|
+
yarn test:goldens --update # rewrite goldens, print a per-field diff summary, drop orphans
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Every family checks its own golden directory: a golden no case writes fails the family, in CI too (`--update` deletes it). Every family also fails on a golden directory that has no `goldens-<dir>.test.sh`, so deleting a whole family script is caught. Families run in parallel locally and sequentially inside the contract CI shards.
|
|
119
|
+
|
|
120
|
+
Each case runs under a safety cap, `GOLDENS_CASE_TIMEOUT` (default 600 s). A case that hits it reports `TIMEOUT <case>` and fails without being compared or written, so `--update` can never accept a truncated capture.
|
|
121
|
+
|
|
122
|
+
Each family is a normal contract test (`tests/contract/goldens-<family>.test.sh`). `gd_init` (`tests/contract/goldens/lib.sh`) builds a sandbox on `ct_init`, and every case runs under `env -i`:
|
|
123
|
+
- `gd_init` first unsets every inherited `MM_HARNESS_*`, `RECIPE_*`, `METAMASK_*` and `FARMSLOT_*` variable, plus device and port pins (`IOS_SIMULATOR`, `ADB_SERIAL`, `WATCHER_PORT`, `METRO_PORT`, `CDP_PORT`, `IDB_PATH`, `ANDROID_HOME`, …). A developer's `MM_HARNESS_BIN` or `MM_HARNESS_IDB_PATH` export therefore cannot redirect a case;
|
|
124
|
+
- the environment is an allowlist: sandbox `HOME`, caches, config, `FARMSLOT_HOME`, `TZ=UTC`, no update probe, and only the variables the family sets itself;
|
|
125
|
+
- every interpreter and tool the goldens launch is resolved to its real binary before `HOME` moves into the sandbox: `node` (`process.execPath`), `git` (`git --exec-path`), `bash` (`$BASH`) and `timeout`. The goldens need no Python. A tool that resolves to a script, the shape of an asdf/mise/volta/pyenv shim, fails the family up front with a message naming it. Each is checked to run with a sandboxed `HOME`;
|
|
126
|
+
- `PATH` is pinned to the case stubs, the sandbox stubs, those real binaries, and `/usr/bin:/bin:/usr/sbin:/sbin`. Host tools (`idb`, `capture-helper`, a developer's Chrome) cannot change a golden;
|
|
127
|
+
- device-tool discovery is pinned: `MM_HARNESS_IDB_PATH` and `MM_HARNESS_ADB_PATH` point at sandbox stubs, which discovery checks before any absolute Homebrew or pipx path;
|
|
128
|
+
- localhost probes cannot reach a real server. `gd_reserve_ports` runs a Node port holder (`tests/contract/goldens/port-holder.mjs`) that holds ephemeral ports on `127.0.0.1` and `::1` and resets every connection it accepts, so Metro `/status` and CDP probes never get a response and nothing real can take the port mid-run. Each family exports the farm's slot port variables (`RECIPE_CDP_PORT`, `WATCHER_PORT`, `METRO_PORT`) set to those ports, so commands without an explicit port use them too. Goldens show them as `<PORT:NAME>`, rewritten only in port positions (`*port` keys, `:<port>`, `port <n>`/`PORT=<n>`, or a string that is exactly the port), so an unrelated equal number such as a 60000 ms timeout is never rewritten;
|
|
129
|
+
- `tmux`, `xcrun` and `adb` are stubbed (`ct_init`, plus one booted simulator and one Android device for Mobile);
|
|
130
|
+
- forbidden stubs record any call to `open`, `osascript`, `lsappinfo`, `screencapture`, `yarn`, `npx`, `npm`, `corepack`, `gh` and Chrome/Chromium, and the family fails if one is called. A browser stub answers only `--version`, and `npm` only `root -g` (install-source detection), with a sandbox path;
|
|
131
|
+
- inert stubs on the case PATH only: `lsof`, `pgrep` and `pkill` find nothing, `ps` passes per-pid queries (`-p`, the CLI identifying its own processes) and lists an empty host for table listings, and `watchman` and `caffeinate` are no-ops. `stop` cannot see or signal a real process or touch a host daemon;
|
|
132
|
+
- `RECIPE_HARNESS_BROWSER` points at the sandbox Chrome stub, so browser resolution never probe-launches Chrome for Testing or reads host Chrome policy;
|
|
133
|
+
- `MM_HARNESS_MOBILE_VERIFY_PROBE_TIMEOUT_MS=2000` shortens Mobile verify's live-bridge wait (default 60 s), because no fixture provides a debug target.
|
|
134
|
+
|
|
135
|
+
Mobile Metro, device, bridge and wallet leaves, the Extension build/browser leaf, and the fixture-state leaf are stubbed through `MM_HARNESS_SCRIPT_BIN_*`. `leaf-calls.log` and `live-calls.log` are frozen per launch case.
|
|
136
|
+
|
|
137
|
+
A golden records:
|
|
138
|
+
- `bin` (only when a case runs another executable) and argv;
|
|
139
|
+
- the exit code;
|
|
140
|
+
- stdout (parsed JSON, or lines), and stderr when the exit code is non-zero;
|
|
141
|
+
- `--file`/`--log` files in full;
|
|
142
|
+
- `--tree` runtime trees: relative paths plus each JSON file's key shape, where arrays record the union of every element;
|
|
143
|
+
- `--value-tree` artifact packages: relative paths plus each JSON file's normalised values;
|
|
144
|
+
- `--shallow-tree` for the internal overlay: paths two levels deep, no shapes.
|
|
145
|
+
|
|
146
|
+
The goldens need the harness to be a git checkout: `execution-provenance.json` `runner.head` and `libraries[0].head` are `null` outside one, which changes the artifact goldens. CI and the farm checkouts satisfy this.
|
|
147
|
+
|
|
148
|
+
### Normaliser
|
|
149
|
+
|
|
150
|
+
`tests/contract/goldens/normalize.mjs` is the only normaliser. It rewrites only volatile values:
|
|
151
|
+
|
|
152
|
+
1. Absolute paths: the sandbox becomes `<SANDBOX>`, the repo `<REPO>`, the real `$HOME` `<HOME>`, the node binary and its prefix `<NODE>`/`<NODE_PREFIX>`, and `os.tmpdir()` `<TMP>`. Both `/var` and `/private/var` spellings are covered. Ports from `gd_reserve_ports` become `<PORT:NAME>` in port positions only: numbers under `*port` keys, and in text after `:`, `port `, `port=` or `PORT=`, or a string that is exactly the port (`^<port>$`; `60000 ms` is left alone).
|
|
153
|
+
2. UUIDs become `<UUID>`. Git object ids (40 hex, or a short hex value under a `*head`/`*gitRef`/`*commit` key) become `<GIT_REF>`. ISO-8601 timestamps become `<TIMESTAMP>`, and compact stamps in names `<STAMP>`. Path-derived slot ids `local-<adapter>-<8 hex>` become `local-<adapter>-<HASH>`.
|
|
154
|
+
3. The package version becomes `<HARNESS_VERSION>`. Any other `x.y.z` under a key ending in `version` becomes `<VERSION>`.
|
|
155
|
+
4. Numbers under volatile keys: `*pid` becomes `<PID>`, `*port` `<PORT>`, `*At`/`*Time`/`*timestamp`/`mtime*` `<TIMESTAMP>`, and `*Ms`/`duration*`/`elapsed*` `<DURATION>`. Values inside `schema` and `examples` subtrees (static manifest content) are never rewritten.
|
|
156
|
+
5. `invocationDigest` becomes `<DIGEST>`: it hashes `recipe-invocation.json`, which carries sandbox paths and timestamps and is frozen itself in normalised form. `runner.sourceFingerprint` becomes `<FINGERPRINT>`: it hashes the harness source tree, so any harness edit changes it. A `status` holding `git status --porcelain` output (provenance `runner.status` and `libraries[].status`, empty when clean) becomes `<GIT_STATUS>`. Content-only digests (recipe and dependency digests) stay frozen.
|
|
157
|
+
6. Durations in text (`0.3s`, `123ms`) become `<DURATION>`.
|
|
158
|
+
|
|
159
|
+
In tree listings, files whose names differ only by a volatile token collapse into one row: a count and the union of their key shapes, or the sorted list of their values in a value tree.
|
|
160
|
+
|
|
161
|
+
The `--update` summary diffs objects by key. Arrays whose entries all carry a unique `id`, `name` or `path` (manifest artifacts) diff by that key, so an inserted entry reads as one addition. Lists of plain values report `+[added] -[removed]`; other arrays diff element by element.
|
|
162
|
+
|
|
163
|
+
The goldens run from the source checkout, so `harness.source` reads `source-checkout/source-checkout` and `harness.executable` is `<REPO>/bin/mm-harness`. On the farm both name the installed package instead.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@deeeed/metamask-harness",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.73.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"bin": {
|
|
6
6
|
"mm-harness": "bin/mm-harness"
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
"check": "node scripts/check.mjs",
|
|
13
13
|
"test:unit": "vitest run --config scripts/vitest.config.mjs",
|
|
14
14
|
"test:coverage": "vitest run --coverage --config scripts/vitest.config.mjs",
|
|
15
|
+
"test:goldens": "bash scripts/test-goldens.sh",
|
|
15
16
|
"qa:human": "node scripts/validate-human-outcomes.mjs",
|
|
16
17
|
"site:contrast": "node scripts/site-contrast.mjs",
|
|
17
18
|
"self-test": "bin/mm-harness self-test",
|
|
@@ -20,11 +21,12 @@
|
|
|
20
21
|
"check:syntax": "find . -name '*.mjs' -print0 | xargs -0 -n1 node --check"
|
|
21
22
|
},
|
|
22
23
|
"dependencies": {
|
|
23
|
-
"@farmslot/agent-runtime": "0.
|
|
24
|
-
"@farmslot/expo-recipe": "0.
|
|
24
|
+
"@farmslot/agent-runtime": "0.16.0",
|
|
25
|
+
"@farmslot/expo-recipe": "0.13.1",
|
|
25
26
|
"@farmslot/handoff": "^0.3.1",
|
|
26
|
-
"@farmslot/protocol": "0.
|
|
27
|
-
"@farmslot/recipe-
|
|
27
|
+
"@farmslot/protocol": "0.34.0",
|
|
28
|
+
"@farmslot/recipe-cli": "0.1.1",
|
|
29
|
+
"@farmslot/recipe-harness": "0.22.1",
|
|
28
30
|
"agent-device": "0.19.3",
|
|
29
31
|
"commander": "^12.0.0",
|
|
30
32
|
"es-module-lexer": "2.3.1",
|