create-mercato-app 0.7.0 → 0.7.1-develop.7103.1.41ff100d93
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/agentic/shared/AGENTS.md.template +3 -3
- package/agentic/shared/ai/harness/README.md +2 -2
- package/agentic/shared/ai/harness/RELEASE.md +8 -6
- package/agentic/shared/ai/harness/cases.json +83 -0
- package/agentic/shared/ai/harness/cases.schema.json +4 -4
- package/agentic/shared/ai/harness/validators.json +1 -1
- package/agentic/shared/scripts/run-agent-harness-release.mjs +37 -11
- package/dist/agentic/guides/module-facts.json +5509 -599
- package/dist/agentic/guides/module-facts.v2.json +5595 -599
- package/dist/agentic/guides/modules/ai_assistant/index.md +1 -1
- package/dist/agentic/guides/modules/api_docs/index.md +1 -1
- package/dist/agentic/guides/modules/api_keys/di-registrations-rich.md +11 -0
- package/dist/agentic/guides/modules/api_keys/di-service-tokens.md +11 -0
- package/dist/agentic/guides/modules/api_keys/exact-override-targets.md +1 -0
- package/dist/agentic/guides/modules/api_keys/index.md +4 -2
- package/dist/agentic/guides/modules/attachments/di-registrations-rich.md +6 -5
- package/dist/agentic/guides/modules/attachments/di-service-tokens.md +5 -4
- package/dist/agentic/guides/modules/attachments/exact-override-targets.md +6 -5
- package/dist/agentic/guides/modules/attachments/index.md +2 -2
- package/dist/agentic/guides/modules/audit_logs/index.md +1 -1
- package/dist/agentic/guides/modules/auth/di-registrations-rich.md +3 -2
- package/dist/agentic/guides/modules/auth/di-service-tokens.md +3 -2
- package/dist/agentic/guides/modules/auth/exact-override-targets.md +3 -2
- package/dist/agentic/guides/modules/auth/index.md +1 -1
- package/dist/agentic/guides/modules/business_rules/index.md +1 -1
- package/dist/agentic/guides/modules/catalog/incoming-installed-contributions.md +3 -0
- package/dist/agentic/guides/modules/catalog/index.md +2 -2
- package/dist/agentic/guides/modules/channel_apns/index.md +1 -1
- package/dist/agentic/guides/modules/channel_discord/acl-features.md +13 -0
- package/dist/agentic/guides/modules/channel_discord/ai-agents.md +11 -0
- package/dist/agentic/guides/modules/channel_discord/backend-pages.md +11 -0
- package/dist/agentic/guides/modules/channel_discord/cli-commands.md +13 -0
- package/dist/agentic/guides/modules/channel_discord/contribution-resolutions.md +14 -0
- package/dist/agentic/guides/modules/channel_discord/di-registrations-rich.md +12 -0
- package/dist/agentic/guides/modules/channel_discord/exact-override-targets.md +26 -0
- package/dist/agentic/guides/modules/channel_discord/index.md +22 -0
- package/dist/agentic/guides/modules/channel_discord/owned-contract-module-metadata.md +11 -0
- package/dist/agentic/guides/modules/channel_discord/setup.md +11 -0
- package/dist/agentic/guides/modules/channel_discord/umes-contributions.md +14 -0
- package/dist/agentic/guides/modules/channel_discord/workers.md +12 -0
- package/dist/agentic/guides/modules/channel_expo/index.md +1 -1
- package/dist/agentic/guides/modules/channel_fcm/index.md +1 -1
- package/dist/agentic/guides/modules/channel_gmail/index.md +1 -1
- package/dist/agentic/guides/modules/channel_imap/index.md +1 -1
- package/dist/agentic/guides/modules/checkout/index.md +1 -1
- package/dist/agentic/guides/modules/communication_channels/di-registrations-rich.md +9 -8
- package/dist/agentic/guides/modules/communication_channels/exact-override-targets.md +9 -8
- package/dist/agentic/guides/modules/communication_channels/incoming-installed-contributions.md +2 -0
- package/dist/agentic/guides/modules/communication_channels/index.md +3 -3
- package/dist/agentic/guides/modules/configs/index.md +1 -1
- package/dist/agentic/guides/modules/content/index.md +1 -1
- package/dist/agentic/guides/modules/currencies/index.md +1 -1
- package/dist/agentic/guides/modules/customer_accounts/index.md +1 -1
- package/dist/agentic/guides/modules/customers/active-extension-bindings.md +6 -3
- package/dist/agentic/guides/modules/customers/contribution-resolutions.md +1 -0
- package/dist/agentic/guides/modules/customers/events.md +2 -1
- package/dist/agentic/guides/modules/customers/incoming-installed-contributions.md +7 -0
- package/dist/agentic/guides/modules/customers/index.md +4 -4
- package/dist/agentic/guides/modules/customers/umes-contributions.md +1 -0
- package/dist/agentic/guides/modules/customers/umes-hosts.md +1 -0
- package/dist/agentic/guides/modules/dashboards/index.md +1 -1
- package/dist/agentic/guides/modules/data_sync/exact-override-targets.md +2 -2
- package/dist/agentic/guides/modules/data_sync/index.md +1 -1
- package/dist/agentic/guides/modules/data_sync/workers.md +2 -2
- package/dist/agentic/guides/modules/design_system/index.md +1 -1
- package/dist/agentic/guides/modules/devices/index.md +1 -1
- package/dist/agentic/guides/modules/dictionaries/index.md +1 -1
- package/dist/agentic/guides/modules/directory/di-registrations-rich.md +12 -0
- package/dist/agentic/guides/modules/directory/di-service-tokens.md +12 -0
- package/dist/agentic/guides/modules/directory/exact-override-targets.md +2 -0
- package/dist/agentic/guides/modules/directory/index.md +5 -3
- package/dist/agentic/guides/modules/documents/acl-features.md +17 -0
- package/dist/agentic/guides/modules/documents/backend-pages.md +13 -0
- package/dist/agentic/guides/modules/documents/contribution-resolutions.md +27 -0
- package/dist/agentic/guides/modules/documents/di-registrations-rich.md +21 -0
- package/dist/agentic/guides/modules/documents/domain-commands.md +37 -0
- package/dist/agentic/guides/modules/documents/encryption.md +11 -0
- package/dist/agentic/guides/modules/documents/entities.md +21 -0
- package/dist/agentic/guides/modules/documents/events.md +25 -0
- package/dist/agentic/guides/modules/documents/exact-override-targets.md +39 -0
- package/dist/agentic/guides/modules/documents/index.md +25 -0
- package/dist/agentic/guides/modules/documents/notifications.md +13 -0
- package/dist/agentic/guides/modules/documents/owned-contract-module-metadata.md +11 -0
- package/dist/agentic/guides/modules/documents/setup.md +11 -0
- package/dist/agentic/guides/modules/documents/umes-contributions.md +27 -0
- package/dist/agentic/guides/modules/documents/umes-hosts.md +36 -0
- package/dist/agentic/guides/modules/entities/index.md +1 -1
- package/dist/agentic/guides/modules/eudr/index.md +1 -1
- package/dist/agentic/guides/modules/events/index.md +1 -1
- package/dist/agentic/guides/modules/feature_toggles/index.md +1 -1
- package/dist/agentic/guides/modules/gateway_stripe/exact-override-targets.md +1 -1
- package/dist/agentic/guides/modules/gateway_stripe/index.md +1 -1
- package/dist/agentic/guides/modules/gateway_stripe/workers.md +1 -1
- package/dist/agentic/guides/modules/generators/index.md +1 -1
- package/dist/agentic/guides/modules/inbox_ops/index.md +1 -1
- package/dist/agentic/guides/modules/integrations/index.md +1 -1
- package/dist/agentic/guides/modules/messages/index.md +1 -1
- package/dist/agentic/guides/modules/notifications/index.md +1 -1
- package/dist/agentic/guides/modules/onboarding/index.md +1 -1
- package/dist/agentic/guides/modules/payment_gateways/index.md +1 -1
- package/dist/agentic/guides/modules/perspectives/index.md +1 -1
- package/dist/agentic/guides/modules/planner/index.md +1 -1
- package/dist/agentic/guides/modules/portal/index.md +1 -1
- package/dist/agentic/guides/modules/progress/index.md +1 -1
- package/dist/agentic/guides/modules/push_notifications/index.md +1 -1
- package/dist/agentic/guides/modules/query_index/index.md +1 -1
- package/dist/agentic/guides/modules/record_locks/index.md +1 -1
- package/dist/agentic/guides/modules/resources/index.md +1 -1
- package/dist/agentic/guides/modules/sales/active-extension-bindings.md +1 -0
- package/dist/agentic/guides/modules/sales/contribution-resolutions.md +1 -0
- package/dist/agentic/guides/modules/sales/exact-override-targets.md +5 -4
- package/dist/agentic/guides/modules/sales/host-extension-points.md +1 -1
- package/dist/agentic/guides/modules/sales/incoming-installed-contributions.md +2 -0
- package/dist/agentic/guides/modules/sales/index.md +3 -3
- package/dist/agentic/guides/modules/sales/setup.md +1 -1
- package/dist/agentic/guides/modules/sales/umes-contributions.md +1 -0
- package/dist/agentic/guides/modules/sales/umes-diagnostics.md +1 -0
- package/dist/agentic/guides/modules/sales/umes-hosts.md +9 -0
- package/dist/agentic/guides/modules/scheduler/cli-commands.md +1 -0
- package/dist/agentic/guides/modules/scheduler/exact-override-targets.md +3 -1
- package/dist/agentic/guides/modules/scheduler/index.md +2 -2
- package/dist/agentic/guides/modules/scheduler/workers.md +2 -1
- package/dist/agentic/guides/modules/search/index.md +1 -1
- package/dist/agentic/guides/modules/security/index.md +1 -1
- package/dist/agentic/guides/modules/security/owned-contract-module-metadata.md +1 -1
- package/dist/agentic/guides/modules/shipping_carriers/index.md +1 -1
- package/dist/agentic/guides/modules/sso/index.md +1 -1
- package/dist/agentic/guides/modules/staff/index.md +1 -1
- package/dist/agentic/guides/modules/storage_s3/index.md +1 -1
- package/dist/agentic/guides/modules/sync_akeneo/index.md +1 -1
- package/dist/agentic/guides/modules/sync_excel/index.md +1 -1
- package/dist/agentic/guides/modules/system_status_overlays/index.md +1 -1
- package/dist/agentic/guides/modules/translations/index.md +1 -1
- package/dist/agentic/guides/modules/warranty_claims/index.md +1 -1
- package/dist/agentic/guides/modules/webhooks/index.md +1 -1
- package/dist/agentic/guides/modules/wms/index.md +1 -1
- package/dist/agentic/guides/modules/workflows/index.md +1 -1
- package/dist/agentic/guides/reference-module-facts.json +2 -2
- package/dist/agentic/guides/reference-modules/example/index.md +1 -1
- package/dist/agentic/guides/upstream/manifest.json +1 -1
- package/dist/agentic/shared/AGENTS.md.template +3 -3
- package/dist/agentic/shared/ai/harness/README.md +2 -2
- package/dist/agentic/shared/ai/harness/RELEASE.md +8 -6
- package/dist/agentic/shared/ai/harness/cases.json +83 -0
- package/dist/agentic/shared/ai/harness/cases.schema.json +4 -4
- package/dist/agentic/shared/ai/harness/validators.json +1 -1
- package/dist/agentic/shared/scripts/run-agent-harness-release.mjs +37 -11
- package/package.json +5 -4
- package/template/.env.example +64 -0
- package/template/.yarnrc.yml.template +9 -5
- package/template/AGENTS.md +3 -3
- package/template/Dockerfile +15 -4
- package/template/docker-compose.fullapp.dev.yml +56 -0
- package/template/docker-compose.fullapp.yml +78 -0
- package/template/next.config.ts +3 -0
- package/template/package.json.template +3 -0
- package/template/src/i18n/de.json +4 -0
- package/template/src/i18n/en.json +4 -0
- package/template/src/i18n/es.json +4 -0
- package/template/src/i18n/ko.json +4 -0
- package/template/src/i18n/pl.json +4 -0
- package/template/src/modules/example/__integration__/TC-EXAMPLE-017-bound-extension-ui.spec.ts +31 -19
- package/template/src/modules/example/components/TodosTable.tsx +2 -0
- package/template/src/modules.ts +12 -0
package/README.md
CHANGED
|
@@ -152,7 +152,7 @@ yarn install-skills
|
|
|
152
152
|
yarn harness:release --runner codex --prepare-targets /absolute/empty-release-targets --acknowledge-writes
|
|
153
153
|
```
|
|
154
154
|
|
|
155
|
-
The target directory must be absolute, new or empty, and outside the controller app. Select one blocking primary runner with `--runner codex` or `--runner claude`; it owns all
|
|
155
|
+
The target directory must be absolute, new or empty, and outside the controller app. Select one blocking primary runner with `--runner codex` or `--runner claude`; it owns all 234 routing cases and every writable/review lane, with no per-case fallback. Optionally add the different authenticated runner through `--portability-runner` for the exact 49-case representative read-only lane. Omitting it is valid and recorded as not requested; once requested, its failures are blocking. Use a fresh, sanitized controller: automatic preparation fails before copying `.env`/`.env.*` local configuration (safe example/sample/template files remain allowed), credential files, or private-key files. The complete gate requires Linux with trusted system Bubblewrap (`bwrap`) and user namespaces because its Playwright API/browser lanes need a loopback namespace isolated from the host. Preflight rejects untrusted/no-op/pass-through executables and proves isolated loopback plus a capability-free payload before target preparation, provider invocation, or writes; native macOS and Windows therefore fail closed. The command also fails closed when a required runner, browser, or test runtime is unavailable. The 234-case catalog includes 93 framework-neutral business prompts and 49 writable implementation/regression cases (20.9%). The release command runs live routing, writable trusted oracles, per-target `generate`/`typecheck`/`lint`/`build`, any declared generated test, and isolated generated-code review for every writable result. Foundation and target validation—including `yarn build`—receive a minimal environment with network access denied, and persisted diagnostics redact sensitive environment values and URL userinfo. Test-authoring coverage executes a Jest unit test plus Linux/Bubblewrap loopback-only Playwright API and browser tests through fixed controller-owned commands against a read-only target; runtime reports must attest at least one passed test and zero skipped, todo, focused, flaky, or expected-failure tests. The suite then writes a schema-valid sanitized mode-`0600` report under `.ai/harness/results/` with the selected primary and optional portability runner policy.
|
|
156
156
|
|
|
157
157
|
Use the bundled `om-evolve-harness` skill to add a real case: reproduce failure first, select one smallest knowledge owner, run any generated unit/integration tests plus target checks, require code review, and finish with the full release suite. Open Mercato framework maintainers use the monorepo-only `$om-refresh-standalone-harness --from <ref> --to <ref>` workflow for every release range and retain its sanitized maintenance report.
|
|
158
158
|
|
|
@@ -12,7 +12,7 @@ Route first; never probe unmatched context.
|
|
|
12
12
|
- Put entities in `src/modules/<id>/data/entities.ts`; API routes need per-method `metadata` + `openApi`.
|
|
13
13
|
- Editable records expose `updated_at`/`updatedAt`; custom update/delete clients send the version and surface 409s.
|
|
14
14
|
- Run `yarn db:generate`, review scoped SQL/snapshot, and ask before applying it.
|
|
15
|
-
- Run `yarn generate` after discovery files
|
|
15
|
+
- Run `yarn generate` after discovery files/`src/modules.ts`/routes/pages/events/widgets/agents/tools/workflows change.
|
|
16
16
|
- Contract-surface changes (route/schema/ID/export/seam/signature/event payload/CLI) MUST read `.ai/guides/upstream/BACKWARD_COMPATIBILITY.md`; tenant/org scope alone is not a contract.
|
|
17
17
|
- Localize strings; use shared UI/tokens and cover loading/empty/error/conflict/keyboard/a11y.
|
|
18
18
|
|
|
@@ -23,8 +23,8 @@ Route first; never probe unmatched context.
|
|
|
23
23
|
## Never
|
|
24
24
|
|
|
25
25
|
- Never leak tenants, trust payload scope, or treat missing scope as unrestricted.
|
|
26
|
-
- Never edit `node_modules
|
|
27
|
-
- Never use cross-module ORM relations; use IDs/snapshots
|
|
26
|
+
- Never edit `node_modules`/`.mercato/generated/**`/generated facts/shipped migrations.
|
|
27
|
+
- Never use cross-module ORM relations; use IDs/snapshots/events/enrichers/extensions/optional DI.
|
|
28
28
|
- Never use raw admin `fetch`/`<form>`, ad hoc crypto/cache/queues, role-name guards, or direct mutations when helpers exist.
|
|
29
29
|
- Never hard-code user strings/status colors; expose secrets/transcripts; or guess answerable contracts.
|
|
30
30
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Agent harness evaluations
|
|
2
2
|
|
|
3
|
-
`cases.json` is the
|
|
3
|
+
`cases.json` is the 234-case standalone-app contract. Run `yarn harness:validate --all` for the deterministic gate. Live routing uses a fresh read-only process per case:
|
|
4
4
|
|
|
5
5
|
UMES routing is fact-first. The additive and unified-override audit evaluations, plus their targeted cases, resolve exact/pattern hosts, outgoing contributions, correlation provenance, round-trip groups, framework-owned targets, and override domain/key/mode from the generated module sheets and `.ai/guides/framework-extension-points.md` before bounded installed source. The repository UMES umbrella spec may appear as optional source-checkout provenance; it is never required in a standalone scaffold.
|
|
6
6
|
|
|
@@ -18,7 +18,7 @@ yarn harness:release --runner codex --prepare-targets /absolute/empty-release-ta
|
|
|
18
18
|
yarn harness:release --runner codex --portability-runner claude --prepare-targets /absolute/empty-release-targets --acknowledge-writes
|
|
19
19
|
```
|
|
20
20
|
|
|
21
|
-
The primary runner owns all
|
|
21
|
+
The primary runner owns all 234 routing cases, all 49 writable cases, and all generative-judge runs. No per-case fallback or mixed primary ownership is allowed. Omitting `--portability-runner` is valid and the sanitized report records `portabilityRunner: null`; explicitly requesting an unavailable or failing secondary runner fails that extended run.
|
|
22
22
|
|
|
23
23
|
Writable evaluation is intentionally opt-in. The expanded catalog has a 49-case writable release target, but only cases registered in `release-matrix.json` and backed by controller-owned fixtures and oracles are executable. Copy or create a fresh standalone app for one registered case, then seed only that case and mark the target disposable:
|
|
24
24
|
|
|
@@ -6,7 +6,7 @@ Run the complete per-release gate from a generated standalone app with one comma
|
|
|
6
6
|
yarn harness:release --runner codex --prepare-targets /absolute/empty-release-targets --acknowledge-writes
|
|
7
7
|
```
|
|
8
8
|
|
|
9
|
-
Choose exactly one blocking primary runner with `--runner codex` or `--runner claude`. That runner owns the complete
|
|
9
|
+
Choose exactly one blocking primary runner with `--runner codex` or `--runner claude`. That runner owns the complete 234-case routing gate and every writable/review lane. To add cross-model portability evidence, explicitly pass the other runner as `--portability-runner claude` or `--portability-runner codex`; it runs only the exact 49-case representative read-only set. The two runners must differ. Omitting the portability option is valid and is recorded as `portabilityRunner: null`; no secondary result is claimed. There is no per-case fallback or mixed primary ownership. Once requested, a portability failure or unavailable runner fails that extended release run.
|
|
10
10
|
|
|
11
11
|
`--prepare-targets` accepts only an absolute, new or empty regular directory outside the controller app. The controller must be a sanitized fresh scaffold: automatic preparation fails before copying when it finds `.env`, `.env.*` (except `.env.example`, `.env.sample`, and `.env.template`), credential files, or private-key files. Never use a configured development or production app as the controller. It copies the fresh scaffold once per catalog case whose `evaluationKind` is `implementation` or `regression`, while excluding `.git`, `node_modules`, build/cache/coverage output, `.ai/harness/results`, `.ai/reports`, and `.ai/framework-context`. Each target receives a guarded link to the controller's installed dependency tree. The OS sandbox resolves that link as read-only during both the writable model run and the target command gate. The release gate also hashes every dependency entry and regular-file body once before execution and once after the complete suite, and fails if any nested content or metadata changed. A generated `release-targets.json` records the local mapping.
|
|
12
12
|
|
|
@@ -34,25 +34,27 @@ For externally prepared apps, `--writable-targets /absolute/release-targets.json
|
|
|
34
34
|
}
|
|
35
35
|
```
|
|
36
36
|
|
|
37
|
-
The current catalog contains
|
|
37
|
+
The current catalog contains 234 cases, including 49 writable implementation/regression cases (20.9%). The command still derives all counts and case IDs from `cases.json`, `validators.json`, and `release-matrix.json`; those figures are documented release facts, not runner constants. The matrix keeps both supported runner model selectors, an exact all-case primary profile, an exact 49-case portability profile, and runner-neutral writable assignments. Run `yarn install-skills` first so the pinned external `om-code-review` skill and ownership evidence are present; the reusable local `om-judge-agent-session` skill ships with the scaffold. Before running a model or writing a fixture, the release command requires complete deterministic, primary live-routing, writable, trusted-oracle, target, generated-test, and generative-judge coverage. Every one of the 49 writable cases must have a judge assignment composing both skills. Missing business fixtures or release-matrix entries fail preflight and are listed by exact case ID in the report.
|
|
38
38
|
|
|
39
39
|
## PR #4529 remediation evidence
|
|
40
40
|
|
|
41
|
-
The PR's focused remediation evidence is not a release-certification substitute. Fresh emitted controllers pass deterministic 192/192, and the field-tested OMH-188–192 generative cohort passes on default Codex, Claude Sonnet, and high-effort gpt-5.4-mini. Fresh OMH-185 writable attempts fixed concrete organization-scope, command-object, module-activation, command-snapshot, schema, custom-field, UI, and Jest guidance defects at their routed owners without relaxing trusted oracles. The final attempt reached the case's fixed 600-second ceiling and is excluded from pass evidence. Issue #4670 now owns the complete selected-primary
|
|
41
|
+
The PR's focused remediation evidence is not a release-certification substitute. Fresh emitted controllers pass deterministic 192/192, and the field-tested OMH-188–192 generative cohort passes on default Codex, Claude Sonnet, and high-effort gpt-5.4-mini. Fresh OMH-185 writable attempts fixed concrete organization-scope, command-object, module-activation, command-snapshot, schema, custom-field, UI, and Jest guidance defects at their routed owners without relaxing trusted oracles. The final attempt reached the case's fixed 600-second ceiling and is excluded from pass evidence. Issue #4670 now owns the complete selected-primary 234-case routing and 49-case writable/generated-test/review certification, prioritizing the generative cohort and recording unavailable Claude lanes without fallback or mixed-runner ownership.
|
|
42
42
|
|
|
43
43
|
After preflight it runs, in order:
|
|
44
44
|
|
|
45
45
|
1. deterministic validation for the complete catalog;
|
|
46
46
|
2. the release matrix's fixed `yarn generate`, `yarn typecheck`, `yarn lint`, and `yarn build` foundation;
|
|
47
|
-
3. the selected primary runner across all
|
|
47
|
+
3. the selected primary runner across all 234 live-routing cases, followed by the optional distinct portability runner across the exact 49-case read-only sample when requested;
|
|
48
48
|
4. fixture preparation and the selected primary runner for every writable case, including the controller-owned AST/behavior oracles and target typecheck;
|
|
49
49
|
5. `yarn generate`, `yarn typecheck`, `yarn lint`, and `yarn build` in every writable target, after its trusted oracles;
|
|
50
50
|
6. real generated-code execution for OMH-163 and OMH-192 through fixed Jest, OMH-164 through API-only Playwright, and OMH-165 through real-browser Playwright; and
|
|
51
51
|
7. explicit isolated `om-judge-agent-session` for every writable result, composing `om-code-review` and applicable design-system guidance, bound to its passing command attestation, any required generated-test result and artifact hash, and the final target fingerprint.
|
|
52
52
|
|
|
53
|
-
Each writable target is single-use because fixture preparation marks it disposable. Externally supplied target realpaths must be pairwise disjoint and neither equal to, contain, nor be contained by the controller. A failed deterministic or foundation-validation step prevents model execution. Once fixture preparation succeeds, all four target commands run even when the writable gate itself fails, so every generated target has exact diagnostics. A writable case may declare `timeoutMs` only to raise the release `--case-timeout` floor (never lower it); OMH-185 and its business-language parity case OMH-193 use 600000 ms because the complete module slice exceeded the generic five-minute evaluator default while actively producing source. Generated tests run only after the trusted writable oracle and all four target commands pass; review then requires all applicable gates. A target command or generated-test failure is recorded with its sanitized diagnostic and review is skipped. Other matrix entries continue so the report remains useful.
|
|
53
|
+
Each writable target is single-use because fixture preparation marks it disposable. Externally supplied target realpaths must be pairwise disjoint and neither equal to, contain, nor be contained by the controller. A failed deterministic or foundation-validation step prevents model execution. Once fixture preparation succeeds, all four target commands run even when the writable gate itself fails, so every generated target has exact diagnostics. A writable case may declare `timeoutMs` only to raise the release `--case-timeout` floor (never lower it); OMH-185 and its business-language parity case OMH-193 use 600000 ms because the complete module slice exceeded the generic five-minute evaluator default while actively producing source. Since the schema caps `timeoutMs` at exactly the shipped `--case-timeout` default, a declared value only raises anything for an operator who lowered that default — it is the floor under a lowered budget, not an addition to the shipped one. Generated tests run only after the trusted writable oracle and all four target commands pass; review then requires all applicable gates. A target command or generated-test failure is recorded with its sanitized diagnostic and review is skipped. Other matrix entries continue so the report remains useful.
|
|
54
54
|
|
|
55
|
-
Routing cases carry no case-local duration budget, and that is a decision rather than an omission. `maxContextFiles` and the byte budgets measure the agent's context discipline, which is intrinsic to the case and portable between machines; duration measures the runner and the attempt count, which are not — the audited cohort spanned 71 s to 231 s for passing routing runs, one case measured 147 s and 132 s on two runs of the same model, and OMH-139 exhausted the evaluator's 300000 ms default outright. The operator budget carries that variance instead of the catalog. On this release path the lever is `--case-timeout` (default
|
|
55
|
+
Routing cases carry no case-local duration budget, and that is a decision rather than an omission. `maxContextFiles` and the byte budgets measure the agent's context discipline, which is intrinsic to the case and portable between machines; duration measures the runner and the attempt count, which are not — the audited cohort spanned 71 s to 231 s for passing routing runs, one case measured 147 s and 132 s on two runs of the same model, and OMH-139 exhausted the evaluator's 300000 ms default outright. The operator budget carries that variance instead of the catalog. On this release path the lever is `--case-timeout` (default 600000 ms), which the release command passes on to the evaluator explicitly for every routing step; because that pass-through marks the timeout explicit, the evaluator's own runner-aware floors — 600000 ms for Claude, 900000 ms for a Codex `gpt-5.4-mini` high-effort run, 300000 ms for every other runner — apply only to direct `evaluate-agent-harness.mjs` invocations and never fire under `yarn harness:release`. Keeping that pass-through is deliberate rather than inherited: the routing step derives its own process budget from the same value, as slack plus the sum of the per-case ceilings it hands out, so letting the floors raise the inner ceiling while the outer budget still followed `--case-timeout` would kill an entire routing step instead of failing one slow case. Leaving it explicit also keeps the budget runner-independent, which is what lets the primary and portability lanes in one report be compared as models rather than as budgets. The default clears the slowest audited passing routing run — 231 s against 600000 ms, about 62% headroom — and matches both the Claude floor and the `timeoutMs` ceiling `cases.schema.json` enforces, so the gate carries one upper number instead of three; lower `--case-timeout` when a hung case should fail sooner, and note that the step's process budget drops with it — though at a lowered budget the declared writable ceilings keep their own slots, so the step's budget drops less than proportionally. That value was chosen from evaluator-path measurements — the audited cohort above and the live evidence recorded on #5068 — rather than from a driven `yarn harness:release` run, because the complete gate fails closed off the Linux-with-Bubblewrap host stated above; #5078 records the decision and that deviation, and #5433 carries the unmet measurement forward so the default is confirmed or corrected against the release path's own numbers. `--case-timeout` is one budget for three lanes, not a routing-only lever: the writable and review lanes resolve their own ceilings from the same value, so raising it raises theirs too. A declared writable `timeoutMs` is combined with whichever operator value applies as a maximum, so it raises the floor and never lowers it — but because the schema caps it at the shipped default, it can only raise a budget an operator has lowered; routing cases declare none, so there the operator value always stands alone. Writable cases keep their own `timeoutMs` because a writable one-shot's cost is dominated by the slice it must produce, which the case does define.
|
|
56
|
+
|
|
57
|
+
`--case-timeout` governs the model lanes and only those: routing, writable, and the judge invocation that reviews a writable result. The two steps that invoke no model carry their own flat ceilings instead. Fixture preparation has always used 120000 ms, and the deterministic step now uses the exported `DETERMINISTIC_STEP_TIMEOUT_MS`, the same 120000 ms, rather than the per-model ceiling multiplied by catalog size it derived before. That earlier derivation handed a model-free step a budget that moved whenever an operator changed how patient the gate is with a language model, and it grew that step's ceiling by a factor of five when #5180 raised the `--case-timeout` default from 120000 ms to 600000 ms — roughly 39 hours over the shipped catalog for a pass that finishes in under a second. The replacement is measured rather than chosen freehand: timed on Linux x86_64, a complete 234-case deterministic pass finished in 768 ms to 998 ms over five runs, and narrowing the selection down to a single case measured 842 ms to 890 ms — inside the same spread, so the pass is dominated by process start and catalog load rather than by how many cases it validates. A ceiling roughly 120 times the slowest observed pass therefore bounds a hang without ever bounding a healthy run, and it has no reason to scale with the catalog. `packages/create-app/src/lib/agent-harness-release.test.ts` pins the deterministic argv together with that budget, so the model ceiling and the model-free one cannot silently converge again.
|
|
56
58
|
|
|
57
59
|
UI-routed implementation reviews receive only the bounded backend UI guide and `om-backend-ui-design` design-system references. Non-UI reviews do not receive that extra context.
|
|
58
60
|
|
|
@@ -21100,5 +21100,88 @@
|
|
|
21100
21100
|
"OMH-039",
|
|
21101
21101
|
"OMH-232"
|
|
21102
21102
|
]
|
|
21103
|
+
},
|
|
21104
|
+
{
|
|
21105
|
+
"id": "OMH-234",
|
|
21106
|
+
"title": "Reuse installed collaborative documents instead of building a private notes system",
|
|
21107
|
+
"family": "architecture",
|
|
21108
|
+
"mode": "analysis",
|
|
21109
|
+
"evaluationKind": "routing",
|
|
21110
|
+
"risk": "high",
|
|
21111
|
+
"prompt": "In a freshly scaffolded standalone Open Mercato app, account teams need shared rich documents attached to customers, deals, quotes, and orders. Several staff members must be able to edit at the same time, discuss selected text, mention colleagues, restore earlier versions, reuse templates, attach files, and grant viewer, commenter, editor, or owner access per document. Decide the smallest safe design against what the installed modules already provide, keep tenant, organization, and record-level visibility boundaries intact, state the access-control posture for reading, commenting, editing, sharing, and managing templates, and identify the smallest relevant validation. Do not model your own document, content, comment, share, version, attachment, or template tables before establishing whether an installed module already owns this capability.",
|
|
21112
|
+
"tags": [
|
|
21113
|
+
"architecture",
|
|
21114
|
+
"module-facts",
|
|
21115
|
+
"reuse-installed",
|
|
21116
|
+
"collaborative-documents"
|
|
21117
|
+
],
|
|
21118
|
+
"owner": {
|
|
21119
|
+
"kind": "facts",
|
|
21120
|
+
"path": ".ai/guides/modules/documents/index.md",
|
|
21121
|
+
"ruleIds": [
|
|
21122
|
+
"BC-01",
|
|
21123
|
+
"BC-04"
|
|
21124
|
+
]
|
|
21125
|
+
},
|
|
21126
|
+
"expectedRouter": {
|
|
21127
|
+
"required": [
|
|
21128
|
+
"architecture"
|
|
21129
|
+
],
|
|
21130
|
+
"allowedExtra": [
|
|
21131
|
+
"module-data",
|
|
21132
|
+
"backend-ui",
|
|
21133
|
+
"umes",
|
|
21134
|
+
"framework-context"
|
|
21135
|
+
]
|
|
21136
|
+
},
|
|
21137
|
+
"requiredSkills": [
|
|
21138
|
+
"om-help"
|
|
21139
|
+
],
|
|
21140
|
+
"context": {
|
|
21141
|
+
"required": [
|
|
21142
|
+
"AGENTS.md",
|
|
21143
|
+
".ai/guides/architecture.md",
|
|
21144
|
+
".ai/skills/om-help/SKILL.md",
|
|
21145
|
+
".ai/guides/modules/documents/index.md"
|
|
21146
|
+
],
|
|
21147
|
+
"allowedExtra": [
|
|
21148
|
+
".ai/guides/contracts.md",
|
|
21149
|
+
".ai/guides/extensions.md",
|
|
21150
|
+
".ai/guides/backend-ui.md"
|
|
21151
|
+
],
|
|
21152
|
+
"forbidden": [
|
|
21153
|
+
".env*",
|
|
21154
|
+
".git/**"
|
|
21155
|
+
]
|
|
21156
|
+
},
|
|
21157
|
+
"requiredDecisions": [
|
|
21158
|
+
"facts-first",
|
|
21159
|
+
"reuse-installed-documents",
|
|
21160
|
+
"per-document-access",
|
|
21161
|
+
"tenant-scope",
|
|
21162
|
+
"acl-features",
|
|
21163
|
+
"smallest-validation"
|
|
21164
|
+
],
|
|
21165
|
+
"forbiddenPatterns": [
|
|
21166
|
+
"node_modules.{0,40}(?:write|edit|patch)",
|
|
21167
|
+
"(?:tenant|organization).{0,30}(?:unscoped|scope optional)"
|
|
21168
|
+
],
|
|
21169
|
+
"validators": [
|
|
21170
|
+
"catalog.schema",
|
|
21171
|
+
"owner.reference",
|
|
21172
|
+
"skills.reference",
|
|
21173
|
+
"router.contract",
|
|
21174
|
+
"context.budget",
|
|
21175
|
+
"context.forbidden",
|
|
21176
|
+
"patterns.forbidden"
|
|
21177
|
+
],
|
|
21178
|
+
"maxContextFiles": 8,
|
|
21179
|
+
"maxInitialContextBytes": 53248,
|
|
21180
|
+
"maxTotalContextBytes": 147456,
|
|
21181
|
+
"relatedCases": [
|
|
21182
|
+
"OMH-002",
|
|
21183
|
+
"OMH-194",
|
|
21184
|
+
"OMH-210"
|
|
21185
|
+
]
|
|
21103
21186
|
}
|
|
21104
21187
|
]
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
"$id": "https://open-mercato.dev/schemas/standalone-harness-cases.schema.json",
|
|
4
4
|
"title": "Open Mercato standalone harness case catalog",
|
|
5
5
|
"type": "array",
|
|
6
|
-
"minItems":
|
|
7
|
-
"maxItems":
|
|
6
|
+
"minItems": 234,
|
|
7
|
+
"maxItems": 234,
|
|
8
8
|
"items": {
|
|
9
9
|
"type": "object",
|
|
10
10
|
"additionalProperties": false,
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
"maxTotalContextBytes", "relatedCases"
|
|
16
16
|
],
|
|
17
17
|
"properties": {
|
|
18
|
-
"id": { "type": "string", "pattern": "^OMH-(00[1-9]|0[1-9][0-9]|1[0-9][0-9]|20[0-9]|21[0-9]|22[0-9]|23[0-
|
|
18
|
+
"id": { "type": "string", "pattern": "^OMH-(00[1-9]|0[1-9][0-9]|1[0-9][0-9]|20[0-9]|21[0-9]|22[0-9]|23[0-4])$" },
|
|
19
19
|
"title": { "type": "string", "minLength": 12, "maxLength": 180 },
|
|
20
20
|
"family": { "enum": ["architecture", "module", "umes", "integration", "ai-workflow", "bugfix", "business", "testing"] },
|
|
21
21
|
"mode": { "enum": ["analysis", "one-shot", "spec", "bugfix", "review"] },
|
|
@@ -168,7 +168,7 @@
|
|
|
168
168
|
"maxInitialContextBytes": { "type": "integer", "minimum": 4096, "maximum": 98304 },
|
|
169
169
|
"maxTotalContextBytes": { "type": "integer", "minimum": 8192, "maximum": 1048576 },
|
|
170
170
|
"timeoutMs": { "type": "integer", "minimum": 1000, "maximum": 600000 },
|
|
171
|
-
"relatedCases": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "type": "string", "pattern": "^OMH-(00[1-9]|0[1-9][0-9]|1[0-9][0-9]|20[0-9]|21[0-9]|22[0-9]|23[0-
|
|
171
|
+
"relatedCases": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "type": "string", "pattern": "^OMH-(00[1-9]|0[1-9][0-9]|1[0-9][0-9]|20[0-9]|21[0-9]|22[0-9]|23[0-4])$" } },
|
|
172
172
|
"source": {
|
|
173
173
|
"type": "object",
|
|
174
174
|
"additionalProperties": false,
|
|
@@ -25,6 +25,11 @@ const GENERATED_TEST_RUNNERS = new Set(['jest', 'playwright-api', 'playwright-br
|
|
|
25
25
|
const RESULT_LIMIT = 262_144
|
|
26
26
|
const ERROR_LIMIT = 2_000
|
|
27
27
|
const VIOLATION_LIMIT = 300
|
|
28
|
+
// The routing step hands this to the evaluator as --timeout and derives its own process budget from
|
|
29
|
+
// the same value, so the two cannot be stated separately; the help text reads it rather than
|
|
30
|
+
// repeating it (#5078).
|
|
31
|
+
export const DEFAULT_CASE_TIMEOUT_MS = 600_000
|
|
32
|
+
export const ROUTING_STEP_SLACK_MS = 60_000
|
|
28
33
|
const COPY_EXCLUDED_PREFIXES = [
|
|
29
34
|
'.git', '.next', '.turbo', '.cache', 'build', 'coverage', 'dist', 'node_modules', 'out',
|
|
30
35
|
'.ai/framework-context', '.ai/harness/results', '.ai/reports',
|
|
@@ -36,6 +41,14 @@ const SENSITIVE_AUTH_DATA_FILE = /^(?:auth|credentials?|secrets?|tokens?)(?:\.(?
|
|
|
36
41
|
const SENSITIVE_ENV_KEY = /(?:^|_)(?:api_?key|auth|credential|credentials|password|passwd|private_?key|secret|token)(?:_|$)/i
|
|
37
42
|
const GENERATED_YARN_CONFIG_PATH = '.yarnrc.yml'
|
|
38
43
|
const GENERATED_YARN_CONFIG_LIMIT = 16_384
|
|
44
|
+
// The deterministic step invokes no model, so its ceiling must not ride --case-timeout. Timed on
|
|
45
|
+
// Linux x86_64, the complete 234-case catalog finished in 768-998 ms, and narrowing the selection
|
|
46
|
+
// down to a single case measured 842-890 ms, inside the same spread, so the run is dominated by
|
|
47
|
+
// fixed process and catalog load rather than by case count. A flat allowance is therefore the
|
|
48
|
+
// honest shape, and 120000 ms is both about 120x the slowest observed run and the value this gate
|
|
49
|
+
// already gives its other model-free step, fixture preparation. It bounds a hang; a healthy run
|
|
50
|
+
// never approaches it.
|
|
51
|
+
export const DETERMINISTIC_STEP_TIMEOUT_MS = 120_000
|
|
39
52
|
|
|
40
53
|
function usage() {
|
|
41
54
|
return `Run the complete standalone agent-harness release gate.
|
|
@@ -50,7 +63,8 @@ Options:
|
|
|
50
63
|
--portability-runner <runner> Optional different runner for the 49-case read-only portability lane
|
|
51
64
|
--prepare-targets <absolute> Clone this fresh scaffold once per writable case under an empty/new directory
|
|
52
65
|
--writable-targets <absolute> JSON map of every writable case to a fresh disposable app
|
|
53
|
-
--case-timeout <ms> Per-model invocation timeout floor
|
|
66
|
+
--case-timeout <ms> Per-model invocation timeout floor for the routing, writable, and review lanes (default: ${DEFAULT_CASE_TIMEOUT_MS})
|
|
67
|
+
The model-free deterministic and fixture-preparation steps carry their own flat ceilings and do not read it.
|
|
54
68
|
--validation-timeout <ms> Timeout for each yarn validation (default: 1800000)
|
|
55
69
|
--acknowledge-writes Required: fixture preparation and validation commands write files
|
|
56
70
|
--help Show this help
|
|
@@ -76,7 +90,7 @@ function parseArgs(argv) {
|
|
|
76
90
|
portabilityRunner: undefined,
|
|
77
91
|
prepareTargets: undefined,
|
|
78
92
|
writableTargets: undefined,
|
|
79
|
-
caseTimeout:
|
|
93
|
+
caseTimeout: DEFAULT_CASE_TIMEOUT_MS,
|
|
80
94
|
validationTimeout: 1_800_000,
|
|
81
95
|
acknowledgeWrites: false,
|
|
82
96
|
help: false,
|
|
@@ -118,6 +132,21 @@ export function effectiveCaseTimeout(cases, caseId, fallback) {
|
|
|
118
132
|
return Math.max(fallback, Number.isInteger(declared) ? declared : 0)
|
|
119
133
|
}
|
|
120
134
|
|
|
135
|
+
export function deterministicInvocation({ evaluator, root }) {
|
|
136
|
+
return { args: [evaluator, '--root', root, '--all'], timeout: DETERMINISTIC_STEP_TIMEOUT_MS }
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export function routingInvocation({ evaluator, root, step, cases, caseTimeout }) {
|
|
140
|
+
const args = [evaluator, '--root', root, '--runner', step.runner]
|
|
141
|
+
if (step.lane === 'primary') args.push('--all')
|
|
142
|
+
args.push('--model', step.modelSelector, '--timeout', String(caseTimeout))
|
|
143
|
+
const timeout = step.expectedCaseIds.reduce(
|
|
144
|
+
(total, caseId) => total + effectiveCaseTimeout(cases, caseId, caseTimeout),
|
|
145
|
+
ROUTING_STEP_SLACK_MS,
|
|
146
|
+
)
|
|
147
|
+
return { args, timeout }
|
|
148
|
+
}
|
|
149
|
+
|
|
121
150
|
function readJson(file) {
|
|
122
151
|
return JSON.parse(fs.readFileSync(file, 'utf8'))
|
|
123
152
|
}
|
|
@@ -1575,11 +1604,12 @@ export function main(argv = process.argv.slice(2)) {
|
|
|
1575
1604
|
const { env: foundationEnv } = createMinimalValidationEnvironment(foundationTempRoot)
|
|
1576
1605
|
let deterministicResult
|
|
1577
1606
|
try {
|
|
1607
|
+
const deterministic = deterministicInvocation({ evaluator, root })
|
|
1578
1608
|
const deterministicExecution = execute(
|
|
1579
1609
|
process.execPath,
|
|
1580
|
-
|
|
1610
|
+
deterministic.args,
|
|
1581
1611
|
root,
|
|
1582
|
-
|
|
1612
|
+
deterministic.timeout,
|
|
1583
1613
|
foundationEnv,
|
|
1584
1614
|
)
|
|
1585
1615
|
const deterministicObserved = sortedUnique((deterministicExecution.stdout ?? '').match(/^PASS (OMH-[0-9]{3})/gm)?.map((entry) => entry.slice('PASS '.length)) ?? [])
|
|
@@ -1614,13 +1644,9 @@ export function main(argv = process.argv.slice(2)) {
|
|
|
1614
1644
|
} else {
|
|
1615
1645
|
for (const step of plan.steps.filter((entry) => entry.kind === 'routing')) {
|
|
1616
1646
|
const before = resultFiles(root)
|
|
1617
|
-
const
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
const routingTimeout = step.expectedCaseIds.reduce(
|
|
1621
|
-
(total, caseId) => total + effectiveCaseTimeout(cases, caseId, options.caseTimeout),
|
|
1622
|
-
60_000,
|
|
1623
|
-
)
|
|
1647
|
+
const { args: routingArgs, timeout: routingTimeout } = routingInvocation({
|
|
1648
|
+
evaluator, root, step, cases, caseTimeout: options.caseTimeout,
|
|
1649
|
+
})
|
|
1624
1650
|
const execution = execute(process.execPath, routingArgs, root, routingTimeout)
|
|
1625
1651
|
const artifacts = readNewResults(root, before)
|
|
1626
1652
|
resultArtifacts.push(...artifacts)
|