@lovrozagar/oat 0.7.4 → 0.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +175 -146
- package/dist/cli.js +26 -1
- package/dist/config/define-config.d.ts +17 -0
- package/dist/config/define-config.d.ts.map +1 -1
- package/dist/config/define-config.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1 -0
- package/dist/report/console.d.ts.map +1 -1
- package/dist/report/console.js +1 -0
- package/dist/report/console.js.map +1 -1
- package/dist/report/render.d.ts +11 -0
- package/dist/report/render.d.ts.map +1 -1
- package/dist/report/render.js +9 -0
- package/dist/report/render.js.map +1 -1
- package/dist/runtime/checks.d.ts.map +1 -1
- package/dist/runtime/checks.js +120 -58
- package/dist/runtime/checks.js.map +1 -1
- package/dist/runtime/client.d.ts +22 -2
- package/dist/runtime/client.d.ts.map +1 -1
- package/dist/runtime/client.js +72 -8
- package/dist/runtime/client.js.map +1 -1
- package/dist/runtime/effects.d.ts +43 -0
- package/dist/runtime/effects.d.ts.map +1 -0
- package/dist/runtime/effects.js +141 -0
- package/dist/runtime/effects.js.map +1 -0
- package/dist/runtime/exchanges.d.ts +37 -0
- package/dist/runtime/exchanges.d.ts.map +1 -0
- package/dist/runtime/exchanges.js +298 -0
- package/dist/runtime/exchanges.js.map +1 -0
- package/dist/runtime/network.d.ts +74 -0
- package/dist/runtime/network.d.ts.map +1 -0
- package/dist/runtime/network.js +270 -0
- package/dist/runtime/network.js.map +1 -0
- package/dist/runtime/progress.d.ts +2 -0
- package/dist/runtime/progress.d.ts.map +1 -1
- package/dist/runtime/progress.js +2 -2
- package/dist/runtime/progress.js.map +1 -1
- package/dist/runtime/run.d.ts +23 -0
- package/dist/runtime/run.d.ts.map +1 -1
- package/dist/runtime/run.js +107 -8
- package/dist/runtime/run.js.map +1 -1
- package/dist/spec/extensions.d.ts +4 -1
- package/dist/spec/extensions.d.ts.map +1 -1
- package/dist/spec/extensions.js +8 -2
- package/dist/spec/extensions.js.map +1 -1
- package/dist/spec/graph.js +1 -1
- package/dist/spec/graph.js.map +1 -1
- package/dist/spec/load.d.ts +2 -1
- package/dist/spec/load.d.ts.map +1 -1
- package/dist/spec/load.js +8 -7
- package/dist/spec/load.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -255,18 +255,19 @@ oat help
|
|
|
255
255
|
|
|
256
256
|
Requires `--config`. CLI flags override the same field in the config when both are set.
|
|
257
257
|
|
|
258
|
-
| flag
|
|
259
|
-
|
|
|
260
|
-
| `--config`
|
|
261
|
-
| `--base-url`
|
|
262
|
-
| `--only`
|
|
263
|
-
| `--seed`
|
|
264
|
-
| `--out`
|
|
265
|
-
| `--max-in-flight`
|
|
266
|
-
| `--keep-fixtures`
|
|
267
|
-
| `--quiet`
|
|
268
|
-
|
|
269
|
-
|
|
258
|
+
| flag | default | meaning |
|
|
259
|
+
| ------------------------------------------ | -------------------------------- | ----------------------------------------------------------------------------------------- |
|
|
260
|
+
| `--config` | required | module or JSON file, default export |
|
|
261
|
+
| `--base-url` | `config.baseUrl` | backend origin |
|
|
262
|
+
| `--only` | `config.only` or all entities | comma-separated entity names as `oat plan` prints them (singularised) |
|
|
263
|
+
| `--seed` | `config.seed` or `1` | fixture generation seed (reproducible) |
|
|
264
|
+
| `--out` | `config.outDir` or `./.oat/runs` | history root; each run writes `<out>/<datetime>/` and updates `latest` |
|
|
265
|
+
| `--max-in-flight` | `config.maxInFlight` or `4` | HTTP requests allowed at once |
|
|
266
|
+
| `--keep-fixtures` | `config.keepFixtures` or false | do not DELETE what the run created |
|
|
267
|
+
| `--quiet` | false | no stderr progress; files under `--out` still update |
|
|
268
|
+
| `--save-exchanges` / `--no-save-exchanges` | on unless `--profile cheap` | persist every HTTP exchange under the run dir (`exchanges.jsonl`, `exchanges/`, `blobs/`) |
|
|
269
|
+
|
|
270
|
+
**Exit codes:** `0` no defects, `1` at least one root-cause finding (`BACKEND_BUG`, `SPEC_BUG`, `SECURITY`, `AMBIGUITY`) **or** the run stopped because the network never came back, `2` usage error (missing `--config`, no principals, unknown flag). `COVERAGE_GAP` and `BLOCKED` do not fail the process.
|
|
270
271
|
|
|
271
272
|
Example:
|
|
272
273
|
|
|
@@ -435,6 +436,8 @@ export default defineConfig({
|
|
|
435
436
|
only: ["store", "product"], // restrict entities
|
|
436
437
|
keepFixtures: false,
|
|
437
438
|
outDir: "./.oat/runs",
|
|
439
|
+
saveExchanges: true, // default on unless --profile cheap; --quiet does not turn this off
|
|
440
|
+
network: { retries: 4, waitMs: 60_000 }, // fetch-threw: retry, then wait for the link
|
|
438
441
|
outOfBand: { attempts: 20, initialMs: 1000, maxMs: 8000 },
|
|
439
442
|
origins: [{ id: "cdn", baseUrl: "https://cdn.example.com", spec: "https://cdn.example.com/openapi.json" }],
|
|
440
443
|
query: {
|
|
@@ -456,29 +459,31 @@ export default defineConfig({
|
|
|
456
459
|
})
|
|
457
460
|
```
|
|
458
461
|
|
|
459
|
-
| field | required | default | notes
|
|
460
|
-
| --------------- | -------- | ---------------------------------------------- |
|
|
461
|
-
| `spec` | yes | | See [Spec loading](#spec-loading)
|
|
462
|
-
| `baseUrl` | yes | | Primary origin. OpenAPI `servers[]` is ignored
|
|
463
|
-
| `principals` | yes | | Non-empty. First is the writer
|
|
464
|
-
| `hooks` | no | | See [Hooks](#hooks)
|
|
465
|
-
| `uploads` | no | | `pool` globs; optional `each` (operationId → globs) and `eachMax`. JSON configs may set all three
|
|
466
|
-
| `globalHeaders` | no | `{}` | Merged first. `resolveHeaders` then caller headers then auth
|
|
467
|
-
| `origins` | no | `[]` | Extra `{ id, baseUrl, spec }` hosts. Auth JWT is reused. Do not merge those routes into `spec`
|
|
468
|
-
| `outOfBand` | no | `{ attempts: 6, initialMs: 200, maxMs: 3000 }` | Backoff for `resolveOutOfBand` and `resolvePrincipalAuth`. See [Hooks](#hooks)
|
|
469
|
-
| `roots` | no | `{}` | Shared path params (merged with each principal's `roots`)
|
|
470
|
-
| `seed` | no | `1` | Integer. Same seed → same fixture bodies
|
|
471
|
-
| `cohortSize` | no | `7` | Sliced from the 7 built-in variants. Larger repeats the pattern
|
|
472
|
-
| `maxInFlight` | no | `4` | Across the whole run
|
|
473
|
-
| `only` | no | all | Entity names from `oat plan`
|
|
474
|
-
| `keepFixtures` | no | `false` | Skip DELETE at the end
|
|
475
|
-
| `outDir` | no | `./.oat/runs` | History root. Each run writes `<outDir>/<datetime>/` and updates `latest`. Also writes `principals.json` after acquire
|
|
476
|
-
| `
|
|
477
|
-
| `
|
|
462
|
+
| field | required | default | notes |
|
|
463
|
+
| --------------- | -------- | ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
464
|
+
| `spec` | yes | | See [Spec loading](#spec-loading) |
|
|
465
|
+
| `baseUrl` | yes | | Primary origin. OpenAPI `servers[]` is ignored |
|
|
466
|
+
| `principals` | yes | | Non-empty. First is the writer |
|
|
467
|
+
| `hooks` | no | | See [Hooks](#hooks) |
|
|
468
|
+
| `uploads` | no | | `pool` globs; optional `each` (operationId → globs) and `eachMax`. JSON configs may set all three |
|
|
469
|
+
| `globalHeaders` | no | `{}` | Merged first. `resolveHeaders` then caller headers then auth |
|
|
470
|
+
| `origins` | no | `[]` | Extra `{ id, baseUrl, spec }` hosts. Auth JWT is reused. Do not merge those routes into `spec` |
|
|
471
|
+
| `outOfBand` | no | `{ attempts: 6, initialMs: 200, maxMs: 3000 }` | Backoff for `resolveOutOfBand` and `resolvePrincipalAuth`. See [Hooks](#hooks) |
|
|
472
|
+
| `roots` | no | `{}` | Shared path params (merged with each principal's `roots`) |
|
|
473
|
+
| `seed` | no | `1` | Integer. Same seed → same fixture bodies |
|
|
474
|
+
| `cohortSize` | no | `7` | Sliced from the 7 built-in variants. Larger repeats the pattern |
|
|
475
|
+
| `maxInFlight` | no | `4` | Across the whole run |
|
|
476
|
+
| `only` | no | all | Entity names from `oat plan` |
|
|
477
|
+
| `keepFixtures` | no | `false` | Skip DELETE at the end |
|
|
478
|
+
| `outDir` | no | `./.oat/runs` | History root. Each run writes `<outDir>/<datetime>/` and updates `latest`. Also writes `principals.json` after acquire |
|
|
479
|
+
| `saveExchanges` | no | on unless `profile` is `cheap` | Persist every HTTP exchange under the run dir. `--save-exchanges` / `--no-save-exchanges` override. `--quiet` does not |
|
|
480
|
+
| `network` | no | `{ retries: 4, waitMs: 60000 }` | When `fetch` throws (offline / DNS / reset / timeout): retry, then wait once for the link. Not a 5xx policy. `requestTimeoutMs` is optional |
|
|
481
|
+
| `query` | no | | Global query-catalog defaults. Overlay after `x-query`. Does not invent operators |
|
|
482
|
+
| `entities` | no | | Per-entity overlays. This release only reads `query`. Unknown names are ignored; `doctor` warns |
|
|
478
483
|
|
|
479
484
|
`spec` may be a path relative to `baseUrl` (`/v1/openapi/spec`) or an absolute URL or a file.
|
|
480
485
|
|
|
481
|
-
CLI `--base-url`, `--only`, `--seed`, `--out`, `--max-in-flight`, `--keep-fixtures` override these when passed.
|
|
486
|
+
CLI `--base-url`, `--only`, `--seed`, `--out`, `--max-in-flight`, `--keep-fixtures`, `--save-exchanges` / `--no-save-exchanges` override these when passed.
|
|
482
487
|
|
|
483
488
|
### Spec loading
|
|
484
489
|
|
|
@@ -1269,11 +1274,20 @@ A spec may leave both `x-async` and `text/event-stream` on the same operation. T
|
|
|
1269
1274
|
|
|
1270
1275
|
```yaml
|
|
1271
1276
|
x-effects:
|
|
1272
|
-
- { entity: table, op: create
|
|
1273
|
-
- { entity:
|
|
1277
|
+
- { entity: table, op: create }
|
|
1278
|
+
- { entity: row, op: append, min: 1 }
|
|
1274
1279
|
```
|
|
1275
1280
|
|
|
1276
|
-
`op`: `create` | `append` | `update` | `delete` | `replace`.
|
|
1281
|
+
`op`: `create` | `append` | `update` | `delete` | `replace`. Each item is `{ entity, op, count?, min? }`.
|
|
1282
|
+
|
|
1283
|
+
- `count` is an **exact** cardinality delta on that entity's list.
|
|
1284
|
+
- `min` is **at-least** (`delta >= min` and `added.length >= min`). Use this when the child count is data-dependent (an extract that appends 1 / 2 / 5 rows).
|
|
1285
|
+
- Omit both → `count: 1` (invite / create).
|
|
1286
|
+
- Set both on one item → rejected at load / `oat doctor` (`x-effects` gap). The item is not checked.
|
|
1287
|
+
|
|
1288
|
+
After a write that declares `create` on A, oat binds the new A id from the write response (`table_id`, or the entity identity) or from A's list delta — the same adopt idea as a 402 plan-limit reuse. Later items in **that same** `x-effects` array whose child list is under A (`GET .../tables/{table_id}/rows`) fill the path with that id. `x-wait` after the same write uses the same bound id. No tester hook.
|
|
1289
|
+
|
|
1290
|
+
An extract-shaped write (create table + append ≥1 row) fails when the new table's row list is empty, and passes when it has 5.
|
|
1277
1291
|
|
|
1278
1292
|
**Fallback:** derived from `x-entity.action` for this entity only.
|
|
1279
1293
|
|
|
@@ -1289,7 +1303,7 @@ x-wait:
|
|
|
1289
1303
|
|
|
1290
1304
|
Put this on the **write**. After that write succeeds, oat polls `operationId` until `until` (JSON path `$.items.0` or JSON pointer `/items/0`) is non-empty, or `hooks.awaitSideEffect` returns `true`. Default `timeoutMs` is **30s**. Timeout is a **finding** (`effects.side-effect-arrives`), not a coverage gap.
|
|
1291
1305
|
|
|
1292
|
-
Use this for queue consumers and webhook inboxes (1–30 s), not for the same request. `x-effects` still asserts cardinality; `x-wait` asserts “this other GET eventually has a body”.
|
|
1306
|
+
Use this for queue consumers and webhook inboxes (1–30 s), not for the same request. `x-effects` still asserts cardinality; `x-wait` asserts “this other GET eventually has a body”. If the poll path needs a parent id the write created, oat binds it the same way `x-effects` does.
|
|
1293
1307
|
|
|
1294
1308
|
**Fallback:** the check does not run.
|
|
1295
1309
|
|
|
@@ -1424,6 +1438,8 @@ Exclusion applies to every invocation, not only create/read/update/delete. `effe
|
|
|
1424
1438
|
|
|
1425
1439
|
The run summary states what a profile skipped: `skipped 12 operation(s) under --profile cheap (12 high-cost)` — the same principle as `did not apply` for a coverage gap: a report that only shows what ran invites the reader to assume the rest was verified.
|
|
1426
1440
|
|
|
1441
|
+
`--profile cheap` also defaults the exchange journal **off**. `--save-exchanges` turns it back on; `--quiet` does not change it.
|
|
1442
|
+
|
|
1427
1443
|
### `x-rate-limit` — pacing oat's own traffic
|
|
1428
1444
|
|
|
1429
1445
|
```yaml
|
|
@@ -1457,102 +1473,102 @@ Order is fixed (foundations first) so cascade suppression has a cause to point a
|
|
|
1457
1473
|
|
|
1458
1474
|
`depends` is the `dependsOn` list: if any of those already failed **for this entity**, this check is `BLOCKED` rather than reported as a second defect. Suppression is transitive.
|
|
1459
1475
|
|
|
1460
|
-
| id | asserts
|
|
1461
|
-
| ---------------------------------------------- |
|
|
1462
|
-
| `list.read-after-write` | a just-created record appears on the list
|
|
1463
|
-
| `create.persists-submitted-fields` | every writable field sent on create is echoed
|
|
1464
|
-
| `payload.string-survives` | a documented-valid string is stored exactly; 4xx after an ASCII control is a fail
|
|
1465
|
-
| `create.status-matches-document` | create status is one the document declared
|
|
1466
|
-
| `response.status-is-documented` | every non-create exchange returns a status that operation names (`default` ≠ 201)
|
|
1467
|
-
| `schema.success-response-matches-document` | create body validates against the success schema
|
|
1468
|
-
| `schema.error-response-matches-document` | an error body validates against the documented error schema
|
|
1469
|
-
| `pagination.limit-bounds-page-size` | page size ≤ the requested limit
|
|
1470
|
-
| `pagination.limit-respects-documented-max` | requesting more than `maxLimit` does not return more
|
|
1471
|
-
| `pagination.has-more-is-accurate` | `hasMore` / `Link rel=next` matches whether another page exists
|
|
1472
|
-
| `pagination.page-walk-covers-set` | walking pages covers the collection with no gaps or dupes
|
|
1473
|
-
| `pagination.cursor-agrees-with-page` | cursor walk and page walk yield the same set
|
|
1474
|
-
| `filter.unknown-field-rejected` | a filter on a field that does not exist is not silently ignored
|
|
1475
|
-
| `filter.equality-selects-exactly-one` | `id.eq.<one>` returns that one record
|
|
1476
|
-
| `filter.zero-match-returns-none` | a filter that matches nothing returns an empty page, not the whole set
|
|
1477
|
-
| `filter.negation-partitions-the-set` | `eq` ∪ `neq` = whole set, intersection empty
|
|
1478
|
-
| `filter.and-composes-as-intersection` | `and(A,B)` = A ∩ B
|
|
1479
|
-
| `filter.or-composes-as-union` | `or(A,B)` = A ∪ B
|
|
1480
|
-
| `filter.like-metacharacters-escaped` | `%` `_` `*` in a value are literals, not wildcards
|
|
1481
|
-
| `filter.numeric-comparison-is-numeric` | `gt`/`lt` on a number uses numeric order, not TEXT (`1,10,2`)
|
|
1482
|
-
| `filter.in-is-union-of-eq` | `in.(a,b)` = `eq.a` ∪ `eq.b`
|
|
1483
|
-
| `filter.nin-complements-in` | `in` ∩ `nin` empty; union = set minus nulls
|
|
1484
|
-
| `filter.gte-is-gt-or-eq` | `gte.x` = `gt.x` ∪ `eq.x`
|
|
1485
|
-
| `filter.lte-is-lt-or-eq` | `lte.x` = `lt.x` ∪ `eq.x`
|
|
1486
|
-
| `filter.ordered-triple-partitions` | `lt` ∪ `eq` ∪ `gt` = set minus nulls; pairwise disjoint
|
|
1487
|
-
| `filter.ilike-is-case-insensitive` | case-flipped value: `ilike` matches; `like` does not (unresolved if both match)
|
|
1488
|
-
| `filter.is-null-selects-nulls` | `is.null` = nulls; `is.notnull` = complement
|
|
1489
|
-
| `filter.contains-membership` | `contains.<el>` = records whose array value includes `el`
|
|
1490
|
-
| `filter.nested-and-or-distributes` | `and(A,or(B,C))` = (A∩B) ∪ (A∩C)
|
|
1491
|
-
| `filter.alias-matches-canonical` | each declared alias token selects the same id-set as its target
|
|
1492
|
-
| `filter.illegal-op-rejected` | one op **not** in that field's `ops` is 4xx
|
|
1493
|
-
| `filter.empty-in` | `in.()` is 4xx (`reject`) or zero rows (`match-none`)
|
|
1494
|
-
| `filter.in-over-limit-rejected` | `in` list of `maxInValues+1` is 4xx
|
|
1495
|
-
| `filter.condition-cap-rejected` | `maxFilterConditions+1` `eq` terms is 4xx
|
|
1496
|
-
| `spec.declared-filterable-ops-accepted` | every declared field × every op in that field's `ops` returns <400
|
|
1497
|
-
| `spec.declared-filterable-illegal-op-rejected` | one illegal op per declared field is 4xx
|
|
1498
|
-
| `error.malformed-filter-not-5xx` | garbage filter text is 4xx, never 5xx
|
|
1499
|
-
| `query.filter-selects-from-whole-set` | a filter is applied to the collection, not to the current page
|
|
1500
|
-
| `sort.order-is-applied` | requesting a sort actually rearranges the page
|
|
1501
|
-
| `sort.reverse-symmetry` | desc is the reverse of asc (nulls included)
|
|
1502
|
-
| `sort.unknown-field-rejected` | `order` on an undeclared field is 4xx, not silent ignore
|
|
1503
|
-
| `sort.numeric-order-is-numeric` | `1,10,2` sorts as numbers, not text
|
|
1504
|
-
| `sort.nulls-first-last` | `nullsfirst` / `nullslast` put nulls at the start / end of asc
|
|
1505
|
-
| `sort.multi-key-tiebreak` | ties on the first key are ordered by the second
|
|
1506
|
-
| `sort.default-order-applied` | omitting `order` matches `defaultOrder`
|
|
1507
|
-
| `sort.stable-tiebreak` | the same `order` twice yields the same sequence
|
|
1508
|
-
| `spec.declared-sortable-nulls-accepted` | each declared nulls token returns <400
|
|
1509
|
-
| `search.q-narrows-result` | a search term that matches one record does not return the whole set
|
|
1510
|
-
| `search.tokens-and` | `q=a b` = intersection (AND). Sameness of AND and OR passes
|
|
1511
|
-
| `search.case-insensitive` | case-flipped token matches the same set (unresolved if the backend is case-sensitive)
|
|
1512
|
-
| `search.empty-q` | `q=` is ignore / match-all / reject as declared
|
|
1513
|
-
| `search.undeclared-field-not-required` | extra recall on a non-searchable field is not `SEARCH_IGNORED`
|
|
1514
|
-
| `search.mode-accepted` | each declared `searchModes` value is <400 on the mode param
|
|
1515
|
-
| `search.modes-differ` | two modes may differ; exact sameness is unresolved, not a fail
|
|
1516
|
-
| `select.projection-honoured` | `select=id,name` does not return undeclared fields
|
|
1517
|
-
| `select.requested-fields-present` | every name in `select=` appears on each returned item
|
|
1518
|
-
| `select.unknown-field-rejected` | unknown select name is 4xx (`reject`) or dropped (`ignore`)
|
|
1519
|
-
| `select.nested-honoured` | `rel(col)` returns `rel` as an object/array carrying only `col`
|
|
1520
|
-
| `count.consistent-with-returned-page` | envelope total ≥ rows on this page, and is not zero when the page is not
|
|
1521
|
-
| `count.matches-filtered-set` | filtered total equals the size of the filtered walk
|
|
1522
|
-
| `query.axes-compose` | filter + sort together: filter still holds on the sorted page
|
|
1523
|
-
| `query.filter-and-select-compose` | filter + select together
|
|
1524
|
-
| `query.search-and-filter-compose` | search + filter together
|
|
1525
|
-
| `query.filter-sort-select-compose` | filter + sort + select
|
|
1526
|
-
| `query.filter-search-sort-compose` | filter + search + sort
|
|
1527
|
-
| `query.filter-search-select-compose` | filter + search + select
|
|
1528
|
-
| `query.sort-and-select-compose` | sort + select: order holds; extras dropped; requested fields present
|
|
1529
|
-
| `query.search-and-select-compose` | search + select: search membership holds; extras dropped
|
|
1530
|
-
| `query.search-and-sort-compose` | search + sort: search membership holds; remaining rows ordered
|
|
1531
|
-
| `query.filter-search-sort-select-compose` | all four: filter ∩ search; order holds; extras dropped
|
|
1532
|
-
| `spec.declared-filterable-is-filterable` | every `x-query.filterable` field actually accepts a filter
|
|
1533
|
-
| `spec.declared-sortable-is-sortable` | every `x-query.sortable` field actually accepts a sort
|
|
1534
|
-
| `spec.declared-selectable-is-selectable` | every `x-query.selectable` field actually accepts a select
|
|
1535
|
-
| `tenant.item-not-readable-cross-tenant` | principal B cannot GET principal A's item
|
|
1536
|
-
| `tenant.denial-does-not-reveal-existence` | 404 vs 403 (or equivalent) does not distinguish "exists other tenant" from "missing"
|
|
1537
|
-
| `tenant.filter-does-not-bypass-scope` | `filter=id.eq.<other tenant>` does not return that row
|
|
1538
|
-
| `auth.rank-is-monotonic` | a lower rank cannot do what a higher rank is denied
|
|
1539
|
-
| `auth.invite-grants-then-revokes` | invite → accept grants; revoke takes it back
|
|
1540
|
-
| `patch.immutable-field-rejected` | PATCHing an `x-immutable` field is rejected or ignored
|
|
1541
|
-
| `softdelete.absent-from-default-list` | a soft-deleted row is gone from the default list
|
|
1542
|
-
| `invalidation.declared-route-changes` | after a write, the other entity's listed route actually changes
|
|
1543
|
-
| `effects.declared-effect-occurs` | `x-effects` cardinality delta is observed on the named list
|
|
1544
|
-
| `effects.side-effect-arrives` | after the write, the named GET’s JSON path is occupied before `timeoutMs`
|
|
1545
|
-
| `async.reaches-terminal-state` | `x-async` reaches `until` (poll, or a terminal SSE frame) before `timeoutMs`
|
|
1546
|
-
| `async.receipt-identifies-the-job` | `idFrom` on the receipt (JSON object or SSE event JSON) resolves to a pollable job
|
|
1547
|
-
| `patch.minimality` | PATCH `{ name }` does not clear other writable fields
|
|
1548
|
-
| `idempotency.replay-does-not-duplicate` | same Idempotency-Key + same body does not create a second row
|
|
1549
|
-
| `delete.absent-record-returns-404` | DELETE of a missing id is 404, not 200
|
|
1550
|
-
| `concurrency.no-lost-update` | two PATCHes to different fields do not clobber each other
|
|
1551
|
-
| `validation.enum-enforced` | a value outside the enum is rejected
|
|
1552
|
-
| `validation.max-length-enforced` | a string over `maxLength` is rejected
|
|
1553
|
-
| `validation.required-enforced` | omitting a required field is rejected
|
|
1554
|
-
| `validation.content-type-enforced` | a wrong Content-Type is 415 when 415 is documented
|
|
1555
|
-
| `consistency.projections-agree` | list, item, and filtered views of the same field agree
|
|
1476
|
+
| id | asserts | needs | depends |
|
|
1477
|
+
| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------- | --------------------------------------- |
|
|
1478
|
+
| `list.read-after-write` | a just-created record appears on the list | create + a seeded record | — |
|
|
1479
|
+
| `create.persists-submitted-fields` | every writable field sent on create is echoed | create that echoes the record | `list.read-after-write` |
|
|
1480
|
+
| `payload.string-survives` | a documented-valid string is stored exactly; 4xx after an ASCII control is a fail | update or create+delete, item GET, unconstrained string | `list.read-after-write` |
|
|
1481
|
+
| `create.status-matches-document` | create status is one the document declared | create | — |
|
|
1482
|
+
| `response.status-is-documented` | every non-create exchange returns a status that operation names (`default` ≠ 201) | a modeled non-create operation oat invoked | — |
|
|
1483
|
+
| `schema.success-response-matches-document` | create body validates against the success schema | success schema on create | `create.status-matches-document` |
|
|
1484
|
+
| `schema.error-response-matches-document` | an error body validates against the documented error schema | error schema on the item route | — |
|
|
1485
|
+
| `pagination.limit-bounds-page-size` | page size ≤ the requested limit | page-size _role_ (aliases include `limit`, `per_page`, …) | `list.read-after-write` |
|
|
1486
|
+
| `pagination.limit-respects-documented-max` | requesting more than `maxLimit` does not return more | declared `maxLimit` and a larger cohort | `pagination.limit-bounds-page-size` |
|
|
1487
|
+
| `pagination.has-more-is-accurate` | `hasMore` / `Link rel=next` matches whether another page exists | page-forward + `hasMore` or `Link rel=next` | `pagination.limit-bounds-page-size` |
|
|
1488
|
+
| `pagination.page-walk-covers-set` | walking pages covers the collection with no gaps or dupes | page or offset + ≥3 records | `pagination.limit-bounds-page-size` |
|
|
1489
|
+
| `pagination.cursor-agrees-with-page` | cursor walk and page walk yield the same set | both cursor and page | `pagination.limit-bounds-page-size` |
|
|
1490
|
+
| `filter.unknown-field-rejected` | a filter on a field that does not exist is not silently ignored | a filter expression | — |
|
|
1491
|
+
| `filter.equality-selects-exactly-one` | `id.eq.<one>` returns that one record | equality on the identity | `list.read-after-write` |
|
|
1492
|
+
| `filter.zero-match-returns-none` | a filter that matches nothing returns an empty page, not the whole set | same | `list.read-after-write` |
|
|
1493
|
+
| `filter.negation-partitions-the-set` | `eq` ∪ `neq` = whole set, intersection empty | eq and neq | `list.read-after-write`, equality |
|
|
1494
|
+
| `filter.and-composes-as-intersection` | `and(A,B)` = A ∩ B | two filterable fields + AND | equality / list |
|
|
1495
|
+
| `filter.or-composes-as-union` | `or(A,B)` = A ∪ B | `or()` — **postgrest grammar only** | equality / list |
|
|
1496
|
+
| `filter.like-metacharacters-escaped` | `%` `_` `*` in a value are literals, not wildcards | like operator | `list.read-after-write` |
|
|
1497
|
+
| `filter.numeric-comparison-is-numeric` | `gt`/`lt` on a number uses numeric order, not TEXT (`1,10,2`) | numeric field + a filter param | `list.read-after-write`, unknown-field |
|
|
1498
|
+
| `filter.in-is-union-of-eq` | `in.(a,b)` = `eq.a` ∪ `eq.b` | field allows `in`; ≥2 distinct values | equality |
|
|
1499
|
+
| `filter.nin-complements-in` | `in` ∩ `nin` empty; union = set minus nulls | field allows `in` and `nin` | `in` |
|
|
1500
|
+
| `filter.gte-is-gt-or-eq` | `gte.x` = `gt.x` ∪ `eq.x` | field allows `gte` and `gt`; ordered type | numeric comparison |
|
|
1501
|
+
| `filter.lte-is-lt-or-eq` | `lte.x` = `lt.x` ∪ `eq.x` | field allows `lte` and `lt` | numeric comparison |
|
|
1502
|
+
| `filter.ordered-triple-partitions` | `lt` ∪ `eq` ∪ `gt` = set minus nulls; pairwise disjoint | field allows all three | numeric comparison |
|
|
1503
|
+
| `filter.ilike-is-case-insensitive` | case-flipped value: `ilike` matches; `like` does not (unresolved if both match) | field allows both | like |
|
|
1504
|
+
| `filter.is-null-selects-nulls` | `is.null` = nulls; `is.notnull` = complement | field allows `is`; cohort has a null | foundations |
|
|
1505
|
+
| `filter.contains-membership` | `contains.<el>` = records whose array value includes `el` | array field or ops include `contains` | foundations |
|
|
1506
|
+
| `filter.nested-and-or-distributes` | `and(A,or(B,C))` = (A∩B) ∪ (A∩C) | postgrest grammar; ≥2 filterable fields | `and` / `or` |
|
|
1507
|
+
| `filter.alias-matches-canonical` | each declared alias token selects the same id-set as its target | `aliases` non-empty | equality |
|
|
1508
|
+
| `filter.illegal-op-rejected` | one op **not** in that field's `ops` is 4xx | closed `ops` list | unknown-field |
|
|
1509
|
+
| `filter.empty-in` | `in.()` is 4xx (`reject`) or zero rows (`match-none`) | `emptyIn` set; field allows `in` | `in` |
|
|
1510
|
+
| `filter.in-over-limit-rejected` | `in` list of `maxInValues+1` is 4xx | `maxInValues` set | foundations |
|
|
1511
|
+
| `filter.condition-cap-rejected` | `maxFilterConditions+1` `eq` terms is 4xx | `maxFilterConditions` set | foundations |
|
|
1512
|
+
| `spec.declared-filterable-ops-accepted` | every declared field × every op in that field's `ops` returns <400 | closed `ops` | declared-filterable |
|
|
1513
|
+
| `spec.declared-filterable-illegal-op-rejected` | one illegal op per declared field is 4xx | closed `ops` | illegal-op |
|
|
1514
|
+
| `error.malformed-filter-not-5xx` | garbage filter text is 4xx, never 5xx | a filter expression | — |
|
|
1515
|
+
| `query.filter-selects-from-whole-set` | a filter is applied to the collection, not to the current page | filterable + ≥3 records | list / walk |
|
|
1516
|
+
| `sort.order-is-applied` | requesting a sort actually rearranges the page | order + a sortable field | `pagination.limit-bounds-page-size` |
|
|
1517
|
+
| `sort.reverse-symmetry` | desc is the reverse of asc (nulls included) | order + asc/desc | order-is-applied |
|
|
1518
|
+
| `sort.unknown-field-rejected` | `order` on an undeclared field is 4xx, not silent ignore | order param | — |
|
|
1519
|
+
| `sort.numeric-order-is-numeric` | `1,10,2` sorts as numbers, not text | numeric sortable field whose lexical order disagrees | order-is-applied |
|
|
1520
|
+
| `sort.nulls-first-last` | `nullsfirst` / `nullslast` put nulls at the start / end of asc | declared nulls token; dotted grammar; cohort has a null | order + reverse |
|
|
1521
|
+
| `sort.multi-key-tiebreak` | ties on the first key are ordered by the second | ≥2 sortable fields; `maxKeys` absent or ≥2 | order + reverse |
|
|
1522
|
+
| `sort.default-order-applied` | omitting `order` matches `defaultOrder` | `defaultOrder` set; walk complete | order + walk |
|
|
1523
|
+
| `sort.stable-tiebreak` | the same `order` twice yields the same sequence | `stableTiebreak` set | order-is-applied |
|
|
1524
|
+
| `spec.declared-sortable-nulls-accepted` | each declared nulls token returns <400 | some field/global declares `nulls` | order-is-applied |
|
|
1525
|
+
| `search.q-narrows-result` | a search term that matches one record does not return the whole set | search param + searchable fields | `list.read-after-write` |
|
|
1526
|
+
| `search.tokens-and` | `q=a b` = intersection (AND). Sameness of AND and OR passes | searchable fields; two tokens that split the cohort | `q-narrows` |
|
|
1527
|
+
| `search.case-insensitive` | case-flipped token matches the same set (unresolved if the backend is case-sensitive) | searchable field with a letter | `q-narrows` |
|
|
1528
|
+
| `search.empty-q` | `q=` is ignore / match-all / reject as declared | `searchEmpty` set | `q-narrows` |
|
|
1529
|
+
| `search.undeclared-field-not-required` | extra recall on a non-searchable field is not `SEARCH_IGNORED` | ≥1 searchable and ≥1 non-searchable string | `q-narrows` |
|
|
1530
|
+
| `search.mode-accepted` | each declared `searchModes` value is <400 on the mode param | `searchModes` set and a mode role | — |
|
|
1531
|
+
| `search.modes-differ` | two modes may differ; exact sameness is unresolved, not a fail | ≥2 `searchModes`; a mode param | `mode-accepted` |
|
|
1532
|
+
| `select.projection-honoured` | `select=id,name` does not return undeclared fields | select param | — |
|
|
1533
|
+
| `select.requested-fields-present` | every name in `select=` appears on each returned item | select param; ≥1 selectable field | projection |
|
|
1534
|
+
| `select.unknown-field-rejected` | unknown select name is 4xx (`reject`) or dropped (`ignore`) | `select.unknown` set | projection |
|
|
1535
|
+
| `select.nested-honoured` | `rel(col)` returns `rel` as an object/array carrying only `col` | `select.nested` and a named relation | projection |
|
|
1536
|
+
| `count.consistent-with-returned-page` | envelope total ≥ rows on this page, and is not zero when the page is not | envelope total | `list.read-after-write` |
|
|
1537
|
+
| `count.matches-filtered-set` | filtered total equals the size of the filtered walk | total + a filter | list, equality |
|
|
1538
|
+
| `query.axes-compose` | filter + sort together: filter still holds on the sorted page | filterable + sortable | filter + sort foundations |
|
|
1539
|
+
| `query.filter-and-select-compose` | filter + select together | filterable + select | same |
|
|
1540
|
+
| `query.search-and-filter-compose` | search + filter together | filterable + search | same |
|
|
1541
|
+
| `query.filter-sort-select-compose` | filter + sort + select | filter + sort + select | same |
|
|
1542
|
+
| `query.filter-search-sort-compose` | filter + search + sort | filter + search + sort | same |
|
|
1543
|
+
| `query.filter-search-select-compose` | filter + search + select | filter + search + select | same |
|
|
1544
|
+
| `query.sort-and-select-compose` | sort + select: order holds; extras dropped; requested fields present | sortable + selectable | sort + select |
|
|
1545
|
+
| `query.search-and-select-compose` | search + select: search membership holds; extras dropped | searchable + selectable | search + select |
|
|
1546
|
+
| `query.search-and-sort-compose` | search + sort: search membership holds; remaining rows ordered | searchable + sortable | search + sort |
|
|
1547
|
+
| `query.filter-search-sort-select-compose` | all four: filter ∩ search; order holds; extras dropped | all four axes declared | the triples |
|
|
1548
|
+
| `spec.declared-filterable-is-filterable` | every `x-query.filterable` field actually accepts a filter | `x-query` naming filterable fields | filter foundations |
|
|
1549
|
+
| `spec.declared-sortable-is-sortable` | every `x-query.sortable` field actually accepts a sort | `x-query` naming sortable fields | sort foundations |
|
|
1550
|
+
| `spec.declared-selectable-is-selectable` | every `x-query.selectable` field actually accepts a select | `x-query` naming selectable fields | select |
|
|
1551
|
+
| `tenant.item-not-readable-cross-tenant` | principal B cannot GET principal A's item | second principal, different `roots`, and a tagged or inferred tenant | — |
|
|
1552
|
+
| `tenant.denial-does-not-reveal-existence` | 404 vs 403 (or equivalent) does not distinguish "exists other tenant" from "missing" | second principal, and a tagged or inferred tenant | `tenant.item-not-readable-cross-tenant` |
|
|
1553
|
+
| `tenant.filter-does-not-bypass-scope` | `filter=id.eq.<other tenant>` does not return that row | second principal, a filter, and a tagged or inferred tenant | `query.filter-selects-from-whole-set` |
|
|
1554
|
+
| `auth.rank-is-monotonic` | a lower rank cannot do what a higher rank is denied | two same-tenant principals at different `rank` | `list.read-after-write` |
|
|
1555
|
+
| `auth.invite-grants-then-revokes` | invite → accept grants; revoke takes it back | `x-invite` + peer with `inviteAs` | list, cross-tenant |
|
|
1556
|
+
| `patch.immutable-field-rejected` | PATCHing an `x-immutable` field is rejected or ignored | `x-immutable` | — |
|
|
1557
|
+
| `softdelete.absent-from-default-list` | a soft-deleted row is gone from the default list | `x-soft-delete` | `list.read-after-write` |
|
|
1558
|
+
| `invalidation.declared-route-changes` | after a write, the other entity's listed route actually changes | `x-invalidate` naming another entity | list, persist |
|
|
1559
|
+
| `effects.declared-effect-occurs` | `x-effects` cardinality delta is observed on the named list (`count` exact, `min` at-least; nested child lists bind the created parent id) | `x-effects` | `list.read-after-write` |
|
|
1560
|
+
| `effects.side-effect-arrives` | after the write, the named GET’s JSON path is occupied before `timeoutMs` | `x-wait` | `list.read-after-write` |
|
|
1561
|
+
| `async.reaches-terminal-state` | `x-async` reaches `until` (poll, or a terminal SSE frame) before `timeoutMs` | `x-async` | — |
|
|
1562
|
+
| `async.receipt-identifies-the-job` | `idFrom` on the receipt (JSON object or SSE event JSON) resolves to a pollable job | `x-async` + `idFrom` | — |
|
|
1563
|
+
| `patch.minimality` | PATCH `{ name }` does not clear other writable fields | update + item route | — |
|
|
1564
|
+
| `idempotency.replay-does-not-duplicate` | same Idempotency-Key + same body does not create a second row | create + documented Idempotency-Key header | list, persist |
|
|
1565
|
+
| `delete.absent-record-returns-404` | DELETE of a missing id is 404, not 200 | delete | — |
|
|
1566
|
+
| `concurrency.no-lost-update` | two PATCHes to different fields do not clobber each other | update + two writable strings | persist + patch |
|
|
1567
|
+
| `validation.enum-enforced` | a value outside the enum is rejected | enum in the request schema | — |
|
|
1568
|
+
| `validation.max-length-enforced` | a string over `maxLength` is rejected | maxLength | — |
|
|
1569
|
+
| `validation.required-enforced` | omitting a required field is rejected | required field | — |
|
|
1570
|
+
| `validation.content-type-enforced` | a wrong Content-Type is 415 when 415 is documented | documented 415 | — |
|
|
1571
|
+
| `consistency.projections-agree` | list, item, and filtered views of the same field agree | item route + a comparable field | list + persist + filter |
|
|
1556
1572
|
|
|
1557
1573
|
On a typical untagged CRUD document (create, list, item, `page`/`limit`, maybe `sort`):
|
|
1558
1574
|
|
|
@@ -1635,19 +1651,31 @@ Written under `--out` (default `./.oat/runs`). Each invocation creates a UTC tim
|
|
|
1635
1651
|
|
|
1636
1652
|
`--out` / `outDir` replace the root, not the leaf — `--out .oat/runs/prod` writes `.oat/runs/prod/<datetime>/`.
|
|
1637
1653
|
|
|
1638
|
-
| file
|
|
1639
|
-
|
|
|
1640
|
-
| `oat-report.md`
|
|
1641
|
-
| `oat-report.json`
|
|
1642
|
-
| `matrix.html`
|
|
1643
|
-
| `matrix.json`
|
|
1644
|
-
| `issue-repro/*.sh`
|
|
1645
|
-
| `progress.log`
|
|
1646
|
-
| `progress.jsonl`
|
|
1647
|
-
| `progress.tsv`
|
|
1648
|
-
| `progress.json`
|
|
1654
|
+
| file | |
|
|
1655
|
+
| ---------------------------- | ------------------------------------------------------------------------------------------------------ |
|
|
1656
|
+
| `oat-report.md` | human report: summary, findings with request/response excerpts, coverage, latency p50/p95/max |
|
|
1657
|
+
| `oat-report.json` | same data for CI. Summary only — full exchanges are not inlined here |
|
|
1658
|
+
| `matrix.html` | visual matrix of entities × checks |
|
|
1659
|
+
| `matrix.json` | the same graph (AI-friendly), including a mermaid string |
|
|
1660
|
+
| `issue-repro/*.sh` | one executable `curl` script per finding that has evidence. Directory is omitted when the run is clean |
|
|
1661
|
+
| `progress.log` | logfmt, one event per line, never truncated |
|
|
1662
|
+
| `progress.jsonl` | same events as JSON |
|
|
1663
|
+
| `progress.tsv` | same columns, tab-separated. `req_id` is the join key to the exchange journal |
|
|
1664
|
+
| `progress.json` | latest snapshot only (overwritten ~1s) |
|
|
1665
|
+
| `exchanges.jsonl` | one line per request (`seq`, `requestId`, method, url, status, bytes, …). Greppable |
|
|
1666
|
+
| `exchanges/<requestId>.json` | full exchange: status, headers, described bodies. Missing id → `seq-<n>.json`. Duplicate id → `-<seq>` |
|
|
1667
|
+
| `blobs/<sha256>` | content-addressed file parts and oversized / binary bodies |
|
|
1668
|
+
|
|
1669
|
+
The journal is oat `Exchange` JSON, not HAR (HAR export is out of scope). Default **on** unless `--profile cheap`. `--no-save-exchanges` skips `exchanges/` entirely. `--quiet` does not.
|
|
1670
|
+
|
|
1671
|
+
Bodies are described, never base64 multipart. JSON / text inline up to 256 KiB, then `blobs/<sha256>`. `FormData` file parts and binary downloads (`application/pdf`, spreadsheet, image, `octet-stream`) are always blobs; the same fixture bytes POSTed N times share one file. SSE is stored as parsed `{ event, data }[]` frames.
|
|
1672
|
+
|
|
1673
|
+
Redaction is on by default (not opt-in), applied on write:
|
|
1674
|
+
|
|
1675
|
+
- Headers: `authorization`, `cookie`, `set-cookie`, `proxy-authorization`, and any `x-*-key` / `x-*-secret` / `x-ia-tester-key` (case-insensitive).
|
|
1676
|
+
- JSON bodies, any depth: `access_token`, `refresh_token`, `id_token`, `password`, `token`, `secret`, `api_key` → `"<redacted>"`.
|
|
1649
1677
|
|
|
1650
|
-
|
|
1678
|
+
Join `progress.tsv` `req_id` to `exchanges.jsonl` `requestId` (response `x-request-id` / `request-id` / `x-correlation-id` / `correlation-id` wins; else what oat sent). `oat-report.md` includes one line such as `1841 exchanges → exchanges/`.
|
|
1651
1679
|
|
|
1652
1680
|
### `oat-report.json`
|
|
1653
1681
|
|
|
@@ -1949,8 +1977,9 @@ oat run --config labs/local.config.ts --base-url <url>
|
|
|
1949
1977
|
|
|
1950
1978
|
These are deliberate. An agent should not invent a flag for them.
|
|
1951
1979
|
|
|
1952
|
-
- **No request timeout.** `fetch` waits until the server answers. Watch `status=in_flight` and `idle_ms` on that request.
|
|
1953
|
-
- **No retry on 5xx
|
|
1980
|
+
- **No request timeout by default.** `fetch` waits until the server answers unless `network.requestTimeoutMs` is set. Watch `status=in_flight` and `idle_ms` on that request.
|
|
1981
|
+
- **No retry on 5xx.** 429 is retried (up to 5, honouring `Retry-After`). One 401 → force refresh + single retry. A second 401 is evidence.
|
|
1982
|
+
- **Network throws are not HTTP.** Offline / DNS / reset / timeout: 4 retries, then one wait (default 60s) for the link. If it does not come back, `net.unreachable` is recorded, remaining work stands down, the report is still written, exit `1`. Progress `status=network`. Failed attempts are journaled as `status: 0` with `{ error: "network", kind }`.
|
|
1954
1983
|
- **No OpenAPI `security`.** Put credentials in `principals`. Cookie auth is a `headers: { cookie: "…" }` (or a flow that sets that header).
|
|
1955
1984
|
- **No `servers[]`.** Always set `baseUrl`. Extra hosts are `origins[]`, each with its own `spec`.
|
|
1956
1985
|
- **No OCR.** Multipart and file parts are sent as dummy / pool / `each` / `resolveUpload` bytes. oat checks HTTP status and JSON responses, not whether a PDF is a real invoice.
|
package/dist/cli.js
CHANGED
|
@@ -8,6 +8,7 @@ import { report } from "./report/console.js";
|
|
|
8
8
|
import { renderMatrixGraph, renderMatrixHtml } from "./report/matrix.js";
|
|
9
9
|
import { ISSUE_REPRO_DIR, renderConsole, renderJson, renderMarkdown, renderRepros } from "./report/render.js";
|
|
10
10
|
import { createProgressPump, createStderrProgress, formatProgressJsonl, formatProgressLine, formatProgressTsv, PROGRESS_GLOSSARY, PROGRESS_TSV_HEADER, } from "./runtime/progress.js";
|
|
11
|
+
import { resolveSaveExchanges } from "./runtime/exchanges.js";
|
|
11
12
|
import { allocateRunDir, DEFAULT_RUNS_ROOT } from "./runtime/runs.js";
|
|
12
13
|
import { renderTeardown } from "./runtime/teardown.js";
|
|
13
14
|
import { buildModel } from "./spec/graph.js";
|
|
@@ -32,6 +33,8 @@ export const KNOWN_FLAGS = new Set([
|
|
|
32
33
|
"max-defects",
|
|
33
34
|
"json",
|
|
34
35
|
"parser",
|
|
36
|
+
"save-exchanges",
|
|
37
|
+
"no-save-exchanges",
|
|
35
38
|
]);
|
|
36
39
|
export function parseArgs(argv) {
|
|
37
40
|
const [command = "help", ...rest] = argv;
|
|
@@ -78,6 +81,8 @@ Flags
|
|
|
78
81
|
--keep-fixtures leave created records in place instead of tearing them down
|
|
79
82
|
--max-in-flight requests allowed in flight at once (default 4)
|
|
80
83
|
--quiet no live progress on stderr (progress.log still written)
|
|
84
|
+
--save-exchanges persist every HTTP exchange under the run dir (default unless --profile cheap)
|
|
85
|
+
--no-save-exchanges skip the exchange journal (cheap CI)
|
|
81
86
|
--untagged serve (or test) a document with every x-* tag stripped
|
|
82
87
|
--backend conformance storage: memory | sqlite | postgres | d1
|
|
83
88
|
(default: all local; d1 is remote and opt-in)
|
|
@@ -108,6 +113,11 @@ async function commandRun(flags) {
|
|
|
108
113
|
.filter(Boolean);
|
|
109
114
|
const only = onlyFromFlag ?? config.only?.filter((name) => name.trim() !== "");
|
|
110
115
|
const profile = str(flags, "profile") ?? config.profile;
|
|
116
|
+
const saveExchanges = resolveSaveExchanges({
|
|
117
|
+
...(flags["no-save-exchanges"] === true ? { flag: false } : flags["save-exchanges"] === true ? { flag: true } : {}),
|
|
118
|
+
...(config.saveExchanges === undefined ? {} : { config: config.saveExchanges }),
|
|
119
|
+
...(profile === undefined ? {} : { profile }),
|
|
120
|
+
});
|
|
111
121
|
/* No cast: the config's principal type *is* the runtime's, so a mistake here is a compile
|
|
112
122
|
* error in the user's own config rather than a surprise mid-run. */
|
|
113
123
|
const principals = config.principals ?? [];
|
|
@@ -170,6 +180,8 @@ async function commandRun(flags) {
|
|
|
170
180
|
maxInFlight: Number.parseInt(str(flags, "max-in-flight") ?? "", 10) || config.maxInFlight || 4,
|
|
171
181
|
seed: seedFlag === undefined ? (config.seed ?? 1) : Number.parseInt(seedFlag, 10),
|
|
172
182
|
onProgress,
|
|
183
|
+
...(saveExchanges ? { exchangeDir: outDir } : {}),
|
|
184
|
+
...(config.network === undefined ? {} : { network: config.network }),
|
|
173
185
|
});
|
|
174
186
|
}
|
|
175
187
|
finally {
|
|
@@ -194,6 +206,8 @@ async function commandRun(flags) {
|
|
|
194
206
|
profile: result.profile,
|
|
195
207
|
profileExclusions: result.profileExclusions,
|
|
196
208
|
startedAt,
|
|
209
|
+
...(result.exchanges === undefined ? {} : { exchanges: result.exchanges }),
|
|
210
|
+
...(result.network === undefined ? {} : { network: result.network }),
|
|
197
211
|
};
|
|
198
212
|
await writeFile(resolve(outDir, "principals.json"), `${JSON.stringify({ principals: result.principals }, null, 2)}\n`);
|
|
199
213
|
await writeFile(resolve(outDir, "oat-report.md"), renderMarkdown(input));
|
|
@@ -216,9 +230,17 @@ async function commandRun(flags) {
|
|
|
216
230
|
process.stdout.write(` matrix: ${resolve(outDir, "matrix.html")}\n`);
|
|
217
231
|
process.stdout.write(` graph: ${resolve(outDir, "matrix.json")}\n`);
|
|
218
232
|
process.stdout.write(` progress: ${resolve(outDir, "progress.log")} · ${resolve(outDir, "progress.jsonl")}\n`);
|
|
233
|
+
if (result.exchanges !== undefined) {
|
|
234
|
+
process.stdout.write(` exchanges: ${result.exchanges.count} → ${resolve(outDir, "exchanges")}\n`);
|
|
235
|
+
}
|
|
236
|
+
if (result.network?.incomplete === true) {
|
|
237
|
+
process.stdout.write(` network: ${result.network.kind} — run incomplete\n`);
|
|
238
|
+
}
|
|
219
239
|
process.stdout.write(` latest: ${allocated.latest}\n\n`);
|
|
220
240
|
/* Exit code counts root causes, not raw findings: gaps and blocked entries are information,
|
|
221
241
|
* not failures, and a CI gate should react to defects only. */
|
|
242
|
+
if (result.network?.incomplete === true)
|
|
243
|
+
return 1;
|
|
222
244
|
return result.findings.filter((f) => f.verdict !== "COVERAGE_GAP" && f.verdict !== "BLOCKED").length > 0 ? 1 : 0;
|
|
223
245
|
}
|
|
224
246
|
export async function main() {
|
|
@@ -233,7 +255,7 @@ export async function main() {
|
|
|
233
255
|
return 0;
|
|
234
256
|
}
|
|
235
257
|
if (command === "conformance") {
|
|
236
|
-
const { postgresAvailable, renderParserSuite, renderSuite, runExampleSpecSuite, runTagUnlockSuite, runParserSuite, runPayloadCatalogSuite, runCoverageReportSuite, runSeedContractSuite, runTenantScopeSuite, runSuite, sqliteAvailable, d1Available, } = await import("./conformance/suite.js");
|
|
258
|
+
const { postgresAvailable, renderParserSuite, renderSuite, runExampleSpecSuite, runTagUnlockSuite, runParserSuite, runPayloadCatalogSuite, runCoverageReportSuite, runSeedContractSuite, runEffectsSuite, runTenantScopeSuite, runSuite, sqliteAvailable, d1Available, } = await import("./conformance/suite.js");
|
|
237
259
|
const { runFeatureGateSuite } = await import("./conformance/feature-gate.js");
|
|
238
260
|
const { runRateLimitSuite } = await import("./conformance/rate-limit.js");
|
|
239
261
|
const parser = renderParserSuite(runParserSuite());
|
|
@@ -262,6 +284,9 @@ export async function main() {
|
|
|
262
284
|
const seedContract = renderParserSuite(await runSeedContractSuite());
|
|
263
285
|
process.stdout.write(seedContract.text);
|
|
264
286
|
parser.failures += seedContract.failures;
|
|
287
|
+
const effects = renderParserSuite(await runEffectsSuite());
|
|
288
|
+
process.stdout.write(effects.text);
|
|
289
|
+
parser.failures += effects.failures;
|
|
265
290
|
const payloads = renderParserSuite(runPayloadCatalogSuite());
|
|
266
291
|
process.stdout.write(payloads.text);
|
|
267
292
|
parser.failures += payloads.failures;
|