@lovrozagar/oat 0.9.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -1936
- package/dist/cli.js +326 -60
- package/dist/config/define-config.d.ts +15 -1
- package/dist/config/define-config.d.ts.map +1 -1
- package/dist/config/define-config.js.map +1 -1
- package/dist/config/validate.d.ts +12 -0
- package/dist/config/validate.d.ts.map +1 -0
- package/dist/config/validate.js +86 -0
- package/dist/config/validate.js.map +1 -0
- package/dist/conformance/effects.d.ts +9 -0
- package/dist/conformance/effects.d.ts.map +1 -0
- package/dist/conformance/effects.js +276 -0
- package/dist/conformance/effects.js.map +1 -0
- package/dist/conformance/expected-failures.d.ts +18 -0
- package/dist/conformance/expected-failures.d.ts.map +1 -0
- package/dist/conformance/expected-failures.js +48 -0
- package/dist/conformance/expected-failures.js.map +1 -0
- package/dist/conformance/feature-gate.d.ts +7 -0
- package/dist/conformance/feature-gate.d.ts.map +1 -0
- package/dist/conformance/feature-gate.js +428 -0
- package/dist/conformance/feature-gate.js.map +1 -0
- package/dist/conformance/fuzz.d.ts +108 -0
- package/dist/conformance/fuzz.d.ts.map +1 -0
- package/dist/conformance/fuzz.js +246 -0
- package/dist/conformance/fuzz.js.map +1 -0
- package/dist/conformance/leaks.d.ts +10 -0
- package/dist/conformance/leaks.d.ts.map +1 -0
- package/dist/conformance/leaks.js +23 -0
- package/dist/conformance/leaks.js.map +1 -0
- package/dist/conformance/pool.d.ts +49 -0
- package/dist/conformance/pool.d.ts.map +1 -0
- package/dist/conformance/pool.js +75 -0
- package/dist/conformance/pool.js.map +1 -0
- package/dist/conformance/rate-limit.d.ts +7 -0
- package/dist/conformance/rate-limit.d.ts.map +1 -0
- package/dist/conformance/rate-limit.js +372 -0
- package/dist/conformance/rate-limit.js.map +1 -0
- package/dist/conformance/scope.d.ts +15 -0
- package/dist/conformance/scope.d.ts.map +1 -0
- package/dist/conformance/scope.js +78 -0
- package/dist/conformance/scope.js.map +1 -0
- package/dist/conformance/seed-contract.d.ts +11 -0
- package/dist/conformance/seed-contract.d.ts.map +1 -0
- package/dist/conformance/seed-contract.js +680 -0
- package/dist/conformance/seed-contract.js.map +1 -0
- package/dist/conformance/shapes.d.ts +35 -0
- package/dist/conformance/shapes.d.ts.map +1 -0
- package/dist/conformance/shapes.js +162 -0
- package/dist/conformance/shapes.js.map +1 -0
- package/dist/conformance/specs.d.ts +21 -0
- package/dist/conformance/specs.d.ts.map +1 -0
- package/dist/conformance/specs.js +306 -0
- package/dist/conformance/specs.js.map +1 -0
- package/dist/conformance/suite.d.ts +209 -0
- package/dist/conformance/suite.d.ts.map +1 -0
- package/dist/conformance/suite.js +1538 -0
- package/dist/conformance/suite.js.map +1 -0
- package/dist/conformance/unique.d.ts +7 -0
- package/dist/conformance/unique.d.ts.map +1 -0
- package/dist/conformance/unique.js +586 -0
- package/dist/conformance/unique.js.map +1 -0
- package/dist/conformance/worker.d.ts +3 -0
- package/dist/conformance/worker.d.ts.map +1 -0
- package/dist/conformance/worker.js +27 -0
- package/dist/conformance/worker.js.map +1 -0
- package/dist/index.d.ts +10 -3
- package/dist/index.js +6 -2
- package/dist/reference/defects.d.ts +71 -1
- package/dist/reference/defects.d.ts.map +1 -1
- package/dist/reference/defects.js +74 -8
- package/dist/reference/defects.js.map +1 -1
- package/dist/reference/dialect.d.ts +6 -4
- package/dist/reference/dialect.d.ts.map +1 -1
- package/dist/reference/dialect.js +1 -0
- package/dist/reference/dialect.js.map +1 -1
- package/dist/reference/http.d.ts +17 -25
- package/dist/reference/http.d.ts.map +1 -1
- package/dist/reference/http.js +574 -224
- package/dist/reference/http.js.map +1 -1
- package/dist/reference/listing.d.ts +51 -0
- package/dist/reference/listing.d.ts.map +1 -0
- package/dist/reference/listing.js +256 -0
- package/dist/reference/listing.js.map +1 -0
- package/dist/reference/model.d.ts +35 -1
- package/dist/reference/model.d.ts.map +1 -1
- package/dist/reference/model.js +42 -0
- package/dist/reference/model.js.map +1 -1
- package/dist/reference/query.d.ts +132 -43
- package/dist/reference/query.d.ts.map +1 -1
- package/dist/reference/query.js +397 -315
- package/dist/reference/query.js.map +1 -1
- package/dist/reference/shapes.d.ts +60 -0
- package/dist/reference/shapes.d.ts.map +1 -0
- package/dist/reference/shapes.js +142 -0
- package/dist/reference/shapes.js.map +1 -0
- package/dist/reference/spec.d.ts +9 -2
- package/dist/reference/spec.d.ts.map +1 -1
- package/dist/reference/spec.js +291 -71
- package/dist/reference/spec.js.map +1 -1
- package/dist/reference/store-api.d.ts +14 -43
- package/dist/reference/store-api.d.ts.map +1 -1
- package/dist/reference/store-api.js +8 -58
- package/dist/reference/store-api.js.map +1 -1
- package/dist/reference/stores/memory.d.ts +12 -15
- package/dist/reference/stores/memory.d.ts.map +1 -1
- package/dist/reference/stores/memory.js +22 -70
- package/dist/reference/stores/memory.js.map +1 -1
- package/dist/reference/stores/postgres.d.ts +11 -23
- package/dist/reference/stores/postgres.d.ts.map +1 -1
- package/dist/reference/stores/postgres.js +53 -334
- package/dist/reference/stores/postgres.js.map +1 -1
- package/dist/reference/stores/sql-compile.d.ts +36 -0
- package/dist/reference/stores/sql-compile.d.ts.map +1 -0
- package/dist/reference/stores/sql-compile.js +95 -0
- package/dist/reference/stores/sql-compile.js.map +1 -0
- package/dist/reference/stores/sqlite.d.ts +12 -27
- package/dist/reference/stores/sqlite.d.ts.map +1 -1
- package/dist/reference/stores/sqlite.js +54 -355
- package/dist/reference/stores/sqlite.js.map +1 -1
- package/dist/report/console.d.ts +2 -1
- package/dist/report/console.d.ts.map +1 -1
- package/dist/report/console.js +56 -10
- package/dist/report/console.js.map +1 -1
- package/dist/report/matrix.d.ts +10 -13
- package/dist/report/matrix.d.ts.map +1 -1
- package/dist/report/matrix.js +82 -18
- package/dist/report/matrix.js.map +1 -1
- package/dist/report/render.d.ts +18 -1
- package/dist/report/render.d.ts.map +1 -1
- package/dist/report/render.js +169 -14
- package/dist/report/render.js.map +1 -1
- package/dist/runtime/async.d.ts +16 -3
- package/dist/runtime/async.d.ts.map +1 -1
- package/dist/runtime/async.js +24 -19
- package/dist/runtime/async.js.map +1 -1
- package/dist/runtime/auth.d.ts +2 -9
- package/dist/runtime/auth.d.ts.map +1 -1
- package/dist/runtime/auth.js +38 -62
- package/dist/runtime/auth.js.map +1 -1
- package/dist/runtime/body.d.ts +15 -3
- package/dist/runtime/body.d.ts.map +1 -1
- package/dist/runtime/body.js +61 -12
- package/dist/runtime/body.js.map +1 -1
- package/dist/runtime/check-costs.d.ts +15 -0
- package/dist/runtime/check-costs.d.ts.map +1 -0
- package/dist/runtime/check-costs.js +114 -0
- package/dist/runtime/check-costs.js.map +1 -0
- package/dist/runtime/check-ids.d.ts +15 -0
- package/dist/runtime/check-ids.d.ts.map +1 -0
- package/dist/runtime/check-ids.js +140 -0
- package/dist/runtime/check-ids.js.map +1 -0
- package/dist/runtime/checks.d.ts +122 -18
- package/dist/runtime/checks.d.ts.map +1 -1
- package/dist/runtime/checks.js +3437 -1933
- package/dist/runtime/checks.js.map +1 -1
- package/dist/runtime/client.d.ts +112 -0
- package/dist/runtime/client.d.ts.map +1 -1
- package/dist/runtime/client.js +271 -39
- package/dist/runtime/client.js.map +1 -1
- package/dist/runtime/cookies.d.ts +16 -2
- package/dist/runtime/cookies.d.ts.map +1 -1
- package/dist/runtime/cookies.js +83 -22
- package/dist/runtime/cookies.js.map +1 -1
- package/dist/runtime/exchanges.d.ts +4 -5
- package/dist/runtime/exchanges.d.ts.map +1 -1
- package/dist/runtime/exchanges.js +33 -55
- package/dist/runtime/exchanges.js.map +1 -1
- package/dist/runtime/exit.d.ts +40 -0
- package/dist/runtime/exit.d.ts.map +1 -0
- package/dist/runtime/exit.js +48 -0
- package/dist/runtime/exit.js.map +1 -0
- package/dist/runtime/feature-gate.d.ts +0 -9
- package/dist/runtime/feature-gate.d.ts.map +1 -1
- package/dist/runtime/feature-gate.js +0 -21
- package/dist/runtime/feature-gate.js.map +1 -1
- package/dist/runtime/finding.d.ts +61 -7
- package/dist/runtime/finding.d.ts.map +1 -1
- package/dist/runtime/finding.js +74 -12
- package/dist/runtime/finding.js.map +1 -1
- package/dist/runtime/fixture.d.ts +35 -28
- package/dist/runtime/fixture.d.ts.map +1 -1
- package/dist/runtime/fixture.js +92 -731
- package/dist/runtime/fixture.js.map +1 -1
- package/dist/runtime/generate.d.ts +63 -0
- package/dist/runtime/generate.d.ts.map +1 -0
- package/dist/runtime/generate.js +635 -0
- package/dist/runtime/generate.js.map +1 -0
- package/dist/runtime/headers.d.ts +6 -0
- package/dist/runtime/headers.d.ts.map +1 -0
- package/dist/runtime/headers.js +16 -0
- package/dist/runtime/headers.js.map +1 -0
- package/dist/runtime/path.d.ts +7 -0
- package/dist/runtime/path.d.ts.map +1 -0
- package/dist/runtime/path.js +44 -0
- package/dist/runtime/path.js.map +1 -0
- package/dist/runtime/payloads.d.ts +2 -0
- package/dist/runtime/payloads.d.ts.map +1 -1
- package/dist/runtime/payloads.js +2 -1
- package/dist/runtime/payloads.js.map +1 -1
- package/dist/runtime/rate-limit.d.ts +6 -4
- package/dist/runtime/rate-limit.d.ts.map +1 -1
- package/dist/runtime/rate-limit.js +22 -33
- package/dist/runtime/rate-limit.js.map +1 -1
- package/dist/runtime/redact.d.ts +24 -0
- package/dist/runtime/redact.d.ts.map +1 -0
- package/dist/runtime/redact.js +98 -0
- package/dist/runtime/redact.js.map +1 -0
- package/dist/runtime/run.d.ts +51 -2
- package/dist/runtime/run.d.ts.map +1 -1
- package/dist/runtime/run.js +463 -195
- package/dist/runtime/run.js.map +1 -1
- package/dist/runtime/runs.d.ts +5 -0
- package/dist/runtime/runs.d.ts.map +1 -1
- package/dist/runtime/runs.js +7 -1
- package/dist/runtime/runs.js.map +1 -1
- package/dist/runtime/scope.d.ts +163 -0
- package/dist/runtime/scope.d.ts.map +1 -0
- package/dist/runtime/scope.js +423 -0
- package/dist/runtime/scope.js.map +1 -0
- package/dist/runtime/teardown.d.ts +17 -4
- package/dist/runtime/teardown.d.ts.map +1 -1
- package/dist/runtime/teardown.js +49 -16
- package/dist/runtime/teardown.js.map +1 -1
- package/dist/runtime/transcript.d.ts +34 -2
- package/dist/runtime/transcript.d.ts.map +1 -1
- package/dist/runtime/transcript.js +119 -24
- package/dist/runtime/transcript.js.map +1 -1
- package/dist/runtime/unique.d.ts +2 -11
- package/dist/runtime/unique.d.ts.map +1 -1
- package/dist/runtime/unique.js +27 -53
- package/dist/runtime/unique.js.map +1 -1
- package/dist/runtime/validate.d.ts +13 -2
- package/dist/runtime/validate.d.ts.map +1 -1
- package/dist/runtime/validate.js +51 -31
- package/dist/runtime/validate.js.map +1 -1
- package/dist/runtime/wait.d.ts +2 -1
- package/dist/runtime/wait.d.ts.map +1 -1
- package/dist/runtime/wait.js +2 -36
- package/dist/runtime/wait.js.map +1 -1
- package/dist/runtime/world.d.ts +31 -3
- package/dist/runtime/world.d.ts.map +1 -1
- package/dist/runtime/world.js +113 -32
- package/dist/runtime/world.js.map +1 -1
- package/dist/spec/collection.d.ts +8 -2
- package/dist/spec/collection.d.ts.map +1 -1
- package/dist/spec/collection.js +47 -7
- package/dist/spec/collection.js.map +1 -1
- package/dist/spec/extensions.d.ts +5 -1
- package/dist/spec/extensions.d.ts.map +1 -1
- package/dist/spec/extensions.js +9 -2
- package/dist/spec/extensions.js.map +1 -1
- package/dist/spec/graph.d.ts +46 -0
- package/dist/spec/graph.d.ts.map +1 -1
- package/dist/spec/graph.js +175 -11
- package/dist/spec/graph.js.map +1 -1
- package/dist/spec/load.d.ts +25 -5
- package/dist/spec/load.d.ts.map +1 -1
- package/dist/spec/load.js +160 -59
- package/dist/spec/load.js.map +1 -1
- package/dist/spec/query-capabilities.d.ts +4 -0
- package/dist/spec/query-capabilities.d.ts.map +1 -1
- package/dist/spec/query-capabilities.js +6 -0
- package/dist/spec/query-capabilities.js.map +1 -1
- package/dist/spec/schema.d.ts +20 -0
- package/dist/spec/schema.d.ts.map +1 -0
- package/dist/spec/schema.js +118 -0
- package/dist/spec/schema.js.map +1 -0
- package/dist/spec/tag-schema.d.ts +12 -0
- package/dist/spec/tag-schema.d.ts.map +1 -0
- package/dist/spec/tag-schema.js +143 -0
- package/dist/spec/tag-schema.js.map +1 -0
- package/dist/spec/types.js +17 -1
- package/dist/spec/types.js.map +1 -1
- package/docs/api.md +59 -0
- package/docs/assumptions.md +99 -0
- package/docs/checks.md +183 -0
- package/docs/commands.md +214 -0
- package/docs/configuration.md +650 -0
- package/docs/model.md +184 -0
- package/docs/reference-defects.md +94 -0
- package/docs/reports.md +211 -0
- package/docs/tags.md +400 -0
- package/package.json +3 -1
package/dist/runtime/run.js
CHANGED
|
@@ -5,39 +5,27 @@
|
|
|
5
5
|
* everything downstream is reported BLOCKED against that single cause, rather than re-failing
|
|
6
6
|
* per case — which is what made prior attempts produce dozens of failures from one broken fixture.
|
|
7
7
|
*/
|
|
8
|
-
import {
|
|
8
|
+
import { randomBytes } from "node:crypto";
|
|
9
|
+
import { isSecretHeaderName, registerSecret, registerSecretHeader } from "./redact.js";
|
|
10
|
+
import { buildModel, operationResolver } from "../spec/graph.js";
|
|
9
11
|
import { dereference, loadSpec } from "../spec/load.js";
|
|
10
|
-
import { isAuthFlow } from "../config/define-config.js";
|
|
11
12
|
import { createPrincipal } from "./auth.js";
|
|
12
13
|
import { createExchangeJournal } from "./exchanges.js";
|
|
13
14
|
import { DEFAULT_NETWORK_RETRIES, DEFAULT_NETWORK_WAIT_MS, NetworkError, createNetworkGate, describeNetworkFailure, isNetworkError, probeOrigin, resolveRequestTimeoutMs, } from "./network.js";
|
|
14
15
|
import { resolveBackoff } from "./poll.js";
|
|
15
16
|
import { persistedToPrincipal, snapshotPrincipal } from "./principals.js";
|
|
16
17
|
import { CHECKS } from "./checks.js";
|
|
17
|
-
import { Client } from "./client.js";
|
|
18
|
-
import { FindingCollector } from "./finding.js";
|
|
19
|
-
import { reportFeatureGateSchemaDrift } from "./feature-gate.js";
|
|
18
|
+
import { AuthRefreshError, Client, } from "./client.js";
|
|
19
|
+
import { FindingCollector, isRootCause } from "./finding.js";
|
|
20
20
|
import { formatUniqueSets } from "../spec/extensions.js";
|
|
21
21
|
import { excludedByProfile, resolveProfile } from "./profile.js";
|
|
22
|
+
import { authStepOperationIds, buildScopeReport, createIsAuthProvisioned, GradeLedger, resolveTargetScope, staticSubjects, } from "./scope.js";
|
|
22
23
|
import { buildRateLimitRules, RateLimiter } from "./rate-limit.js";
|
|
23
24
|
import { resolveEntityCapabilities } from "./query-capabilities.js";
|
|
24
25
|
import { Ledger } from "./teardown.js";
|
|
25
26
|
import { SchemaValidator } from "./validate.js";
|
|
26
27
|
import { isOverflowError, overflowFrom } from "./fixture.js";
|
|
27
|
-
import { SeedError, fillPath, listExisting, probeCreateFixtures,
|
|
28
|
-
function readPointer(body, pointer) {
|
|
29
|
-
const path = pointer
|
|
30
|
-
.replace(/^\$\.?/, "")
|
|
31
|
-
.split(".")
|
|
32
|
-
.filter(Boolean);
|
|
33
|
-
let node = body;
|
|
34
|
-
for (const segment of path) {
|
|
35
|
-
if (node === null || typeof node !== "object")
|
|
36
|
-
return undefined;
|
|
37
|
-
node = node[segment];
|
|
38
|
-
}
|
|
39
|
-
return node;
|
|
40
|
-
}
|
|
28
|
+
import { SeedError, fillPath, listExisting, probeCreateFixtures, resolvePathScope, seedCohort, } from "./world.js";
|
|
41
29
|
function sameTenant(a, b) {
|
|
42
30
|
const keys = [...new Set([...Object.keys(a), ...Object.keys(b)])];
|
|
43
31
|
if (keys.length === 0)
|
|
@@ -81,37 +69,6 @@ async function teardownPrincipals(principals, hooks, findings) {
|
|
|
81
69
|
}
|
|
82
70
|
}
|
|
83
71
|
}
|
|
84
|
-
/**
|
|
85
|
-
* A 429 is a finding only when the request that drew it was demonstrably under a rate the
|
|
86
|
-
* *document* declared — `rateLimitSource === "tag"` and the bucket had a free token, meaning oat
|
|
87
|
-
* did not have to wait for one. A 429 against a config-supplied rate is the operator's own guess
|
|
88
|
-
* about the environment, not a claim the API made, so it is paced around and never reported; a
|
|
89
|
-
* 429 that only arrived after oat's own bucket made the request wait means oat's rate model was
|
|
90
|
-
* too generous, which is oat's fault, not the backend's.
|
|
91
|
-
*
|
|
92
|
-
* Grouped by category rather than reported per request: the point is "the declared rate for X is
|
|
93
|
-
* wrong", once, not a flood of identical findings for every request that category serves.
|
|
94
|
-
*/
|
|
95
|
-
function reportRateLimitViolations(client, findings) {
|
|
96
|
-
const byCategory = new Map();
|
|
97
|
-
for (const exchange of client.transcript) {
|
|
98
|
-
if (exchange.status !== 429)
|
|
99
|
-
continue;
|
|
100
|
-
if (exchange.rateLimitSource !== "tag" || exchange.rateLimitHadRoom !== true)
|
|
101
|
-
continue;
|
|
102
|
-
const category = exchange.rateLimitCategory ?? "unknown";
|
|
103
|
-
const list = byCategory.get(category) ?? [];
|
|
104
|
-
list.push(exchange);
|
|
105
|
-
byCategory.set(category, list);
|
|
106
|
-
}
|
|
107
|
-
for (const [category, exchanges] of byCategory) {
|
|
108
|
-
findings.spec("spec.declared-rate-limit-is-honoured", category, `the declared "${category}" rate limit is not honoured`, `${exchanges.length} request(s) to the "${category}" category returned 429 despite oat ` +
|
|
109
|
-
"pacing them within the rate x-rate-limit declares — each had a free token in its own " +
|
|
110
|
-
"bucket at the moment it was sent, so this is not oat outrunning its own model. Either " +
|
|
111
|
-
`the declared rate is wrong, or the backend enforces a stricter one than "${category}" ` +
|
|
112
|
-
"documents.", exchanges.slice(0, 3));
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
72
|
/**
|
|
116
73
|
* Resolves a principal to something that can produce headers on demand.
|
|
117
74
|
*
|
|
@@ -124,6 +81,13 @@ async function resolvePrincipal(principal, model, client, hooks, outOfBand, orig
|
|
|
124
81
|
/* A principal without a flow authenticates by static header — a long-lived API key needs no
|
|
125
82
|
* acquisition at all, and forcing one would be ceremony. */
|
|
126
83
|
if (principal.auth === undefined) {
|
|
84
|
+
/* A static key is a credential like any other: never written down in clear. */
|
|
85
|
+
for (const [name, value] of Object.entries(principal.headers ?? {})) {
|
|
86
|
+
if (!isSecretHeaderName(name) && !/key|token|secret|auth/i.test(name))
|
|
87
|
+
continue;
|
|
88
|
+
registerSecretHeader(name);
|
|
89
|
+
registerSecret(value);
|
|
90
|
+
}
|
|
127
91
|
return {
|
|
128
92
|
headers: () => principal.headers ?? {},
|
|
129
93
|
id: principal.id,
|
|
@@ -166,7 +130,7 @@ async function resolvePrincipal(principal, model, client, hooks, outOfBand, orig
|
|
|
166
130
|
};
|
|
167
131
|
}
|
|
168
132
|
/** Reads whatever the list endpoint already returns, for degraded read-only coverage. */
|
|
169
|
-
async function readExisting(listOp, client, headers, roots) {
|
|
133
|
+
async function readExisting(listOp, model, client, headers, roots) {
|
|
170
134
|
/* Seed with every known root, not only the list route's own parameters. Sibling routes for
|
|
171
135
|
* the same entity are often scoped differently — a global list beside a tenant-scoped item
|
|
172
136
|
* route — and a scope built from the list alone leaves those unresolvable. */
|
|
@@ -175,42 +139,9 @@ async function readExisting(listOp, client, headers, roots) {
|
|
|
175
139
|
if (scope[param] === undefined)
|
|
176
140
|
return { records: [], scope };
|
|
177
141
|
}
|
|
178
|
-
const records = await listExisting(listOp, client, headers, scope);
|
|
142
|
+
const records = await listExisting(listOp, model, client, headers, scope);
|
|
179
143
|
return { records, scope };
|
|
180
144
|
}
|
|
181
|
-
/** operationIds named in any principal `auth.steps` — those creates already ran during login. */
|
|
182
|
-
function authStepOperationIds(principals) {
|
|
183
|
-
const ids = new Set();
|
|
184
|
-
for (const principal of principals) {
|
|
185
|
-
const auth = principal.auth;
|
|
186
|
-
if (auth === undefined || !isAuthFlow(auth))
|
|
187
|
-
continue;
|
|
188
|
-
for (const step of auth.steps) {
|
|
189
|
-
if ("operationId" in step)
|
|
190
|
-
ids.add(step.operationId);
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
return ids;
|
|
194
|
-
}
|
|
195
|
-
function createIsAuthProvisioned(createOp, authCreates) {
|
|
196
|
-
return createOp.freshPrincipal || authCreates.has(createOp.operationId);
|
|
197
|
-
}
|
|
198
|
-
function testableEntities(model, only, authCreates) {
|
|
199
|
-
return [...model.entities.values()]
|
|
200
|
-
.filter((entity) => {
|
|
201
|
-
if (only !== undefined && only.length > 0 && !only.includes(entity.name))
|
|
202
|
-
return false;
|
|
203
|
-
const createOp = entity.create === undefined ? undefined : model.byOperationId.get(entity.create);
|
|
204
|
-
const authProvisioned = createOp !== undefined && createIsAuthProvisioned(createOp, authCreates);
|
|
205
|
-
/* Register-as-create with no item route is the auth flow, not a fixture. */
|
|
206
|
-
if (authProvisioned && entity.read === undefined)
|
|
207
|
-
return false;
|
|
208
|
-
if (entity.invite !== null && entity.list !== undefined)
|
|
209
|
-
return true;
|
|
210
|
-
return entity.list !== undefined && entity.create !== undefined && entity.trackable;
|
|
211
|
-
})
|
|
212
|
-
.sort((a, b) => a.name.localeCompare(b.name));
|
|
213
|
-
}
|
|
214
145
|
export async function run(options) {
|
|
215
146
|
const startedAt = Date.now();
|
|
216
147
|
const findings = new FindingCollector();
|
|
@@ -223,7 +154,7 @@ export async function run(options) {
|
|
|
223
154
|
let currentCheck;
|
|
224
155
|
let currentEntityIndex;
|
|
225
156
|
let entityTotal;
|
|
226
|
-
const defectCount = () => findings.findings.filter(
|
|
157
|
+
const defectCount = () => findings.findings.filter(isRootCause).length;
|
|
227
158
|
const oldestInflight = () => {
|
|
228
159
|
let oldest;
|
|
229
160
|
for (const probe of inflight) {
|
|
@@ -327,9 +258,11 @@ export async function run(options) {
|
|
|
327
258
|
remember(exchange);
|
|
328
259
|
if (journal !== null) {
|
|
329
260
|
try {
|
|
261
|
+
/* The exchange names its own check and entity: a shared "current check" was wrong
|
|
262
|
+
* whenever checks ran concurrently, and journaled a whole batch under the last one. */
|
|
330
263
|
await journal.record(exchange, {
|
|
331
|
-
...(
|
|
332
|
-
...(
|
|
264
|
+
...(exchange.check === undefined ? {} : { check: exchange.check }),
|
|
265
|
+
...(exchange.subject === undefined ? {} : { entity: exchange.subject }),
|
|
333
266
|
phase: currentPhase,
|
|
334
267
|
});
|
|
335
268
|
}
|
|
@@ -387,13 +320,44 @@ export async function run(options) {
|
|
|
387
320
|
client = new Client(options.baseUrl, options.globalHeaders ?? {}, options.maxInFlight ?? 4, onExchange, rateLimiter, httpHooks, networkClient);
|
|
388
321
|
if (hooks.resolveHeaders !== undefined)
|
|
389
322
|
client.setResolveHeaders(hooks.resolveHeaders);
|
|
390
|
-
|
|
323
|
+
client.setOperationResolver(operationResolver(model));
|
|
324
|
+
const validator = new SchemaValidator(model.defs);
|
|
391
325
|
const ledger = new Ledger();
|
|
392
326
|
const seed = options.seed ?? 1;
|
|
327
|
+
/* Fresh per run: the seed reproduces a run, the nonce keeps two runs' records apart. */
|
|
328
|
+
const nonce = options.nonce ?? randomBytes(4).toString("hex");
|
|
393
329
|
const outOfBand = resolveBackoff(options.outOfBand);
|
|
330
|
+
/* Write paths that have had the whole payload catalog, shared by every entity in the run. */
|
|
331
|
+
const payloadPaths = new Map();
|
|
394
332
|
if (options.principals[0] === undefined)
|
|
395
333
|
throw new Error("oat: at least one principal is required");
|
|
396
334
|
const originClients = await loadOriginClients(options.origins ?? [], hooks, options.maxInFlight ?? 4, options.globalHeaders ?? {}, onExchange, httpHooks, networkClient);
|
|
335
|
+
/* Resolved before any principal signs up: a typo in --ops must cost nothing. */
|
|
336
|
+
const runScope = resolveTargetScope(model, {
|
|
337
|
+
...(options.ops === undefined ? {} : { ops: options.ops }),
|
|
338
|
+
...(options.only === undefined ? {} : { only: options.only }),
|
|
339
|
+
authCreates,
|
|
340
|
+
exclusion: { profile: profile.name, reason: (op) => excludedByProfile(op, profile.spec) },
|
|
341
|
+
origins: new Map([...originClients].map(([id, origin]) => [id, origin.model])),
|
|
342
|
+
});
|
|
343
|
+
const grades = new GradeLedger();
|
|
344
|
+
tick({
|
|
345
|
+
message: runScope.mode === "full"
|
|
346
|
+
? "scope: full"
|
|
347
|
+
: `scope: targeted · ${runScope.targets.size} operation(s) · ${runScope.entities.length} entit${runScope.entities.length === 1 ? "y" : "ies"}`,
|
|
348
|
+
phase: "load",
|
|
349
|
+
requests: client.transcript.length,
|
|
350
|
+
});
|
|
351
|
+
const scopeReport = () => buildScopeReport({
|
|
352
|
+
authCreates,
|
|
353
|
+
baseUrl: options.baseUrl,
|
|
354
|
+
findings: findings.findings,
|
|
355
|
+
inconclusive: findings.inconclusive,
|
|
356
|
+
ledger: grades,
|
|
357
|
+
model,
|
|
358
|
+
scope: runScope,
|
|
359
|
+
transcript: client.transcript,
|
|
360
|
+
});
|
|
397
361
|
const uploads = {
|
|
398
362
|
seed,
|
|
399
363
|
...(options.uploads === undefined ? {} : { uploads: options.uploads }),
|
|
@@ -405,17 +369,21 @@ export async function run(options) {
|
|
|
405
369
|
const resolved = [];
|
|
406
370
|
try {
|
|
407
371
|
for (const principal of options.principals) {
|
|
408
|
-
resolved.push(await resolvePrincipal(principal, model, client, hooks, outOfBand, originClients.size === 0 ? undefined : originClients));
|
|
372
|
+
resolved.push(await resolvePrincipal(principal, model, client.view({ purpose: "auth" }), hooks, outOfBand, originClients.size === 0 ? undefined : originClients));
|
|
409
373
|
}
|
|
410
374
|
}
|
|
411
375
|
catch (error) {
|
|
412
376
|
if (!isNetworkError(error))
|
|
413
377
|
throw error;
|
|
414
378
|
noteNetwork(error, "auth");
|
|
379
|
+
await journal?.flush().catch(() => undefined);
|
|
415
380
|
return {
|
|
381
|
+
checksOutOfScope: [],
|
|
416
382
|
checksRun: [],
|
|
417
383
|
checksSkipped: [],
|
|
418
384
|
checksSuppressed: [],
|
|
385
|
+
checkNotes: [],
|
|
386
|
+
checkTimings: [],
|
|
419
387
|
client,
|
|
420
388
|
created: 0,
|
|
421
389
|
entitiesTested: [],
|
|
@@ -425,6 +393,7 @@ export async function run(options) {
|
|
|
425
393
|
principals: [],
|
|
426
394
|
profile: profile.name,
|
|
427
395
|
profileExclusions,
|
|
396
|
+
scope: scopeReport(),
|
|
428
397
|
teardown: null,
|
|
429
398
|
...(journal === null ? {} : { exchanges: { count: journal.count } }),
|
|
430
399
|
...(networkOutcome === undefined ? {} : { network: networkOutcome }),
|
|
@@ -435,14 +404,56 @@ export async function run(options) {
|
|
|
435
404
|
phase: "auth",
|
|
436
405
|
requests: client.transcript.length,
|
|
437
406
|
});
|
|
407
|
+
/* Every exchange names the principal whose credential it carried. */
|
|
408
|
+
client.setPrincipalResolver((sent) => resolved.find((principal) => {
|
|
409
|
+
const own = principal.headers();
|
|
410
|
+
return Object.keys(own).length > 0 && Object.entries(own).every(([key, value]) => sent[key] === value);
|
|
411
|
+
})?.id);
|
|
438
412
|
const alpha = resolved[0];
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
413
|
+
const ownerOf = (principal) => ({ headers: principal.headers, id: principal.id });
|
|
414
|
+
const alphaOwner = ownerOf(alpha);
|
|
415
|
+
/* A record a principal created and could not remove itself — a member cannot delete — may be
|
|
416
|
+
* removed by a stronger principal of the same tenant, and by nobody else. */
|
|
417
|
+
const fallbackDeleters = (owner) => {
|
|
418
|
+
const creator = resolved.find((principal) => principal.id === owner.id);
|
|
419
|
+
if (creator === undefined)
|
|
420
|
+
return [];
|
|
421
|
+
return resolved
|
|
422
|
+
.filter((principal) => principal.id !== creator.id && sameTenant(principal.roots, creator.roots))
|
|
423
|
+
.sort((a, b) => b.rank - a.rank)
|
|
424
|
+
.map(ownerOf);
|
|
425
|
+
};
|
|
426
|
+
const recorder = (owner) => (entity, id, values) => {
|
|
427
|
+
ledger.record(entity, id, values, owner);
|
|
428
|
+
};
|
|
429
|
+
/**
|
|
430
|
+
* Proactive refreshes, before each entity and before the credentials are persisted. A failure
|
|
431
|
+
* is reported — the backend would not renew a session — and the run carries on to teardown,
|
|
432
|
+
* where an exception here used to end it with every record still in place.
|
|
433
|
+
*/
|
|
434
|
+
const refreshAll = async (where) => {
|
|
435
|
+
for (const principal of resolved) {
|
|
436
|
+
try {
|
|
437
|
+
await principal.runtime?.refreshIfStale();
|
|
438
|
+
}
|
|
439
|
+
catch (error) {
|
|
440
|
+
findings.blocked("auth.refresh", where, `refreshing the credential of "${principal.id}" failed`, error instanceof Error ? error.message : String(error));
|
|
441
|
+
return false;
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
return true;
|
|
445
|
+
};
|
|
442
446
|
const entitiesTested = [];
|
|
443
447
|
const checksRun = new Set();
|
|
444
448
|
const checksSkipped = [];
|
|
445
449
|
const checksSuppressed = [];
|
|
450
|
+
const checkTimings = [];
|
|
451
|
+
const checkNotes = [];
|
|
452
|
+
const checksOutOfScope = [];
|
|
453
|
+
const profileExcludes = (operationId) => {
|
|
454
|
+
const op = model.byOperationId.get(operationId);
|
|
455
|
+
return op !== undefined && excludedByProfile(op, profile.spec) !== null;
|
|
456
|
+
};
|
|
446
457
|
const excludedIds = new Set();
|
|
447
458
|
const excludeOp = (entity, op) => {
|
|
448
459
|
if (op === undefined)
|
|
@@ -454,7 +465,9 @@ export async function run(options) {
|
|
|
454
465
|
return true;
|
|
455
466
|
excludedIds.add(op.operationId);
|
|
456
467
|
profileExclusions.push({ entity: entity.name, operationId: op.operationId, reason });
|
|
457
|
-
findings
|
|
468
|
+
findings
|
|
469
|
+
.attributed([op.operationId])
|
|
470
|
+
.gap("profile.skip", entity.name, `${op.operationId} excluded by profile "${profile.name}"`, `${reason}, under --profile ${profile.name}. Checks that depend on this operation stand ` +
|
|
458
471
|
"down rather than run against data oat did not create through it.");
|
|
459
472
|
return true;
|
|
460
473
|
};
|
|
@@ -463,8 +476,27 @@ export async function run(options) {
|
|
|
463
476
|
const createOp = model.byOperationId.get(entity.create ?? "");
|
|
464
477
|
if (listOp === undefined)
|
|
465
478
|
return;
|
|
479
|
+
/* Cohort variants the backend refused. A check that stands down for want of cohort data
|
|
480
|
+
* says which variants never arrived, so the gap points at the seed rather than the API. */
|
|
481
|
+
let lostVariants = [];
|
|
482
|
+
/* This entity's own scratch parents, shared by its records and its principals' scopes but by
|
|
483
|
+
* no other entity: entities whose parent chains are disjoint run side by side, and one that
|
|
484
|
+
* shares a parent record with another could see that entity's writes through it. */
|
|
485
|
+
const ancestors = new Map();
|
|
486
|
+
const withLostVariants = (needs) => lostVariants.length === 0 || !/cohort|record/.test(needs)
|
|
487
|
+
? needs
|
|
488
|
+
: `${needs} (the cohort is missing its ${lostVariants.map((v) => `"${v}"`).join(", ")} ` +
|
|
489
|
+
`variant${lostVariants.length === 1 ? "" : "s"}, which failed to seed)`;
|
|
466
490
|
if (networkGate.exhausted)
|
|
467
491
|
return;
|
|
492
|
+
/* Targets this entity's checks can grade. Seeding findings land on them: under --ops a seed
|
|
493
|
+
* that fails leaves exactly these ungraded, and the report has to say which. */
|
|
494
|
+
const entityTargets = runScope.mode === "full"
|
|
495
|
+
? createOp === undefined
|
|
496
|
+
? []
|
|
497
|
+
: [createOp.operationId]
|
|
498
|
+
: [...staticSubjects(entity, model)].filter((id) => runScope.inScope(id));
|
|
499
|
+
const entityFindings = entityTargets.length === 0 ? findings : findings.attributed(entityTargets);
|
|
468
500
|
const inviteOnly = entity.invite !== null && createOp === undefined;
|
|
469
501
|
const authProvisioned = createOp !== undefined && createIsAuthProvisioned(createOp, authCreates);
|
|
470
502
|
if (createOp === undefined && !inviteOnly)
|
|
@@ -477,6 +509,8 @@ export async function run(options) {
|
|
|
477
509
|
currentEntity = entity.name;
|
|
478
510
|
currentCheck = undefined;
|
|
479
511
|
currentPhase = "seed";
|
|
512
|
+
/* Everything this entity's world-building sends is seeding, attributed to the entity. */
|
|
513
|
+
const seeding = client.view({ purpose: "seed", subject: entity.name });
|
|
480
514
|
tick({
|
|
481
515
|
entity: entity.name,
|
|
482
516
|
entityIndex: currentEntityIndex,
|
|
@@ -485,8 +519,8 @@ export async function run(options) {
|
|
|
485
519
|
phase: "seed",
|
|
486
520
|
requests: client.transcript.length,
|
|
487
521
|
});
|
|
488
|
-
|
|
489
|
-
|
|
522
|
+
if (!(await refreshAll(entity.name)))
|
|
523
|
+
return;
|
|
490
524
|
const rootValues = { ...options.roots, ...alpha.roots };
|
|
491
525
|
let scope;
|
|
492
526
|
let records;
|
|
@@ -496,12 +530,12 @@ export async function run(options) {
|
|
|
496
530
|
/* Same fallback a failed create takes below: read-only coverage against whatever
|
|
497
531
|
* already exists beats no coverage, and it is exactly the state most likely to hide a
|
|
498
532
|
* read-path bug. The gap finding excludeOp already reported names the reason. */
|
|
499
|
-
const existing = await readExisting(listOp,
|
|
533
|
+
const existing = await readExisting(listOp, model, seeding, alpha.headers(), {
|
|
500
534
|
...options.roots,
|
|
501
535
|
...alpha.roots,
|
|
502
536
|
});
|
|
503
537
|
if (existing.records.length === 0) {
|
|
504
|
-
|
|
538
|
+
entityFindings.blocked("profile.skip", entity.name, `could not test "${entity.name}"`, `create is excluded by profile "${profile.name}" and the list route returned no ` +
|
|
505
539
|
"existing records to fall back on.");
|
|
506
540
|
return;
|
|
507
541
|
}
|
|
@@ -512,7 +546,7 @@ export async function run(options) {
|
|
|
512
546
|
else if (createOp === undefined || authProvisioned) {
|
|
513
547
|
/* Invite is not a fixture create; register / x-fresh-principal already ran in auth.
|
|
514
548
|
* The invite check is the only thing that POSTs a grant, and it uses inviteAs. */
|
|
515
|
-
const existing = await readExisting(listOp,
|
|
549
|
+
const existing = await readExisting(listOp, model, seeding, alpha.headers(), {
|
|
516
550
|
...options.roots,
|
|
517
551
|
...alpha.roots,
|
|
518
552
|
});
|
|
@@ -522,8 +556,12 @@ export async function run(options) {
|
|
|
522
556
|
}
|
|
523
557
|
else {
|
|
524
558
|
try {
|
|
525
|
-
scope = await
|
|
559
|
+
scope = await resolvePathScope(createOp, model, seeding, {
|
|
526
560
|
authHeaders: alpha.headers,
|
|
561
|
+
ancestors,
|
|
562
|
+
onCreate: recorder(alphaOwner),
|
|
563
|
+
principal: alpha.id,
|
|
564
|
+
nonce,
|
|
527
565
|
roots: rootValues,
|
|
528
566
|
seed,
|
|
529
567
|
uploads,
|
|
@@ -533,70 +571,70 @@ export async function run(options) {
|
|
|
533
571
|
* a tenant-scoped item route — and a scope built from create alone leaves those
|
|
534
572
|
* unresolvable, which shows up as a wall of "could not complete" gaps. */
|
|
535
573
|
scope.values = { ...rootValues, ...scope.values };
|
|
536
|
-
const cohort = await seedCohort(createOp, model,
|
|
574
|
+
const cohort = await seedCohort(createOp, model, seeding, {
|
|
537
575
|
authHeaders: alpha.headers,
|
|
538
576
|
...(options.cohortSize === undefined ? {} : { cohortSize: options.cohortSize }),
|
|
577
|
+
ancestors,
|
|
578
|
+
onCreate: recorder(alphaOwner),
|
|
579
|
+
principal: alpha.id,
|
|
580
|
+
nonce,
|
|
539
581
|
roots: rootValues,
|
|
540
582
|
seed,
|
|
541
583
|
uploads,
|
|
542
584
|
}, scope);
|
|
543
585
|
if (cohort.uniqueGap !== undefined) {
|
|
544
|
-
|
|
586
|
+
entityFindings.gap("world.seed", entity.name, cohort.uniqueGap, cohort.uniqueGap);
|
|
587
|
+
}
|
|
588
|
+
lostVariants = (cohort.failedVariants ?? []).map((failed) => failed.variant);
|
|
589
|
+
for (const failed of cohort.failedVariants ?? []) {
|
|
590
|
+
const said = JSON.stringify(failed.exchange.responseBody).slice(0, 300);
|
|
591
|
+
if (failed.exchange.status >= 500) {
|
|
592
|
+
entityFindings.backend("create.does-not-error", entity.name, `creating the "${failed.variant}" variant of a "${entity.name}" fails with a server error`, `${createOp.operationId} returned ${failed.exchange.status} for a body generated from the ` +
|
|
593
|
+
`documented schema (${said}). The other variants were created, so this input is what ` +
|
|
594
|
+
"breaks the handler. Checks that need this variant's data run without it.", [failed.exchange]);
|
|
595
|
+
}
|
|
596
|
+
else {
|
|
597
|
+
entityFindings.gap("world.seed", entity.name, `the "${failed.variant}" variant of "${entity.name}" was refused with ${failed.exchange.status}`, `${createOp.operationId} refused a body the document permits (${said}). Checks that ` +
|
|
598
|
+
"rely on this variant's values run without them.");
|
|
599
|
+
}
|
|
545
600
|
}
|
|
546
601
|
if (cohort.adopted === true) {
|
|
547
602
|
/* Plan limit after an effect already created the row: keep the id so children
|
|
548
603
|
* (row after extract→table) can seed, but do not assert write-path oracles
|
|
549
604
|
* against a body oat never submitted. */
|
|
550
|
-
|
|
605
|
+
entityFindings.gap("world.seed", entity.name, `seeding "${entity.name}" hit a plan limit; using an existing same-tenant record`, `${createOp.operationId} returned a plan-limit refusal and the list already ` +
|
|
551
606
|
"had a record — likely an earlier x-effects create. Write-path checks stand " +
|
|
552
607
|
"down rather than treat payment_required as a backend defect.");
|
|
608
|
+
/* Adopted, not created: what oat did create was recorded as it happened, and
|
|
609
|
+
* this record is somebody else's to keep. */
|
|
553
610
|
records = cohort.records;
|
|
554
611
|
degraded = true;
|
|
555
|
-
for (const ancestor of scope.created) {
|
|
556
|
-
ledger.record(ancestor.entity, ancestor.id, scope.values);
|
|
557
|
-
}
|
|
558
|
-
for (const record of records) {
|
|
559
|
-
const id = record[entity.identity ?? "id"];
|
|
560
|
-
if (typeof id === "string" || typeof id === "number") {
|
|
561
|
-
ledger.record(entity.name, String(id), scope.values);
|
|
562
|
-
}
|
|
563
|
-
}
|
|
564
612
|
}
|
|
565
613
|
else if (cohort.uniqueAdopted === true) {
|
|
566
614
|
/* Unique 409 on the first variant with a same-tenant list: adopt like 402,
|
|
567
615
|
* but unique-conflict checks still run against that row. Write-path oracles
|
|
568
616
|
* that need a body oat submitted stay skipped. The seed 409 is not the
|
|
569
617
|
* unique check passing — that check is the explicit second POST. */
|
|
570
|
-
|
|
618
|
+
entityFindings.gap("world.seed", entity.name, `seeding "${entity.name}" could not insert because of x-unique`, `${createOp.operationId} returned 409 and the list already had a record. ` +
|
|
571
619
|
`x-unique: ${formatUniqueSets(createOp.unique)}. The create could not insert; ` +
|
|
572
620
|
"unique-conflict checks still run against that row. Write-path oracles that " +
|
|
573
621
|
"need a body oat submitted stand down.");
|
|
574
622
|
records = cohort.records;
|
|
575
623
|
degraded = true;
|
|
576
624
|
uniqueAdopted = true;
|
|
577
|
-
for (const ancestor of scope.created) {
|
|
578
|
-
ledger.record(ancestor.entity, ancestor.id, scope.values);
|
|
579
|
-
}
|
|
580
|
-
for (const record of records) {
|
|
581
|
-
const id = record[entity.identity ?? "id"];
|
|
582
|
-
if (typeof id === "string" || typeof id === "number") {
|
|
583
|
-
ledger.record(entity.name, String(id), scope.values);
|
|
584
|
-
}
|
|
585
|
-
}
|
|
586
625
|
}
|
|
587
626
|
else if (cohort.featureGate !== null) {
|
|
588
627
|
/* Same degradation a profile-excluded create takes: the tag said this
|
|
589
628
|
* principal cannot create the row, so a correct 403 is coverage, not a
|
|
590
|
-
* seed defect.
|
|
591
|
-
|
|
592
|
-
findings.gap("world.seed", entity.name, `seeding "${entity.name}" is gated by ${cohort.featureGate.detail}`, `${cohort.featureGate.detail}. Checks that need a row oat created stand down ` +
|
|
629
|
+
* seed defect. schema.error-response-matches-document still judges its body. */
|
|
630
|
+
entityFindings.gap("world.seed", entity.name, `seeding "${entity.name}" is gated by ${cohort.featureGate.detail}`, `${cohort.featureGate.detail}. Checks that need a row oat created stand down ` +
|
|
593
631
|
"rather than treat the documented 403 as a defect.");
|
|
594
|
-
const existing = await readExisting(listOp,
|
|
632
|
+
const existing = await readExisting(listOp, model, seeding, alpha.headers(), {
|
|
595
633
|
...options.roots,
|
|
596
634
|
...alpha.roots,
|
|
597
635
|
});
|
|
598
636
|
if (existing.records.length === 0) {
|
|
599
|
-
|
|
637
|
+
entityFindings.blocked("world.seed", entity.name, `could not test "${entity.name}"`, `${cohort.featureGate.detail} and the list route returned no existing ` + "records to fall back on.");
|
|
600
638
|
return;
|
|
601
639
|
}
|
|
602
640
|
scope = { created: [], values: existing.scope };
|
|
@@ -604,15 +642,9 @@ export async function run(options) {
|
|
|
604
642
|
degraded = true;
|
|
605
643
|
}
|
|
606
644
|
else {
|
|
645
|
+
/* Every ancestor and cohort record was ledgered as it was created, in creation
|
|
646
|
+
* order — the unwind reverses it, so children go before their parents. */
|
|
607
647
|
records = cohort.records;
|
|
608
|
-
/* Ancestors first, then the cohort — the unwind reverses this, so children are
|
|
609
|
-
* always removed before the parents they hang from. */
|
|
610
|
-
for (const ancestor of scope.created) {
|
|
611
|
-
ledger.record(ancestor.entity, ancestor.id, scope.values);
|
|
612
|
-
}
|
|
613
|
-
for (const record of records) {
|
|
614
|
-
ledger.record(entity.name, String(record[entity.identity ?? "id"]), scope.values);
|
|
615
|
-
}
|
|
616
648
|
}
|
|
617
649
|
}
|
|
618
650
|
catch (error) {
|
|
@@ -622,13 +654,13 @@ export async function run(options) {
|
|
|
622
654
|
}
|
|
623
655
|
if (isOverflowError(error)) {
|
|
624
656
|
const overflow = overflowFrom(error, createOp.operationId);
|
|
625
|
-
|
|
626
|
-
const existing = await readExisting(listOp,
|
|
657
|
+
entityFindings.gap("world.seed", entity.name, overflow.message, overflow.message);
|
|
658
|
+
const existing = await readExisting(listOp, model, seeding, alpha.headers(), {
|
|
627
659
|
...options.roots,
|
|
628
660
|
...alpha.roots,
|
|
629
661
|
});
|
|
630
662
|
if (existing.records.length === 0) {
|
|
631
|
-
|
|
663
|
+
entityFindings.blocked("world.seed", entity.name, `could not seed "${entity.name}"`, overflow.message);
|
|
632
664
|
return;
|
|
633
665
|
}
|
|
634
666
|
scope = { created: [], values: existing.scope };
|
|
@@ -639,26 +671,39 @@ export async function run(options) {
|
|
|
639
671
|
const cause = error instanceof SeedError ? error.cause_ : "unknown";
|
|
640
672
|
const status = error instanceof SeedError ? error.status : undefined;
|
|
641
673
|
const message = error instanceof Error ? error.message : String(error);
|
|
674
|
+
const failedOp = (error instanceof SeedError ? error.operationId : undefined) ?? createOp.operationId;
|
|
675
|
+
const evidence = error instanceof SeedError && error.exchange !== undefined
|
|
676
|
+
? [error.exchange]
|
|
677
|
+
: await Promise.all(client.transcript
|
|
678
|
+
.filter((e) => e.status >= 500)
|
|
679
|
+
.slice(-1)
|
|
680
|
+
.map((e) => client.hydrate(e)));
|
|
681
|
+
/* Under --ops a create nobody targeted is support. Its failure leaves the targets
|
|
682
|
+
* ungraded — blocked, with the evidence — but it is not what this run was asked to
|
|
683
|
+
* judge, so it is not reported as a defect of its own. */
|
|
684
|
+
const supportFailed = runScope.mode === "targeted" && !runScope.inScope(failedOp);
|
|
642
685
|
/* A create that fails with 5xx is not a fixture problem — it is the defect.
|
|
643
686
|
* Reporting it as merely "blocked" buries the most serious thing oat found. */
|
|
644
|
-
if (status !== undefined && status >= 500) {
|
|
645
|
-
findings
|
|
687
|
+
if (status !== undefined && status >= 500 && !supportFailed) {
|
|
688
|
+
findings
|
|
689
|
+
.attributed([failedOp])
|
|
690
|
+
.backend("create.does-not-error", entity.name, `creating a "${entity.name}" fails with a server error`, `${message}. The request body was generated from the documented schema, so ` +
|
|
646
691
|
"either the handler rejects input the document permits, or it is failing " +
|
|
647
692
|
"outright. Everything downstream of this entity is untestable until it is " +
|
|
648
|
-
"fixed.",
|
|
693
|
+
"fixed.", evidence);
|
|
649
694
|
}
|
|
650
695
|
/* Fall back to whatever already exists. A backend whose create is broken can still
|
|
651
696
|
* have a working list, and read-only coverage beats no coverage — this is exactly
|
|
652
697
|
* the state in which a read-path bug is most likely to be sitting undiscovered. */
|
|
653
|
-
const existing = await readExisting(listOp,
|
|
698
|
+
const existing = await readExisting(listOp, model, seeding, alpha.headers(), {
|
|
654
699
|
...options.roots,
|
|
655
700
|
...alpha.roots,
|
|
656
701
|
});
|
|
657
702
|
if (existing.records.length === 0) {
|
|
658
|
-
|
|
703
|
+
entityFindings.blocked("world.seed", entity.name, `could not seed "${entity.name}"`, supportFailed ? `support operation ${failedOp} failing: ${message}` : `${cause}: ${message}`, evidence);
|
|
659
704
|
return;
|
|
660
705
|
}
|
|
661
|
-
|
|
706
|
+
entityFindings.gap("world.seed", entity.name, `seeding "${entity.name}" failed; running read-only checks against existing records`, `${cause}: ${message}. Write-path and lifecycle checks are skipped for this entity.`);
|
|
662
707
|
scope = { created: [], values: existing.scope };
|
|
663
708
|
records = existing.records;
|
|
664
709
|
degraded = true;
|
|
@@ -668,8 +713,13 @@ export async function run(options) {
|
|
|
668
713
|
const actorOf = async (principal, seedOffset) => {
|
|
669
714
|
const roots = { ...options.roots, ...principal.roots };
|
|
670
715
|
try {
|
|
671
|
-
const next = await
|
|
716
|
+
const next = await resolvePathScope(listOp, model, seeding, {
|
|
672
717
|
authHeaders: principal.headers,
|
|
718
|
+
/* Created as this principal, so removed as this principal. */
|
|
719
|
+
ancestors,
|
|
720
|
+
onCreate: recorder(ownerOf(principal)),
|
|
721
|
+
principal: principal.id,
|
|
722
|
+
nonce,
|
|
673
723
|
roots,
|
|
674
724
|
seed: seed + seedOffset,
|
|
675
725
|
uploads: worldUploads(seedOffset),
|
|
@@ -725,9 +775,22 @@ export async function run(options) {
|
|
|
725
775
|
* reporting that it had cleaned up everything it made. Recording centrally rather than at
|
|
726
776
|
* each call site means a new check cannot forget: the wrapper sees every request.
|
|
727
777
|
*/
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
:
|
|
778
|
+
let createPath;
|
|
779
|
+
try {
|
|
780
|
+
createPath = createOp === undefined ? undefined : fillPath(createOp.path, scope.values);
|
|
781
|
+
}
|
|
782
|
+
catch {
|
|
783
|
+
createPath = undefined;
|
|
784
|
+
}
|
|
785
|
+
/* Whoever sent the request owns what it created: matched on the credential it carried. */
|
|
786
|
+
const ownerByHeaders = (sent) => {
|
|
787
|
+
const rendered = JSON.stringify(typeof sent === "function" ? sent() : (sent ?? {}));
|
|
788
|
+
const match = resolved.find((principal) => JSON.stringify(principal.headers()) === rendered);
|
|
789
|
+
return match === undefined ? alphaOwner : ownerOf(match);
|
|
790
|
+
};
|
|
791
|
+
const tracked = (base) => createOp === undefined || degraded
|
|
792
|
+
? base
|
|
793
|
+
: new Proxy(base, {
|
|
731
794
|
get(target, property, receiver) {
|
|
732
795
|
if (property !== "request")
|
|
733
796
|
return Reflect.get(target, property, receiver);
|
|
@@ -735,15 +798,18 @@ export async function run(options) {
|
|
|
735
798
|
const exchange = await target.request(method, path, options ?? {});
|
|
736
799
|
if (method.toUpperCase() !== "POST" || exchange.status >= 300)
|
|
737
800
|
return exchange;
|
|
801
|
+
/* Only this entity's own create makes one of its records. Any other 2xx
|
|
802
|
+
* POST carrying an `id` — an invite, an action — names something else, and
|
|
803
|
+
* on an integer-id API deleting by that number removes an unrelated row. */
|
|
804
|
+
if (path !== createPath)
|
|
805
|
+
return exchange;
|
|
738
806
|
const body = exchange.responseBody;
|
|
739
807
|
if (body === null || typeof body !== "object")
|
|
740
808
|
return exchange;
|
|
741
809
|
const id = body[entity.identity ?? "id"];
|
|
742
810
|
if (typeof id !== "string" && typeof id !== "number")
|
|
743
811
|
return exchange;
|
|
744
|
-
|
|
745
|
-
* on an already-removed record — and missing one is not. */
|
|
746
|
-
ledger.record(entity.name, String(id), scope.values);
|
|
812
|
+
ledger.record(entity.name, String(id), scope.values, ownerByHeaders(options?.headers));
|
|
747
813
|
return exchange;
|
|
748
814
|
};
|
|
749
815
|
},
|
|
@@ -786,14 +852,19 @@ export async function run(options) {
|
|
|
786
852
|
actors,
|
|
787
853
|
altAuth,
|
|
788
854
|
altScope,
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
855
|
+
/* Narrowed to targets: these are the subjects their checks invoke, and invoking an
|
|
856
|
+
* untargeted one is exactly the cost --ops exists to skip. */
|
|
857
|
+
asyncOps: model.operations.filter((op) => op.entity === entity.name && op.async !== null && invocable(op) && runScope.inScope(op.operationId)),
|
|
858
|
+
effectOps: model.operations.filter((op) => op.entity === entity.name && op.effects.length > 0 && invocable(op) && runScope.inScope(op.operationId)),
|
|
859
|
+
waitOps: model.operations.filter((op) => op.entity === entity.name && op.wait !== null && invocable(op) && runScope.inScope(op.operationId)),
|
|
860
|
+
inScope: (op) => runScope.inScope(op.operationId),
|
|
792
861
|
hooks,
|
|
793
862
|
outOfBand,
|
|
863
|
+
payloads: { policy: options.payloads ?? "per-write-path", ran: payloadPaths },
|
|
794
864
|
auth: alpha.headers,
|
|
865
|
+
recordCreated: recorder(alphaOwner),
|
|
795
866
|
...(alpha.runtime === undefined ? {} : { refreshIfStale: alpha.runtime.refreshIfStale }),
|
|
796
|
-
client:
|
|
867
|
+
client: tracked(client.view({ purpose: "assertion", subject: entity.name })),
|
|
797
868
|
collectionKey: listOp.collection?.key ?? null,
|
|
798
869
|
/* In degraded mode oat did not write these records, so it has no oracle for them —
|
|
799
870
|
* every write-path check must sit out rather than assert against data it did not
|
|
@@ -817,6 +888,7 @@ export async function run(options) {
|
|
|
817
888
|
softDelete: model.operations.find((op) => op.entity === entity.name && op.softDelete !== null)?.softDelete ??
|
|
818
889
|
listOp.softDelete,
|
|
819
890
|
seed,
|
|
891
|
+
nonce,
|
|
820
892
|
updateOp: degraded || updateExcluded ? undefined : updateOpModel,
|
|
821
893
|
uploads,
|
|
822
894
|
validator,
|
|
@@ -833,7 +905,28 @@ export async function run(options) {
|
|
|
833
905
|
: undefined,
|
|
834
906
|
};
|
|
835
907
|
entitiesTested.push(entity.name);
|
|
908
|
+
/* Subjects each check grades on this entity: in scope, and not excluded by the profile. */
|
|
909
|
+
const gradedBy = new Map();
|
|
910
|
+
/* What each check's plan gathered, and how each check ended, on this entity. */
|
|
911
|
+
const plans = new Map();
|
|
912
|
+
const outcomes = new Map();
|
|
836
913
|
const runOne = async (check) => {
|
|
914
|
+
const graded = gradedBy.get(check.id) ?? [];
|
|
915
|
+
let judged;
|
|
916
|
+
/* Its own client: every exchange names this check, even while a batch runs concurrently.
|
|
917
|
+
* A check that does not mutate gets one that refuses to write. */
|
|
918
|
+
const stamp = { check: check.id, purpose: "assertion", subject: entity.name };
|
|
919
|
+
const attributed = graded.length === 0 ? findings : findings.attributed(graded);
|
|
920
|
+
const view = {
|
|
921
|
+
...ctx,
|
|
922
|
+
client: tracked(check.mutates === true ? client.view(stamp) : client.readOnlyView(stamp)),
|
|
923
|
+
findings: attributed.ownedBy(check.id),
|
|
924
|
+
judged: (operationIds) => {
|
|
925
|
+
judged ??= new Set();
|
|
926
|
+
for (const id of operationIds)
|
|
927
|
+
judged.add(id);
|
|
928
|
+
},
|
|
929
|
+
};
|
|
837
930
|
checksRun.add(check.id);
|
|
838
931
|
currentCheck = check.id;
|
|
839
932
|
currentPhase = "test";
|
|
@@ -845,16 +938,47 @@ export async function run(options) {
|
|
|
845
938
|
phase: "test",
|
|
846
939
|
requests: client.transcript.length,
|
|
847
940
|
});
|
|
941
|
+
const start = findings.findings.length;
|
|
942
|
+
const began = performance.now();
|
|
943
|
+
let outcome;
|
|
848
944
|
try {
|
|
849
|
-
await check.run(
|
|
945
|
+
outcome = await check.run(view, plans.get(check.id));
|
|
850
946
|
}
|
|
851
947
|
catch (error) {
|
|
852
948
|
if (isNetworkError(error)) {
|
|
853
949
|
noteNetwork(error, entity.name);
|
|
950
|
+
outcomes.set(check.id, { kind: "unresolved", reason: "the network failed" });
|
|
951
|
+
return;
|
|
952
|
+
}
|
|
953
|
+
if (error instanceof AuthRefreshError) {
|
|
954
|
+
outcomes.set(check.id, { kind: "unresolved", reason: "a credential could not be refreshed" });
|
|
955
|
+
attributed.blocked("auth.refresh", entity.name, `refreshing the credential${error.principal === undefined ? "" : ` of "${error.principal}"`} failed`, `${error.message}. ${check.id} could not complete.`);
|
|
854
956
|
return;
|
|
855
957
|
}
|
|
856
|
-
findings.gap(check.id, entity.name, `check "${check.id}" could not complete`, error instanceof Error ? error.message : String(error));
|
|
958
|
+
outcome = view.findings.gap(check.id, entity.name, `check "${check.id}" could not complete`, error instanceof Error ? error.message : String(error));
|
|
959
|
+
}
|
|
960
|
+
checkTimings.push({ check: check.id, entity: entity.name, ms: Math.round(performance.now() - began) });
|
|
961
|
+
/* A check that reported along the way and then fell through to "asserted" did not
|
|
962
|
+
* assert anything: what it reported is the outcome. */
|
|
963
|
+
if (outcome.kind === "asserted") {
|
|
964
|
+
const reported = findings.findings
|
|
965
|
+
.slice(start)
|
|
966
|
+
.find((f) => f.check === check.id && f.entity === entity.name && isRootCause(f));
|
|
967
|
+
if (reported !== undefined)
|
|
968
|
+
outcome = { kind: "finding", verdict: reported.verdict };
|
|
857
969
|
}
|
|
970
|
+
outcomes.set(check.id, outcome);
|
|
971
|
+
if (outcome.kind === "asserted" && outcome.note !== undefined) {
|
|
972
|
+
checkNotes.push({ check: check.id, entity: entity.name, note: outcome.note });
|
|
973
|
+
}
|
|
974
|
+
if (outcome.kind === "stood-down") {
|
|
975
|
+
/* Not a pass: the property was never tested here, and the report says why. */
|
|
976
|
+
const needs = withLostVariants(outcome.reason);
|
|
977
|
+
checksSkipped.push({ check: check.id, entity: entity.name, needs });
|
|
978
|
+
grades.skipped(graded, needs);
|
|
979
|
+
return;
|
|
980
|
+
}
|
|
981
|
+
grades.graded(judged === undefined ? graded : graded.filter((id) => judged?.has(id) === true), check.id);
|
|
858
982
|
};
|
|
859
983
|
/* Cascade suppression: a check whose premise is already known broken would report a
|
|
860
984
|
* consequence, not a defect. One root cause, one finding. */
|
|
@@ -874,13 +998,20 @@ export async function run(options) {
|
|
|
874
998
|
const suppressedBy = new Map();
|
|
875
999
|
const suppressed = (check) => {
|
|
876
1000
|
for (const dependency of check.dependsOn ?? []) {
|
|
877
|
-
|
|
1001
|
+
/* Only a failed premise suppresses. A dependency that stood down, could not reach a
|
|
1002
|
+
* verdict, or was blocked says nothing about whether its premise holds. */
|
|
1003
|
+
const outcome = outcomes.get(dependency);
|
|
1004
|
+
const failed = outcome === undefined
|
|
1005
|
+
? /* Not run as a check here, but reported against — during seeding, say. */
|
|
1006
|
+
findings.findings.some((f) => f.check === dependency && f.entity === entity.name && isRootCause(f))
|
|
1007
|
+
: outcome.kind === "finding";
|
|
878
1008
|
const inherited = suppressedBy.get(dependency);
|
|
879
1009
|
if (!failed && inherited === undefined)
|
|
880
1010
|
continue;
|
|
881
1011
|
const because = failed ? dependency : inherited;
|
|
882
1012
|
suppressedBy.set(check.id, because);
|
|
883
1013
|
checksSuppressed.push({ because, check: check.id, entity: entity.name });
|
|
1014
|
+
grades.suppress(gradedBy.get(check.id) ?? [], check.id);
|
|
884
1015
|
return true;
|
|
885
1016
|
}
|
|
886
1017
|
return false;
|
|
@@ -902,17 +1033,28 @@ export async function run(options) {
|
|
|
902
1033
|
await Promise.all(pending.map(runOne));
|
|
903
1034
|
};
|
|
904
1035
|
for (const check of CHECKS) {
|
|
905
|
-
if (
|
|
1036
|
+
if (options.signal?.aborted === true)
|
|
1037
|
+
break;
|
|
1038
|
+
const graded = check.subjects(entity, model).filter((id) => runScope.inScope(id) && !profileExcludes(id));
|
|
1039
|
+
/* Out of scope is not a skip: the check could have run, this run was not asked to. */
|
|
1040
|
+
if (runScope.mode === "targeted" && graded.length === 0) {
|
|
1041
|
+
checksOutOfScope.push({ check: check.id, entity: entity.name });
|
|
1042
|
+
continue;
|
|
1043
|
+
}
|
|
1044
|
+
gradedBy.set(check.id, graded);
|
|
1045
|
+
const planned = check.plan(ctx);
|
|
1046
|
+
if (!planned.ok) {
|
|
906
1047
|
/* Recorded, not dropped: on an API shaped unlike the fixture this is most of the
|
|
907
1048
|
* suite, and a silent skip reads exactly like a clean result. */
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
needs: check.needs ?? "an unstated precondition",
|
|
912
|
-
});
|
|
1049
|
+
const needs = withLostVariants(planned.needs ?? check.needs ?? "an unstated precondition");
|
|
1050
|
+
checksSkipped.push({ check: check.id, entity: entity.name, needs });
|
|
1051
|
+
grades.skipped(graded, needs);
|
|
913
1052
|
continue;
|
|
914
1053
|
}
|
|
915
|
-
|
|
1054
|
+
plans.set(check.id, planned.value);
|
|
1055
|
+
/* A writer runs alone; so does a check that judges the transcript, once everything
|
|
1056
|
+
* queued before it has finished sending. */
|
|
1057
|
+
if (check.mutates === true || check.judgesTranscript === true) {
|
|
916
1058
|
await flush();
|
|
917
1059
|
if (suppressed(check))
|
|
918
1060
|
continue;
|
|
@@ -929,22 +1071,55 @@ export async function run(options) {
|
|
|
929
1071
|
await flush();
|
|
930
1072
|
};
|
|
931
1073
|
/*
|
|
932
|
-
*
|
|
933
|
-
*
|
|
934
|
-
*
|
|
935
|
-
*
|
|
936
|
-
*
|
|
1074
|
+
* Checks inside an entity stay ordered because cascade suppression consults findings already
|
|
1075
|
+
* reported for it — concurrent checks would let a root cause and its consequences race and
|
|
1076
|
+
* both be reported. Nested graphs couple entities too: a child create in flight while a
|
|
1077
|
+
* parent page-walk runs invents pagination findings. So entities that can observe each
|
|
1078
|
+
* other share a lane and run in series, and only lanes run side by side.
|
|
937
1079
|
*/
|
|
938
|
-
const queue =
|
|
1080
|
+
const queue = runScope.entities;
|
|
939
1081
|
entityTotal = queue.length;
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
1082
|
+
let interrupted = false;
|
|
1083
|
+
let started = 0;
|
|
1084
|
+
let stopNoted = false;
|
|
1085
|
+
/* Entities that share nothing run side by side; see `entityLanes`. */
|
|
1086
|
+
const runLane = async (lane) => {
|
|
1087
|
+
for (const entity of lane) {
|
|
1088
|
+
/* Stopped from outside — Ctrl-C, a deadline. Nothing more is tested, and everything
|
|
1089
|
+
* made so far is still removed below. */
|
|
1090
|
+
if (options.signal?.aborted === true || networkGate.exhausted) {
|
|
1091
|
+
if (stopNoted)
|
|
1092
|
+
return;
|
|
1093
|
+
stopNoted = true;
|
|
1094
|
+
const left = queue.length - started;
|
|
1095
|
+
if (options.signal?.aborted === true) {
|
|
1096
|
+
interrupted = true;
|
|
1097
|
+
findings.gap("run.interrupted", "run", `stopped before ${left} remaining entit${left === 1 ? "y" : "ies"}`, "the run was interrupted; what was tested is reported, and what oat created is removed");
|
|
1098
|
+
}
|
|
1099
|
+
else {
|
|
1100
|
+
findings.blocked("net.unreachable", "run", `skipped ${left} remaining entit${left === 1 ? "y" : "ies"}`, "network down");
|
|
1101
|
+
}
|
|
1102
|
+
return;
|
|
1103
|
+
}
|
|
1104
|
+
started += 1;
|
|
1105
|
+
currentEntityIndex = started;
|
|
1106
|
+
try {
|
|
1107
|
+
await testEntity(entity);
|
|
1108
|
+
}
|
|
1109
|
+
catch (error) {
|
|
1110
|
+
/* An error oat did not expect must not skip teardown: it is reported, and the run
|
|
1111
|
+
* carries on to remove what it made and write its report. */
|
|
1112
|
+
findings.blocked("run.error", entity.name, `testing "${entity.name}" stopped on an unexpected error`, errorText(error));
|
|
1113
|
+
}
|
|
945
1114
|
}
|
|
946
|
-
|
|
947
|
-
|
|
1115
|
+
};
|
|
1116
|
+
try {
|
|
1117
|
+
await Promise.all(entityLanes(model, queue).map(runLane));
|
|
1118
|
+
}
|
|
1119
|
+
catch (error) {
|
|
1120
|
+
/* Each entity's errors are caught where it runs; this is the scheduling around them. What
|
|
1121
|
+
* was created is still removed below, and the report still written. */
|
|
1122
|
+
findings.blocked("run.error", "run", "testing stopped on an unexpected error", errorText(error));
|
|
948
1123
|
}
|
|
949
1124
|
/* Unwind after every check has run, never per case: a check may legitimately depend on records
|
|
950
1125
|
* another one created, and tearing down early turns that into a phantom defect. */
|
|
@@ -955,25 +1130,32 @@ export async function run(options) {
|
|
|
955
1130
|
phase: "teardown",
|
|
956
1131
|
requests: client.transcript.length,
|
|
957
1132
|
});
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
1133
|
+
let teardown = null;
|
|
1134
|
+
if (options.keepFixtures !== true && ledger.size > 0) {
|
|
1135
|
+
try {
|
|
1136
|
+
teardown = await ledger.unwind(model, client.view({ purpose: "teardown" }), fallbackDeleters, (done, total, item) => {
|
|
1137
|
+
if (done % 25 !== 0 && done !== total)
|
|
1138
|
+
return;
|
|
1139
|
+
tick({
|
|
1140
|
+
entity: item.entity,
|
|
1141
|
+
message: `${done}/${total} ${item.entity} ${item.id}`,
|
|
1142
|
+
phase: "teardown",
|
|
1143
|
+
requests: client.transcript.length,
|
|
1144
|
+
});
|
|
968
1145
|
});
|
|
969
|
-
}
|
|
1146
|
+
}
|
|
1147
|
+
catch (error) {
|
|
1148
|
+
findings.blocked("world.teardown", "run", `teardown stopped on an unexpected error with ${ledger.size} record(s) ledgered`, errorText(error));
|
|
1149
|
+
}
|
|
1150
|
+
}
|
|
970
1151
|
if (teardown !== null && teardown.unsupported.length > 0) {
|
|
971
1152
|
findings.gap("world.teardown", teardown.unsupported.join(", "), "records created during the run could not be removed", `no delete operation is reachable for ${teardown.unsupported.join(", ")}, so this run left ` +
|
|
972
1153
|
"fixtures behind. Declare x-cleanup to name the route that removes them.");
|
|
973
1154
|
}
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
1155
|
+
await refreshAll("run");
|
|
1156
|
+
/* Every check has judged the transcript; bodies moved out of memory are no longer needed. */
|
|
1157
|
+
await client.dispose();
|
|
1158
|
+
await journal?.flush().catch(() => undefined);
|
|
977
1159
|
const persisted = resolved.map((principal) => snapshotPrincipal({
|
|
978
1160
|
headers: principal.headers,
|
|
979
1161
|
id: principal.id,
|
|
@@ -982,8 +1164,14 @@ export async function run(options) {
|
|
|
982
1164
|
rank: principal.rank,
|
|
983
1165
|
...(principal.inviteAs === undefined ? {} : { inviteAs: principal.inviteAs }),
|
|
984
1166
|
}));
|
|
1167
|
+
const coverage = scopeReport();
|
|
985
1168
|
if ((options.origins ?? []).length > 0 && options.skipPrincipalTeardown !== true) {
|
|
986
|
-
|
|
1169
|
+
try {
|
|
1170
|
+
await runSecondaryOrigins(options, persisted, findings, checksRun, checksSkipped, checksSuppressed, checksOutOfScope, entitiesTested, runScope, coverage);
|
|
1171
|
+
}
|
|
1172
|
+
catch (error) {
|
|
1173
|
+
findings.blocked("run.error", "origins", "the secondary origins stopped on an unexpected error", errorText(error));
|
|
1174
|
+
}
|
|
987
1175
|
}
|
|
988
1176
|
if (options.skipPrincipalTeardown !== true) {
|
|
989
1177
|
await teardownPrincipals(resolved, hooks, findings);
|
|
@@ -994,6 +1182,9 @@ export async function run(options) {
|
|
|
994
1182
|
requests: client.transcript.length,
|
|
995
1183
|
});
|
|
996
1184
|
return {
|
|
1185
|
+
checkNotes,
|
|
1186
|
+
checkTimings,
|
|
1187
|
+
checksOutOfScope,
|
|
997
1188
|
checksRun: [...checksRun].sort(),
|
|
998
1189
|
checksSkipped,
|
|
999
1190
|
checksSuppressed,
|
|
@@ -1006,7 +1197,9 @@ export async function run(options) {
|
|
|
1006
1197
|
principals: persisted,
|
|
1007
1198
|
profile: profile.name,
|
|
1008
1199
|
profileExclusions,
|
|
1200
|
+
scope: coverage,
|
|
1009
1201
|
teardown,
|
|
1202
|
+
...(interrupted ? { interrupted } : {}),
|
|
1010
1203
|
...(journal === null ? {} : { exchanges: { count: journal.count } }),
|
|
1011
1204
|
...(networkOutcome === undefined ? {} : { network: networkOutcome }),
|
|
1012
1205
|
};
|
|
@@ -1028,8 +1221,18 @@ async function loadOriginClients(origins, hooks, maxInFlight, globalHeaders, onE
|
|
|
1028
1221
|
}
|
|
1029
1222
|
return map;
|
|
1030
1223
|
}
|
|
1031
|
-
|
|
1224
|
+
function errorText(error) {
|
|
1225
|
+
return error instanceof Error ? (error.stack ?? error.message) : String(error);
|
|
1226
|
+
}
|
|
1227
|
+
async function runSecondaryOrigins(options, persisted, findings, checksRun, checksSkipped, checksSuppressed, checksOutOfScope, entitiesTested, runScope, coverage) {
|
|
1032
1228
|
for (const origin of options.origins ?? []) {
|
|
1229
|
+
/* Targets were resolved against every origin up front; an origin none of them name has
|
|
1230
|
+
* nothing to grade, and running it anyway is the cost --ops exists to skip. */
|
|
1231
|
+
const originOps = runScope.origins.get(origin.id);
|
|
1232
|
+
if (runScope.mode === "targeted" && originOps === undefined) {
|
|
1233
|
+
coverage.originsSkipped.push(origin.id);
|
|
1234
|
+
continue;
|
|
1235
|
+
}
|
|
1033
1236
|
const originHooks = options.hooks === undefined
|
|
1034
1237
|
? undefined
|
|
1035
1238
|
: (() => {
|
|
@@ -1048,7 +1251,7 @@ async function runSecondaryOrigins(options, persisted, findings, checksRun, chec
|
|
|
1048
1251
|
...(options.roots === undefined ? {} : { roots: options.roots }),
|
|
1049
1252
|
...(options.seed === undefined ? {} : { seed: options.seed }),
|
|
1050
1253
|
...(options.cohortSize === undefined ? {} : { cohortSize: options.cohortSize }),
|
|
1051
|
-
...(
|
|
1254
|
+
...(originOps === undefined ? {} : { ops: originOps }),
|
|
1052
1255
|
...(options.profiles === undefined ? {} : { profiles: options.profiles }),
|
|
1053
1256
|
...(options.profile === undefined ? {} : { profile: options.profile }),
|
|
1054
1257
|
...(options.rateLimits === undefined ? {} : { rateLimits: options.rateLimits }),
|
|
@@ -1073,9 +1276,74 @@ async function runSecondaryOrigins(options, persisted, findings, checksRun, chec
|
|
|
1073
1276
|
}
|
|
1074
1277
|
for (const entity of result.entitiesTested)
|
|
1075
1278
|
entitiesTested.push(`${origin.id}:${entity}`);
|
|
1279
|
+
for (const item of result.checksOutOfScope) {
|
|
1280
|
+
checksOutOfScope.push({ ...item, entity: `${origin.id}:${item.entity}` });
|
|
1281
|
+
}
|
|
1282
|
+
for (const op of result.scope.operations)
|
|
1283
|
+
coverage.operations.push({ ...op, origin: origin.id });
|
|
1284
|
+
for (const use of result.scope.support)
|
|
1285
|
+
coverage.support.push({ ...use, origin: origin.id });
|
|
1076
1286
|
for (const item of result.inconclusive) {
|
|
1077
1287
|
findings.unresolved(item.check, `${origin.id}:${item.entity}`, item.reason);
|
|
1078
1288
|
}
|
|
1079
1289
|
}
|
|
1080
1290
|
}
|
|
1291
|
+
/**
|
|
1292
|
+
* Entities grouped into lanes that can be tested side by side.
|
|
1293
|
+
*
|
|
1294
|
+
* Two entities share a lane when one can observe the other's writes: one's collection lives
|
|
1295
|
+
* under the other's (a child, created inside a parent oat also makes), an operation of one
|
|
1296
|
+
* invalidates a route of the other, or one's effects, waits or async polls land on the other.
|
|
1297
|
+
* Inside a lane entities keep their order and run one after another; lanes share nothing, so
|
|
1298
|
+
* nothing one does can turn up as a finding in another.
|
|
1299
|
+
*/
|
|
1300
|
+
export function entityLanes(model, entities) {
|
|
1301
|
+
const names = entities.map((entity) => entity.name);
|
|
1302
|
+
const parent = new Map(names.map((name) => [name, name]));
|
|
1303
|
+
const find = (name) => {
|
|
1304
|
+
let root = name;
|
|
1305
|
+
while (parent.get(root) !== root)
|
|
1306
|
+
root = parent.get(root) ?? root;
|
|
1307
|
+
return root;
|
|
1308
|
+
};
|
|
1309
|
+
const join = (a, b) => {
|
|
1310
|
+
if (a === null || a === undefined || b === null || b === undefined)
|
|
1311
|
+
return;
|
|
1312
|
+
if (!parent.has(a) || !parent.has(b))
|
|
1313
|
+
return;
|
|
1314
|
+
parent.set(find(a), find(b));
|
|
1315
|
+
};
|
|
1316
|
+
const collection = (entity) => {
|
|
1317
|
+
const op = model.byOperationId.get(entity.list ?? entity.create ?? "");
|
|
1318
|
+
return op?.path.replace(/\/+$/, "");
|
|
1319
|
+
};
|
|
1320
|
+
for (const a of entities) {
|
|
1321
|
+
const pathA = collection(a);
|
|
1322
|
+
for (const b of entities) {
|
|
1323
|
+
const pathB = collection(b);
|
|
1324
|
+
if (a === b || pathA === undefined || pathB === undefined)
|
|
1325
|
+
continue;
|
|
1326
|
+
if (pathB.startsWith(`${pathA}/`))
|
|
1327
|
+
join(a.name, b.name);
|
|
1328
|
+
}
|
|
1329
|
+
}
|
|
1330
|
+
for (const op of model.operations) {
|
|
1331
|
+
for (const route of op.invalidates)
|
|
1332
|
+
join(op.entity, model.byRoute.get(route)?.entity);
|
|
1333
|
+
for (const effect of op.effects)
|
|
1334
|
+
join(op.entity, effect.entity);
|
|
1335
|
+
if (op.wait !== null)
|
|
1336
|
+
join(op.entity, model.byOperationId.get(op.wait.operationId)?.entity);
|
|
1337
|
+
if (op.async !== null) {
|
|
1338
|
+
const poll = model.byOperationId.get(op.async.poll) ?? model.byRoute.get(op.async.poll);
|
|
1339
|
+
join(op.entity, poll?.entity);
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
const lanes = new Map();
|
|
1343
|
+
for (const entity of entities) {
|
|
1344
|
+
const root = find(entity.name);
|
|
1345
|
+
lanes.set(root, [...(lanes.get(root) ?? []), entity]);
|
|
1346
|
+
}
|
|
1347
|
+
return [...lanes.values()];
|
|
1348
|
+
}
|
|
1081
1349
|
//# sourceMappingURL=run.js.map
|