@intentius/chant 0.63.0 → 0.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/behaviour.d.ts +999 -0
- package/dist/behaviour.d.ts.map +1 -0
- package/dist/discovery/fold-import.d.ts +12 -0
- package/dist/discovery/fold-import.d.ts.map +1 -1
- package/dist/fold/fold.d.ts +10 -0
- package/dist/fold/fold.d.ts.map +1 -1
- package/dist/fold/subset.d.ts +26 -2
- package/dist/fold/subset.d.ts.map +1 -1
- package/dist/identity.d.ts +28 -0
- package/dist/identity.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/lexicon.d.ts +45 -0
- package/dist/lexicon.d.ts.map +1 -1
- package/dist/op/activities/reconcile.d.ts +79 -16
- package/dist/op/activities/reconcile.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/behaviour.test.ts +1961 -0
- package/src/behaviour.ts +1875 -0
- package/src/discovery/fold-import.ts +12 -0
- package/src/fold/fold.ts +10 -0
- package/src/fold/subset-doc-parity.test.ts +60 -0
- package/src/fold/subset-public-export.test.ts +27 -0
- package/src/fold/subset.ts +26 -2
- package/src/identity.ts +31 -2
- package/src/index.ts +6 -0
- package/src/lexicon.ts +46 -0
- package/src/op/activities/reconcile.test.ts +84 -0
- package/src/op/activities/reconcile.ts +90 -22
|
@@ -0,0 +1,1961 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The behaviour prediction contract (#2356).
|
|
3
|
+
*
|
|
4
|
+
* Two halves. The first is a test lexicon — `acme-sim` below — that implements
|
|
5
|
+
* `predictBehaviour()` against a fixture engine, run through the shared
|
|
6
|
+
* conformance suite so the contract's rules are checked by the same code every
|
|
7
|
+
* real lexicon will be checked by (#2357 onwards).
|
|
8
|
+
*
|
|
9
|
+
* The second half is the four rules the epic states, each one asserted where it
|
|
10
|
+
* is actually enforced. Three of them are enforced by the type system, so their
|
|
11
|
+
* assertions are `@ts-expect-error` and their red is a typecheck failure, not a
|
|
12
|
+
* vitest one. That is the point: a rule enforced at runtime is a rule a lexicon
|
|
13
|
+
* author discovers after shipping.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { describe, test, expect } from "vitest";
|
|
17
|
+
import { readFileSync } from "node:fs";
|
|
18
|
+
import { join } from "node:path";
|
|
19
|
+
import {
|
|
20
|
+
describeBehaviourConformance,
|
|
21
|
+
behaviourConformanceGaps,
|
|
22
|
+
probeTrafficLevel,
|
|
23
|
+
probeReadsEdges,
|
|
24
|
+
probeEdgelessConsistency,
|
|
25
|
+
probeEchoesCoverage,
|
|
26
|
+
} from "@intentius/chant-test-utils";
|
|
27
|
+
import type { LexiconPlugin } from "./lexicon";
|
|
28
|
+
import type { IREdge } from "./graph-ir";
|
|
29
|
+
import {
|
|
30
|
+
BEHAVIOUR_BASES,
|
|
31
|
+
BEHAVIOUR_UNPREDICTED_REASONS,
|
|
32
|
+
assertNoCredentialInOptions,
|
|
33
|
+
copyEdgeCoverage,
|
|
34
|
+
behaviourEngineFrom,
|
|
35
|
+
behaviourEngineVariables,
|
|
36
|
+
behaviourReport,
|
|
37
|
+
compareFigures,
|
|
38
|
+
compareProvenance,
|
|
39
|
+
figureMismatches,
|
|
40
|
+
findCredentialsInOptions,
|
|
41
|
+
screenBehaviourRequest,
|
|
42
|
+
validateEdgeCoverage,
|
|
43
|
+
FIGURE_MISMATCHES,
|
|
44
|
+
isComparableFigure,
|
|
45
|
+
isComparableProvenance,
|
|
46
|
+
isEdgeCoverageVerdict,
|
|
47
|
+
outOfCreditBehaviourEngineRefusal,
|
|
48
|
+
overQuotaBehaviourEngineRefusal,
|
|
49
|
+
redactEngineAddress,
|
|
50
|
+
validateBehaviourBlock,
|
|
51
|
+
EDGE_COVERAGE_VERDICTS,
|
|
52
|
+
RESILIENCE_VERDICTS,
|
|
53
|
+
isBehaviourBasis,
|
|
54
|
+
isBehaviourRefusalReport,
|
|
55
|
+
isBehaviourResult,
|
|
56
|
+
isBehaviourUnpredictedReason,
|
|
57
|
+
noBehaviourEngineMessage,
|
|
58
|
+
noBehaviourEngineRefusal,
|
|
59
|
+
predictedRate,
|
|
60
|
+
renderBehaviourRefusal,
|
|
61
|
+
unreachableBehaviourEngineRefusal,
|
|
62
|
+
type BehaviourHeadroom,
|
|
63
|
+
type FigureMismatch,
|
|
64
|
+
type BehaviourResult,
|
|
65
|
+
type BehaviourUnpredictedReason,
|
|
66
|
+
type PredictBehaviourOptions,
|
|
67
|
+
type PredictedRate,
|
|
68
|
+
type PredictedBehaviour,
|
|
69
|
+
} from "./behaviour";
|
|
70
|
+
|
|
71
|
+
/* -------------------------------------------------------------------------- */
|
|
72
|
+
/* The fixture engine, and a lexicon that predicts against it */
|
|
73
|
+
/* -------------------------------------------------------------------------- */
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* What a behaviour engine answers with. A fixture, not a client: it stands in
|
|
77
|
+
* for whatever #2359's first adapter talks to, and its only job here is to be
|
|
78
|
+
* present in one test and absent in the other.
|
|
79
|
+
*/
|
|
80
|
+
const FIXTURE_ENGINE: Record<string, { perHour: number; cpu: number; latency: number }> = {
|
|
81
|
+
"AWS::EC2::Instance": { perHour: 0.0416, cpu: 0.62, latency: 0.41 },
|
|
82
|
+
"AWS::RDS::DBInstance": { perHour: 0.272, cpu: 0.35, latency: 0.28 },
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
const FIXTURE_ENGINE_VERSION = "1.4.2";
|
|
86
|
+
|
|
87
|
+
/** Six significant figures, so a float comparison in a test is stable. */
|
|
88
|
+
const round = (n: number): number => Number(n.toPrecision(6));
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* How busy the stated level is, relative to the fixture's 100 rps baseline. The
|
|
92
|
+
* fixture reads the level rather than echoing it; a `×10` suffix (which the
|
|
93
|
+
* conformance suite appends when probing) multiplies.
|
|
94
|
+
*/
|
|
95
|
+
function trafficIntensity(traffic: string): number {
|
|
96
|
+
const multiplier = /×\s*(\d+(?:\.\d+)?)/.exec(traffic);
|
|
97
|
+
const rps = /(\d+(?:\.\d+)?)\s*rps/i.exec(traffic);
|
|
98
|
+
const base = rps ? Number(rps[1]) / 100 : 1;
|
|
99
|
+
return base * (multiplier ? Number(multiplier[1]) : 1);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** The declared estate the lexicon is asked about, in both scenarios. */
|
|
103
|
+
const DECLARED = new Map<string, { entityType: string; props: Record<string, unknown> }>([
|
|
104
|
+
["web", { entityType: "AWS::EC2::Instance", props: { instanceType: "t3.medium" } }],
|
|
105
|
+
["db", { entityType: "AWS::RDS::DBInstance", props: { instanceClass: "db.t3.medium" } }],
|
|
106
|
+
["queue", { entityType: "AWS::SQS::Queue", props: {} }],
|
|
107
|
+
]);
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* The edges between them. `web` reads the queue and writes the database, which
|
|
111
|
+
* is the whole reason an engine can say anything about a path rather than about
|
|
112
|
+
* three boxes in isolation.
|
|
113
|
+
*/
|
|
114
|
+
const EDGES: IREdge[] = [
|
|
115
|
+
{ from: "web", to: "db", kind: "ref", viaAttr: "dbEndpoint", toAttr: "endpoint" },
|
|
116
|
+
{ from: "web", to: "queue", kind: "ref", viaAttr: "queueUrl", toAttr: "url" },
|
|
117
|
+
];
|
|
118
|
+
|
|
119
|
+
const REQUEST: PredictBehaviourOptions = {
|
|
120
|
+
environment: "prod",
|
|
121
|
+
buildOutput: "/tmp/build",
|
|
122
|
+
entityNames: [...DECLARED.keys()],
|
|
123
|
+
entities: DECLARED,
|
|
124
|
+
edges: EDGES,
|
|
125
|
+
edgeCoverage: { verdict: "complete" },
|
|
126
|
+
region: "us-east-1",
|
|
127
|
+
traffic: "100 rps, p50",
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* A lexicon that predicts. It resolves its engine from the environment first —
|
|
132
|
+
* refusing by name when nothing does — and only then prices anything, which is
|
|
133
|
+
* the ordering the contract requires: a lexicon that prices first and checks
|
|
134
|
+
* the engine afterwards has already decided what zero means.
|
|
135
|
+
*/
|
|
136
|
+
function acmeSim(env: Record<string, string | undefined>): LexiconPlugin["predictBehaviour"] {
|
|
137
|
+
return async (options: PredictBehaviourOptions): Promise<BehaviourResult> => {
|
|
138
|
+
// The one entry point, and the first thing in the method. Calling
|
|
139
|
+
// `assertNoCredentialInOptions` here instead applies one rule of three and
|
|
140
|
+
// drops the other two, so a token past the walk's depth budget and an
|
|
141
|
+
// `awsSecretAccessKey` in `props` both went out with the request. This
|
|
142
|
+
// fixture is the shape #2357, #2359 and #2360 copy, so it has to be right.
|
|
143
|
+
const unsafe = screenBehaviourRequest("acme", options);
|
|
144
|
+
if (unsafe) return unsafe;
|
|
145
|
+
|
|
146
|
+
const endpoint = behaviourEngineFrom("acme", env);
|
|
147
|
+
if (!endpoint) return noBehaviourEngineRefusal("acme");
|
|
148
|
+
// The engine answered in each of these three cases; only the first is a
|
|
149
|
+
// problem with the address.
|
|
150
|
+
if (endpoint.value === "fixture://down") {
|
|
151
|
+
return unreachableBehaviourEngineRefusal("acme", endpoint, "connection refused");
|
|
152
|
+
}
|
|
153
|
+
if (endpoint.value === "fixture://broke") {
|
|
154
|
+
return outOfCreditBehaviourEngineRefusal("acme", endpoint, "balance 0.00 USD");
|
|
155
|
+
}
|
|
156
|
+
if (endpoint.value === "fixture://throttled") {
|
|
157
|
+
return overQuotaBehaviourEngineRefusal("acme", endpoint, "5000/5000 predictions this hour");
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
const entities: Record<string, PredictedBehaviour> = {};
|
|
161
|
+
const unpredicted: Record<string, { type?: string; reason: "unsupported-kind" }> = {};
|
|
162
|
+
|
|
163
|
+
// An entity nothing points at and that points at nothing carries no traffic
|
|
164
|
+
// at the stated level, so its headroom is the engine's baseline. This is the
|
|
165
|
+
// whole reason `edges` is on the request: without it the engine can only
|
|
166
|
+
// price boxes.
|
|
167
|
+
// Degree, not a boolean. A node's load is proportional to how many edges
|
|
168
|
+
// touch it, so removing any one edge moves the figures of both its ends —
|
|
169
|
+
// which is what the conformance probe requires and what "reads the graph"
|
|
170
|
+
// actually means.
|
|
171
|
+
const degree = new Map<string, number>();
|
|
172
|
+
for (const edge of options.edges) {
|
|
173
|
+
degree.set(edge.from, (degree.get(edge.from) ?? 0) + 1);
|
|
174
|
+
degree.set(edge.to, (degree.get(edge.to) ?? 0) + 1);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
for (const name of options.entityNames) {
|
|
178
|
+
const declared = options.entities.get(name);
|
|
179
|
+
if (!declared) continue;
|
|
180
|
+
const modeled = FIXTURE_ENGINE[declared.entityType];
|
|
181
|
+
if (!modeled) {
|
|
182
|
+
// The rule that matters: no model for this kind means say so, never
|
|
183
|
+
// price it at nothing.
|
|
184
|
+
unpredicted[name] = { type: declared.entityType, reason: "unsupported-kind" };
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
const load = degree.get(name) ?? 0;
|
|
188
|
+
// Busier traffic spends headroom and costs more. Crude, and the point is
|
|
189
|
+
// only that the level is *read*: a fixture that echoed `traffic` and
|
|
190
|
+
// ignored it is exactly what the conformance suite now refuses to pass.
|
|
191
|
+
const intensity = trafficIntensity(options.traffic);
|
|
192
|
+
const spend = (free: number): number =>
|
|
193
|
+
Math.max(0, Math.min(1, 1 - (1 - free) * intensity));
|
|
194
|
+
entities[name] = {
|
|
195
|
+
at: { traffic: options.traffic },
|
|
196
|
+
cost: predictedRate(round(modeled.perHour * intensity), "USD"),
|
|
197
|
+
headroom: {
|
|
198
|
+
cpu: round(spend(modeled.cpu) ** load),
|
|
199
|
+
latency: round(spend(modeled.latency) ** load),
|
|
200
|
+
},
|
|
201
|
+
errorRate: round(Math.min(1, 0.001 * intensity * load)),
|
|
202
|
+
resilience: {
|
|
203
|
+
failure: "one zone lost",
|
|
204
|
+
verdict: name === "db" ? "degrades" : "survives",
|
|
205
|
+
...(name === "db" ? { note: "single-AZ; a failover costs about 90 seconds" } : {}),
|
|
206
|
+
},
|
|
207
|
+
...(name === "web"
|
|
208
|
+
? { rightSize: { suggestion: "t3.small", reason: "62% CPU headroom at the stated level" } }
|
|
209
|
+
: {}),
|
|
210
|
+
provenance: {
|
|
211
|
+
engine: "acme-sim",
|
|
212
|
+
version: FIXTURE_ENGINE_VERSION,
|
|
213
|
+
tolerance: "±15%",
|
|
214
|
+
basis: "modeled",
|
|
215
|
+
},
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
return behaviourReport(
|
|
220
|
+
options,
|
|
221
|
+
{ engine: "acme-sim", version: FIXTURE_ENGINE_VERSION },
|
|
222
|
+
entities,
|
|
223
|
+
unpredicted,
|
|
224
|
+
);
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const REACHABLE = { CHANT_BEHAVIOUR_ENGINE: "fixture://acme-sim" };
|
|
229
|
+
const CONFIGURED_BUT_DOWN = { CHANT_BEHAVIOUR_ENGINE: "fixture://down" };
|
|
230
|
+
const OUT_OF_CREDIT = { CHANT_BEHAVIOUR_ENGINE: "fixture://broke" };
|
|
231
|
+
const OVER_QUOTA = { CHANT_BEHAVIOUR_ENGINE: "fixture://throttled" };
|
|
232
|
+
const NOTHING_CONFIGURED: Record<string, string | undefined> = {};
|
|
233
|
+
|
|
234
|
+
describeBehaviourConformance({
|
|
235
|
+
lexicon: "acme-sim (fixture)",
|
|
236
|
+
scenarios: [
|
|
237
|
+
{
|
|
238
|
+
name: "the engine is up",
|
|
239
|
+
declared: [...DECLARED.keys()],
|
|
240
|
+
traffic: "100 rps, p50",
|
|
241
|
+
run: () => acmeSim(REACHABLE)!(REQUEST),
|
|
242
|
+
// The pair the suite varies. Without them it can only ask once, and a
|
|
243
|
+
// lexicon that echoes the traffic level without reading it, and never
|
|
244
|
+
// looks at `edges`, is indistinguishable from one that does the work.
|
|
245
|
+
request: REQUEST,
|
|
246
|
+
predict: (options) => acmeSim(REACHABLE)!(options),
|
|
247
|
+
otherTraffic: "1000 rps, p50",
|
|
248
|
+
expectPredicted: ["web", "db"],
|
|
249
|
+
expectUnpredicted: { queue: "unsupported-kind" },
|
|
250
|
+
},
|
|
251
|
+
{
|
|
252
|
+
name: "nothing names an engine",
|
|
253
|
+
declared: [...DECLARED.keys()],
|
|
254
|
+
run: () => acmeSim(NOTHING_CONFIGURED)!(REQUEST),
|
|
255
|
+
expectRefusal: true,
|
|
256
|
+
expectRefusalCause: "no-engine",
|
|
257
|
+
},
|
|
258
|
+
{
|
|
259
|
+
name: "an engine is named and does not answer",
|
|
260
|
+
declared: [...DECLARED.keys()],
|
|
261
|
+
run: () => acmeSim(CONFIGURED_BUT_DOWN)!(REQUEST),
|
|
262
|
+
expectRefusal: true,
|
|
263
|
+
expectRefusalCause: "engine-unreachable",
|
|
264
|
+
},
|
|
265
|
+
{
|
|
266
|
+
name: "the engine answers and the account is out of credit",
|
|
267
|
+
declared: [...DECLARED.keys()],
|
|
268
|
+
run: () => acmeSim(OUT_OF_CREDIT)!(REQUEST),
|
|
269
|
+
expectRefusal: true,
|
|
270
|
+
expectRefusalCause: "engine-out-of-credit",
|
|
271
|
+
},
|
|
272
|
+
{
|
|
273
|
+
name: "the engine answers and a limit is spent",
|
|
274
|
+
declared: [...DECLARED.keys()],
|
|
275
|
+
run: () => acmeSim(OVER_QUOTA)!(REQUEST),
|
|
276
|
+
expectRefusal: true,
|
|
277
|
+
expectRefusalCause: "engine-over-quota",
|
|
278
|
+
},
|
|
279
|
+
],
|
|
280
|
+
});
|
|
281
|
+
|
|
282
|
+
describe("a lexicon predicting against the fixture engine (#2356)", () => {
|
|
283
|
+
test("prices the kinds it models, at the traffic level it was asked for", async () => {
|
|
284
|
+
const result = await acmeSim(REACHABLE)!(REQUEST);
|
|
285
|
+
if (isBehaviourRefusalReport(result)) throw new Error("expected a report");
|
|
286
|
+
|
|
287
|
+
expect(result.behaviour).toBe("v1");
|
|
288
|
+
expect(result.meta.at.traffic).toBe("100 rps, p50");
|
|
289
|
+
expect(result.entities.web.cost).toEqual({ rate: "per-hour", perHour: 0.0416, currency: "USD" });
|
|
290
|
+
// `web` sits on both edges, so its headroom is spent twice over. The
|
|
291
|
+
// figure moves with the graph, which is the whole point of `edges`.
|
|
292
|
+
expect(result.entities.web.headroom).toEqual({ cpu: 0.3844, latency: 0.1681 });
|
|
293
|
+
expect(result.entities.web.errorRate).toBe(0.002);
|
|
294
|
+
expect(result.entities.web.resilience).toEqual({ failure: "one zone lost", verdict: "survives" });
|
|
295
|
+
expect(result.entities.web.rightSize?.suggestion).toBe("t3.small");
|
|
296
|
+
expect(result.entities.db.resilience.verdict).toBe("degrades");
|
|
297
|
+
});
|
|
298
|
+
|
|
299
|
+
test("says unsupported-kind for a kind it has no model for, and never zero", async () => {
|
|
300
|
+
const result = await acmeSim(REACHABLE)!(REQUEST);
|
|
301
|
+
if (isBehaviourRefusalReport(result)) throw new Error("expected a report");
|
|
302
|
+
|
|
303
|
+
expect(result.unpredicted?.queue).toEqual({
|
|
304
|
+
type: "AWS::SQS::Queue",
|
|
305
|
+
reason: "unsupported-kind",
|
|
306
|
+
});
|
|
307
|
+
expect(result.entities.queue).toBeUndefined();
|
|
308
|
+
});
|
|
309
|
+
|
|
310
|
+
test("every figure names its engine, version, tolerance and basis", async () => {
|
|
311
|
+
const result = await acmeSim(REACHABLE)!(REQUEST);
|
|
312
|
+
if (isBehaviourRefusalReport(result)) throw new Error("expected a report");
|
|
313
|
+
|
|
314
|
+
for (const block of Object.values(result.entities)) {
|
|
315
|
+
expect(block.provenance).toEqual({
|
|
316
|
+
engine: "acme-sim",
|
|
317
|
+
version: "1.4.2",
|
|
318
|
+
tolerance: "±15%",
|
|
319
|
+
basis: "modeled",
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
});
|
|
323
|
+
});
|
|
324
|
+
|
|
325
|
+
/* -------------------------------------------------------------------------- */
|
|
326
|
+
/* The refusal: named, red, and pointing at a variable */
|
|
327
|
+
/* -------------------------------------------------------------------------- */
|
|
328
|
+
|
|
329
|
+
describe("an unreachable engine refuses by name (#2356)", () => {
|
|
330
|
+
test("nothing configured is `no-engine`, and the message names the variable to set", async () => {
|
|
331
|
+
const result = await acmeSim(NOTHING_CONFIGURED)!(REQUEST);
|
|
332
|
+
expect(isBehaviourRefusalReport(result)).toBe(true);
|
|
333
|
+
if (!isBehaviourRefusalReport(result)) return;
|
|
334
|
+
|
|
335
|
+
expect(result.refusal.cause).toBe("no-engine");
|
|
336
|
+
expect(result.refusal.reason).toContain("CHANT_BEHAVIOUR_ENGINE");
|
|
337
|
+
expect(result.refusal.reason).toContain("CHANT_BEHAVIOUR_ENGINE_ACME");
|
|
338
|
+
expect(result.refusal.reason).toContain("BEHAVIOUR_ENGINE");
|
|
339
|
+
expect(result.refusal.remedy).toBe("Set CHANT_BEHAVIOUR_ENGINE to the engine's address.");
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
test("a configured engine that does not answer is `engine-unreachable`, naming which variable pointed at it", async () => {
|
|
343
|
+
const result = await acmeSim(CONFIGURED_BUT_DOWN)!(REQUEST);
|
|
344
|
+
if (!isBehaviourRefusalReport(result)) throw new Error("expected a refusal");
|
|
345
|
+
|
|
346
|
+
expect(result.refusal.cause).toBe("engine-unreachable");
|
|
347
|
+
expect(result.refusal.source).toBe("CHANT_BEHAVIOUR_ENGINE");
|
|
348
|
+
expect(result.refusal.reason).toContain("fixture://down");
|
|
349
|
+
expect(result.refusal.reason).toContain("connection refused");
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
test("the refusal renders red, with the reason and the remedy", async () => {
|
|
353
|
+
const result = await acmeSim(NOTHING_CONFIGURED)!(REQUEST);
|
|
354
|
+
if (!isBehaviourRefusalReport(result)) throw new Error("expected a refusal");
|
|
355
|
+
|
|
356
|
+
const rendered = renderBehaviourRefusal(result.refusal, { color: true });
|
|
357
|
+
expect(rendered.startsWith("\x1b[31m")).toBe(true);
|
|
358
|
+
expect(rendered.endsWith("\x1b[0m")).toBe(true);
|
|
359
|
+
expect(rendered).toContain("behaviour: refused (no-engine)");
|
|
360
|
+
expect(rendered).toContain("CHANT_BEHAVIOUR_ENGINE");
|
|
361
|
+
expect(rendered).toContain("Set CHANT_BEHAVIOUR_ENGINE to the engine's address.");
|
|
362
|
+
|
|
363
|
+
// No colour requested, no escapes — the same text a log file gets.
|
|
364
|
+
const plain = renderBehaviourRefusal(result.refusal, { color: false });
|
|
365
|
+
expect(plain).not.toContain("\x1b[");
|
|
366
|
+
expect(plain).toContain("behaviour: refused (no-engine)");
|
|
367
|
+
});
|
|
368
|
+
|
|
369
|
+
test("a refusal carries no entities map to be mistaken for an empty estate", async () => {
|
|
370
|
+
const result = await acmeSim(NOTHING_CONFIGURED)!(REQUEST);
|
|
371
|
+
if (!isBehaviourRefusalReport(result)) throw new Error("expected a refusal");
|
|
372
|
+
|
|
373
|
+
expect("entities" in result).toBe(false);
|
|
374
|
+
expect("meta" in result).toBe(false);
|
|
375
|
+
expect(isBehaviourResult(result)).toBe(true);
|
|
376
|
+
});
|
|
377
|
+
});
|
|
378
|
+
|
|
379
|
+
describe("an engine that answers and still refuses (#2359)", () => {
|
|
380
|
+
test("an empty account is `engine-out-of-credit`, and the remedy is about money", async () => {
|
|
381
|
+
const result = await acmeSim(OUT_OF_CREDIT)!(REQUEST);
|
|
382
|
+
if (!isBehaviourRefusalReport(result)) throw new Error("expected a refusal");
|
|
383
|
+
|
|
384
|
+
expect(result.refusal.cause).toBe("engine-out-of-credit");
|
|
385
|
+
expect(result.refusal.source).toBe("CHANT_BEHAVIOUR_ENGINE");
|
|
386
|
+
expect(result.refusal.reason).toContain("balance 0.00 USD");
|
|
387
|
+
expect(result.refusal.reason).toContain("out of credit");
|
|
388
|
+
expect(result.refusal.remedy).toBe(
|
|
389
|
+
"Add credit to the account behind CHANT_BEHAVIOUR_ENGINE, or repoint it at a funded engine.",
|
|
390
|
+
);
|
|
391
|
+
// The remedy must not send anybody to check a network that is working.
|
|
392
|
+
expect(result.refusal.remedy).not.toMatch(/reachable/i);
|
|
393
|
+
});
|
|
394
|
+
|
|
395
|
+
test("a spent limit is `engine-over-quota`, and the remedy is about the limit", async () => {
|
|
396
|
+
const result = await acmeSim(OVER_QUOTA)!(REQUEST);
|
|
397
|
+
if (!isBehaviourRefusalReport(result)) throw new Error("expected a refusal");
|
|
398
|
+
|
|
399
|
+
expect(result.refusal.cause).toBe("engine-over-quota");
|
|
400
|
+
expect(result.refusal.reason).toContain("5000/5000 predictions this hour");
|
|
401
|
+
expect(result.refusal.remedy).toBe(
|
|
402
|
+
"Wait for the engine's window to roll over, or raise the limit on the account behind CHANT_BEHAVIOUR_ENGINE.",
|
|
403
|
+
);
|
|
404
|
+
// A spent quota is not an empty account, and telling somebody to pay for
|
|
405
|
+
// one they have already paid for is the wrong instruction.
|
|
406
|
+
expect(result.refusal.remedy).not.toMatch(/credit|fund/i);
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
test("the three engine-answered causes are three distinct verdicts", async () => {
|
|
410
|
+
const causes = await Promise.all(
|
|
411
|
+
[CONFIGURED_BUT_DOWN, OUT_OF_CREDIT, OVER_QUOTA].map(async (env) => {
|
|
412
|
+
const result = await acmeSim(env)!(REQUEST);
|
|
413
|
+
if (!isBehaviourRefusalReport(result)) throw new Error("expected a refusal");
|
|
414
|
+
return result.refusal.cause;
|
|
415
|
+
}),
|
|
416
|
+
);
|
|
417
|
+
expect(new Set(causes).size).toBe(3);
|
|
418
|
+
expect(causes).toEqual(["engine-unreachable", "engine-out-of-credit", "engine-over-quota"]);
|
|
419
|
+
});
|
|
420
|
+
|
|
421
|
+
test("both render red, naming their own cause", () => {
|
|
422
|
+
const endpoint = { value: "fixture://broke", source: "CHANT_BEHAVIOUR_ENGINE" };
|
|
423
|
+
const broke = renderBehaviourRefusal(
|
|
424
|
+
outOfCreditBehaviourEngineRefusal("acme", endpoint, "balance 0.00 USD").refusal,
|
|
425
|
+
{ color: true },
|
|
426
|
+
);
|
|
427
|
+
const throttled = renderBehaviourRefusal(
|
|
428
|
+
overQuotaBehaviourEngineRefusal("acme", endpoint, "5000/5000").refusal,
|
|
429
|
+
{ color: true },
|
|
430
|
+
);
|
|
431
|
+
for (const rendered of [broke, throttled]) {
|
|
432
|
+
expect(rendered.startsWith("\x1b[31m")).toBe(true);
|
|
433
|
+
expect(rendered.endsWith("\x1b[0m")).toBe(true);
|
|
434
|
+
expect(rendered).toContain("CHANT_BEHAVIOUR_ENGINE");
|
|
435
|
+
}
|
|
436
|
+
expect(broke).toContain("behaviour: refused (engine-out-of-credit)");
|
|
437
|
+
expect(throttled).toContain("behaviour: refused (engine-over-quota)");
|
|
438
|
+
});
|
|
439
|
+
});
|
|
440
|
+
|
|
441
|
+
/* -------------------------------------------------------------------------- */
|
|
442
|
+
/* The request carries edges, not just nodes */
|
|
443
|
+
/* -------------------------------------------------------------------------- */
|
|
444
|
+
|
|
445
|
+
describe("the request is a graph, not a bag of nodes (#2355)", () => {
|
|
446
|
+
test("edges are required — a nodes-only request will not compile", () => {
|
|
447
|
+
// @ts-expect-error the epic's input is "entities ... and the edges between
|
|
448
|
+
// them", so a request without them is not a resource graph.
|
|
449
|
+
const nodesOnly: PredictBehaviourOptions = {
|
|
450
|
+
environment: "prod",
|
|
451
|
+
buildOutput: "/tmp/build",
|
|
452
|
+
entityNames: [...DECLARED.keys()],
|
|
453
|
+
entities: DECLARED,
|
|
454
|
+
traffic: "100 rps, p50",
|
|
455
|
+
};
|
|
456
|
+
expect(nodesOnly).toBeDefined();
|
|
457
|
+
});
|
|
458
|
+
|
|
459
|
+
test("edges reach the engine, and change what it says", async () => {
|
|
460
|
+
const connected = await acmeSim(REACHABLE)!(REQUEST);
|
|
461
|
+
const isolated = await acmeSim(REACHABLE)!({ ...REQUEST, edges: [] });
|
|
462
|
+
if (isBehaviourRefusalReport(connected) || isBehaviourRefusalReport(isolated)) {
|
|
463
|
+
throw new Error("expected reports");
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
// `web` is on both edges, so it carries load and its headroom is spent
|
|
467
|
+
// accordingly. With no edges the same entity is an island.
|
|
468
|
+
expect(connected.entities.web.headroom).toEqual({ cpu: 0.3844, latency: 0.1681 });
|
|
469
|
+
expect(isolated.entities.web.headroom).toEqual({ cpu: 1, latency: 1 });
|
|
470
|
+
expect(connected.entities.web.errorRate).toBe(0.002);
|
|
471
|
+
expect(isolated.entities.web.errorRate).toBe(0);
|
|
472
|
+
});
|
|
473
|
+
|
|
474
|
+
test("edges are `IREdge`, the type both the declared and live paths already produce", () => {
|
|
475
|
+
// Not a structural coincidence: this is the assertion that stops a future
|
|
476
|
+
// change from forking a second edge type for behaviour, which would put a
|
|
477
|
+
// lossy translation on the live side (#2360) and the declared side both.
|
|
478
|
+
const fromGraphIr: IREdge = { from: "web", to: "db", kind: "ref", viaAttr: "dbEndpoint" };
|
|
479
|
+
const request: PredictBehaviourOptions = { ...REQUEST, edges: [fromGraphIr] };
|
|
480
|
+
expect(request.edges[0].kind).toBe("ref");
|
|
481
|
+
});
|
|
482
|
+
});
|
|
483
|
+
|
|
484
|
+
/* -------------------------------------------------------------------------- */
|
|
485
|
+
/* Deltas: the invariant lives here, the presentation does not */
|
|
486
|
+
/* -------------------------------------------------------------------------- */
|
|
487
|
+
|
|
488
|
+
describe("provenance does not survive subtraction (#2358, #2360)", () => {
|
|
489
|
+
const modeled = { engine: "acme-sim", version: "1.4.2", tolerance: "±15%", basis: "modeled" } as const;
|
|
490
|
+
|
|
491
|
+
test("same engine, same basis is a delta of like things", () => {
|
|
492
|
+
expect(compareProvenance(modeled, { ...modeled })).toBe("comparable");
|
|
493
|
+
expect(isComparableProvenance(modeled, { ...modeled })).toBe(true);
|
|
494
|
+
});
|
|
495
|
+
|
|
496
|
+
test("a modeled figure against a validated one is `mixed-basis`", () => {
|
|
497
|
+
const validated = { ...modeled, basis: "validated" } as const;
|
|
498
|
+
expect(compareProvenance(modeled, validated)).toBe("mixed-basis");
|
|
499
|
+
expect(isComparableProvenance(modeled, validated)).toBe(false);
|
|
500
|
+
});
|
|
501
|
+
|
|
502
|
+
test("a different engine, version or tolerance is `mixed-engine`", () => {
|
|
503
|
+
expect(compareProvenance(modeled, { ...modeled, engine: "other-sim" })).toBe("mixed-engine");
|
|
504
|
+
expect(compareProvenance(modeled, { ...modeled, version: "1.5.0" })).toBe("mixed-engine");
|
|
505
|
+
expect(compareProvenance(modeled, { ...modeled, tolerance: "±40%" })).toBe("mixed-engine");
|
|
506
|
+
});
|
|
507
|
+
|
|
508
|
+
test("a mismatched engine outranks a mismatched basis", () => {
|
|
509
|
+
// Both differ. The engine verdict is the one that must survive, because
|
|
510
|
+
// two models are not one scale whatever their bases say.
|
|
511
|
+
expect(
|
|
512
|
+
compareProvenance(modeled, { ...modeled, engine: "other-sim", basis: "validated" }),
|
|
513
|
+
).toBe("mixed-engine");
|
|
514
|
+
});
|
|
515
|
+
|
|
516
|
+
test("two entities in one report can be priced by different engines", async () => {
|
|
517
|
+
// Which is why provenance is per entity, and why a consumer summing across
|
|
518
|
+
// a report has to ask before it subtracts.
|
|
519
|
+
const result = await acmeSim(REACHABLE)!(REQUEST);
|
|
520
|
+
if (isBehaviourRefusalReport(result)) throw new Error("expected a report");
|
|
521
|
+
const foreign = { ...result.entities.db.provenance, engine: "other-sim" };
|
|
522
|
+
expect(compareProvenance(result.entities.web.provenance, foreign)).toBe("mixed-engine");
|
|
523
|
+
});
|
|
524
|
+
|
|
525
|
+
test("two runs of the same estate at different levels are a mixed-level pair", async () => {
|
|
526
|
+
const quiet = await acmeSim(REACHABLE)!(REQUEST);
|
|
527
|
+
const busy = await acmeSim(REACHABLE)!({ ...REQUEST, traffic: "1000 rps, p50" });
|
|
528
|
+
if (isBehaviourRefusalReport(quiet) || isBehaviourRefusalReport(busy)) {
|
|
529
|
+
throw new Error("expected reports");
|
|
530
|
+
}
|
|
531
|
+
expect(figureMismatches(quiet.entities.web, busy.entities.web)).toEqual(["mixed-level"]);
|
|
532
|
+
// And the figures genuinely moved, so the level was read and not echoed.
|
|
533
|
+
expect(busy.entities.web.cost.perHour).toBeGreaterThan(quiet.entities.web.cost.perHour);
|
|
534
|
+
});
|
|
535
|
+
|
|
536
|
+
test("the same estate at two traffic levels is `mixed-level`, not comparable", () => {
|
|
537
|
+
// The gap that mattered: `at` lives on the block, not on provenance, so a
|
|
538
|
+
// provenance-only comparison called 100 rps and 1000 rps a like-for-like
|
|
539
|
+
// delta — and the binding rule is exactly what a consumer follows to draw
|
|
540
|
+
// that delta unmarked.
|
|
541
|
+
const busier: PredictedBehaviour = { ...GOOD, at: { traffic: "1000 rps, p99" } };
|
|
542
|
+
expect(figureMismatches(GOOD, busier)).toEqual(["mixed-level"]);
|
|
543
|
+
expect(isComparableFigure(GOOD, busier)).toBe(false);
|
|
544
|
+
|
|
545
|
+
// The provenance-only function still says comparable, and says so in its
|
|
546
|
+
// own doc. This assertion pins that it is the wrong function to call.
|
|
547
|
+
expect(compareProvenance(GOOD.provenance, busier.provenance)).toBe("comparable");
|
|
548
|
+
});
|
|
549
|
+
|
|
550
|
+
test("same level, same engine, same basis is comparable", () => {
|
|
551
|
+
expect(compareFigures(GOOD, { ...GOOD }).size).toBe(0);
|
|
552
|
+
expect(isComparableFigure(GOOD, { ...GOOD })).toBe(true);
|
|
553
|
+
});
|
|
554
|
+
|
|
555
|
+
test("every mismatching axis is reported, not just the first", () => {
|
|
556
|
+
// The gap: returning one label meant a pair differing in level AND basis
|
|
557
|
+
// reported `mixed-level` and dropped the basis crossing silently, so a
|
|
558
|
+
// consumer captioned it "different traffic level" and showed a
|
|
559
|
+
// modeled-minus-validated difference underneath with nothing said.
|
|
560
|
+
const both: PredictedBehaviour = {
|
|
561
|
+
...GOOD,
|
|
562
|
+
at: { traffic: "1000 rps" },
|
|
563
|
+
provenance: { ...GOOD.provenance, basis: "validated" },
|
|
564
|
+
};
|
|
565
|
+
expect(figureMismatches(GOOD, both)).toEqual(["mixed-level", "mixed-basis"]);
|
|
566
|
+
|
|
567
|
+
const everything: PredictedBehaviour = {
|
|
568
|
+
...GOOD,
|
|
569
|
+
at: { traffic: "1000 rps" },
|
|
570
|
+
cost: predictedRate(0.04, "EUR"),
|
|
571
|
+
resilience: { failure: "region lost", verdict: "fails" },
|
|
572
|
+
provenance: { ...GOOD.provenance, engine: "other-sim", basis: "validated" },
|
|
573
|
+
};
|
|
574
|
+
expect(figureMismatches(GOOD, everything)).toEqual([
|
|
575
|
+
"mixed-engine",
|
|
576
|
+
"mixed-level",
|
|
577
|
+
"mixed-currency",
|
|
578
|
+
"mixed-basis",
|
|
579
|
+
"mixed-failure",
|
|
580
|
+
]);
|
|
581
|
+
});
|
|
582
|
+
|
|
583
|
+
test("USD minus EUR is not a plain difference", () => {
|
|
584
|
+
// behold already refuses to SUM mixed currencies; permitting them to be
|
|
585
|
+
// DIFFERENCED left this contract laxer than its own consumer. chant
|
|
586
|
+
// converts nothing, so the subtraction is not a number.
|
|
587
|
+
const inEuros: PredictedBehaviour = { ...GOOD, cost: predictedRate(0.0416, "EUR") };
|
|
588
|
+
expect(figureMismatches(GOOD, inEuros)).toEqual(["mixed-currency"]);
|
|
589
|
+
expect(isComparableFigure(GOOD, inEuros)).toBe(false);
|
|
590
|
+
});
|
|
591
|
+
|
|
592
|
+
test("two verdicts about two different failures are not a like-for-like delta", () => {
|
|
593
|
+
const regionLost: PredictedBehaviour = {
|
|
594
|
+
...GOOD,
|
|
595
|
+
resilience: { failure: "region lost", verdict: "fails" },
|
|
596
|
+
};
|
|
597
|
+
expect(figureMismatches(GOOD, regionLost)).toEqual(["mixed-failure"]);
|
|
598
|
+
expect(isComparableFigure(GOOD, regionLost)).toBe(false);
|
|
599
|
+
});
|
|
600
|
+
|
|
601
|
+
test("the display order is most fundamental first, and derived from a total witness", () => {
|
|
602
|
+
expect(FIGURE_MISMATCHES).toEqual([
|
|
603
|
+
"mixed-engine",
|
|
604
|
+
"mixed-level",
|
|
605
|
+
"mixed-currency",
|
|
606
|
+
"mixed-basis",
|
|
607
|
+
"mixed-failure",
|
|
608
|
+
]);
|
|
609
|
+
|
|
610
|
+
// The witness is what stops the array drifting from the union. Without it,
|
|
611
|
+
// a sixth axis added to `FigureMismatch` and to `compareFigures` but not to
|
|
612
|
+
// the array is reported by the set and silently dropped by
|
|
613
|
+
// `figureMismatches` — the "a mismatch went unsaid" failure the set return
|
|
614
|
+
// was added to prevent, reappearing in the display path.
|
|
615
|
+
const witness: Record<FigureMismatch, true> = {
|
|
616
|
+
"mixed-engine": true,
|
|
617
|
+
"mixed-level": true,
|
|
618
|
+
"mixed-currency": true,
|
|
619
|
+
"mixed-basis": true,
|
|
620
|
+
"mixed-failure": true,
|
|
621
|
+
};
|
|
622
|
+
expect([...FIGURE_MISMATCHES].sort()).toEqual(Object.keys(witness).sort());
|
|
623
|
+
});
|
|
624
|
+
|
|
625
|
+
test("nothing compareFigures can return is dropped by figureMismatches", () => {
|
|
626
|
+
// The two must agree on membership. This is the pairing the witness makes
|
|
627
|
+
// structural; asserting it here keeps the property visible in the suite.
|
|
628
|
+
const everything: PredictedBehaviour = {
|
|
629
|
+
...GOOD,
|
|
630
|
+
at: { traffic: "1000 rps" },
|
|
631
|
+
cost: predictedRate(0.04, "EUR"),
|
|
632
|
+
resilience: { failure: "region lost", verdict: "fails" },
|
|
633
|
+
provenance: { ...GOOD.provenance, engine: "other-sim", basis: "validated" },
|
|
634
|
+
};
|
|
635
|
+
const set = compareFigures(GOOD, everything);
|
|
636
|
+
expect(figureMismatches(GOOD, everything)).toHaveLength(set.size);
|
|
637
|
+
for (const axis of set) expect(FIGURE_MISMATCHES).toContain(axis);
|
|
638
|
+
});
|
|
639
|
+
|
|
640
|
+
test("a report cannot hold entities at different levels in the first place", () => {
|
|
641
|
+
// The other half: `behaviourReport` refuses a block whose `at` disagrees
|
|
642
|
+
// with `meta.at`, so a mixed-level pair can only arise across two reports.
|
|
643
|
+
expect(() =>
|
|
644
|
+
behaviourReport(req(["web", "db"]), STAMP, {
|
|
645
|
+
web: GOOD,
|
|
646
|
+
db: { ...GOOD, at: { traffic: "1000 rps" } },
|
|
647
|
+
}),
|
|
648
|
+
).toThrow(/One run, one level/);
|
|
649
|
+
});
|
|
650
|
+
});
|
|
651
|
+
|
|
652
|
+
/* -------------------------------------------------------------------------- */
|
|
653
|
+
/* Totality, and the 18 rules behold applies on arrival */
|
|
654
|
+
/* -------------------------------------------------------------------------- */
|
|
655
|
+
|
|
656
|
+
const STAMP = { engine: "acme-sim", version: "1.4.2" };
|
|
657
|
+
|
|
658
|
+
/**
|
|
659
|
+
* The half of a request `behaviourReport` reads. Everything that must agree
|
|
660
|
+
* between the request and the report now comes from one object, so a test
|
|
661
|
+
* cannot accidentally construct a disagreement the real path could not.
|
|
662
|
+
*/
|
|
663
|
+
const req = (
|
|
664
|
+
entityNames: string[],
|
|
665
|
+
traffic = "100 rps, p50",
|
|
666
|
+
): Pick<PredictBehaviourOptions, "entityNames" | "traffic" | "edgeCoverage"> => ({
|
|
667
|
+
entityNames,
|
|
668
|
+
traffic,
|
|
669
|
+
edgeCoverage: { verdict: "complete" },
|
|
670
|
+
});
|
|
671
|
+
const GOOD: PredictedBehaviour = {
|
|
672
|
+
at: { traffic: "100 rps, p50" },
|
|
673
|
+
cost: predictedRate(0.0416, "USD"),
|
|
674
|
+
headroom: { cpu: 0.62, latency: 0.41 },
|
|
675
|
+
errorRate: 0.001,
|
|
676
|
+
resilience: { failure: "one zone lost", verdict: "survives" },
|
|
677
|
+
provenance: { engine: "acme-sim", version: "1.4.2", tolerance: "±15%", basis: "modeled" },
|
|
678
|
+
};
|
|
679
|
+
|
|
680
|
+
describe("totality is enforced, not merely claimed (#2356)", () => {
|
|
681
|
+
test("an entity in neither map is refused, named", () => {
|
|
682
|
+
expect(() => behaviourReport(req(["web", "cache"]), STAMP, { web: GOOD })).toThrow(
|
|
683
|
+
/gave no verdict at all for "cache"/,
|
|
684
|
+
);
|
|
685
|
+
});
|
|
686
|
+
|
|
687
|
+
test("the empty report for a non-empty request is refused", () => {
|
|
688
|
+
// The reproduction: `behaviourReport(meta, {}, {})` used to return a
|
|
689
|
+
// well-formed report claiming nothing, because it never saw entityNames.
|
|
690
|
+
expect(() => behaviourReport(req(["web"]), STAMP, {}, {})).toThrow(/gave no verdict at all for "web"/);
|
|
691
|
+
});
|
|
692
|
+
|
|
693
|
+
test("an entity in both maps is refused", () => {
|
|
694
|
+
expect(() =>
|
|
695
|
+
behaviourReport(req(["web"]), STAMP, { web: GOOD }, { web: { reason: "unsupported-kind" } }),
|
|
696
|
+
).toThrow(/both priced and unpriced/);
|
|
697
|
+
});
|
|
698
|
+
|
|
699
|
+
test("a figure for something nobody asked about is refused", () => {
|
|
700
|
+
expect(() => behaviourReport(req(["web"]), STAMP, { web: GOOD, ghost: GOOD })).toThrow(
|
|
701
|
+
/returned a figure for "ghost", which was not in entityNames/,
|
|
702
|
+
);
|
|
703
|
+
});
|
|
704
|
+
|
|
705
|
+
test("an unpredicted entry with a bogus reason is refused", () => {
|
|
706
|
+
expect(() =>
|
|
707
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
708
|
+
behaviourReport(req(["web"]), STAMP, {}, { web: { reason: "no-credentials" as any } }),
|
|
709
|
+
).toThrow(/is not one of/);
|
|
710
|
+
});
|
|
711
|
+
|
|
712
|
+
test("the check is a check, not a proof — and the doc says so", () => {
|
|
713
|
+
// `BehaviourReport` is a plain interface. A lexicon that hands its own keys
|
|
714
|
+
// back as the asked-for list self-certifies, and a hand-built object skips
|
|
715
|
+
// the constructor entirely. What passing the request buys is that the
|
|
716
|
+
// ordinary route is checked and the check names what is wrong; a consumer
|
|
717
|
+
// needing the guarantee validates on arrival, which is #2358's and #2360's.
|
|
718
|
+
const selfCertified = behaviourReport(req(["web"]), STAMP, { web: GOOD });
|
|
719
|
+
const handBuilt = { ...selfCertified, entities: {} };
|
|
720
|
+
expect(isBehaviourResult(handBuilt)).toBe(true);
|
|
721
|
+
expect(Object.keys(handBuilt.entities)).toEqual([]);
|
|
722
|
+
});
|
|
723
|
+
|
|
724
|
+
test("a whole valid report still builds", () => {
|
|
725
|
+
const ok = behaviourReport(req(["web", "queue"]), STAMP, { web: GOOD }, {
|
|
726
|
+
queue: { reason: "unsupported-kind" },
|
|
727
|
+
});
|
|
728
|
+
expect(ok.entities.web).toBe(GOOD);
|
|
729
|
+
expect(ok.unpredicted?.queue.reason).toBe("unsupported-kind");
|
|
730
|
+
});
|
|
731
|
+
});
|
|
732
|
+
|
|
733
|
+
describe("every rule behold applies on arrival is applied here first (#2356)", () => {
|
|
734
|
+
// behold/src/behaviour.ts drops a block failing any of these, so a block
|
|
735
|
+
// chant accepts and behold rejects renders nothing while looking legal.
|
|
736
|
+
const cases: [string, PredictedBehaviour, RegExp][] = [
|
|
737
|
+
["2 at.traffic empty", { ...GOOD, at: { traffic: "" } }, /at\.traffic is missing/],
|
|
738
|
+
["3 perHour not finite", { ...GOOD, cost: { rate: "per-hour", perHour: NaN, currency: "USD" } }, /not a finite number/],
|
|
739
|
+
["4 perHour negative", { ...GOOD, cost: { rate: "per-hour", perHour: -1, currency: "USD" } }, /is negative/],
|
|
740
|
+
["5 currency empty", { ...GOOD, cost: { rate: "per-hour", perHour: 1, currency: "" } }, /currency is missing/],
|
|
741
|
+
["7 cpu out of range", { ...GOOD, headroom: { cpu: 1.4 } }, /headroom\.cpu is not a fraction/],
|
|
742
|
+
["8 latency out of range", { ...GOOD, headroom: { latency: -0.1 } }, /headroom\.latency is not a fraction/],
|
|
743
|
+
["9 no axis at all", { ...GOOD, headroom: {} as never }, /neither cpu nor latency/],
|
|
744
|
+
["10 errorRate out of range", { ...GOOD, errorRate: 1.2 }, /errorRate is not a fraction/],
|
|
745
|
+
["11 failure empty", { ...GOOD, resilience: { failure: "", verdict: "survives" } }, /failure is missing/],
|
|
746
|
+
["12 bogus verdict", { ...GOOD, resilience: { failure: "one zone lost", verdict: "melts" as never } }, /not survives\/degrades\/fails/],
|
|
747
|
+
["13 rightSize with no suggestion", { ...GOOD, rightSize: { suggestion: "" } }, /rightSize is present without a suggestion/],
|
|
748
|
+
["14 engine empty", { ...GOOD, provenance: { ...GOOD.provenance, engine: "" } }, /provenance\.engine is missing/],
|
|
749
|
+
["15 version empty", { ...GOOD, provenance: { ...GOOD.provenance, version: "" } }, /provenance\.version is missing/],
|
|
750
|
+
["16 tolerance empty", { ...GOOD, provenance: { ...GOOD.provenance, tolerance: " " } }, /tolerance is missing/],
|
|
751
|
+
["17 bogus basis", { ...GOOD, provenance: { ...GOOD.provenance, basis: "estimated" as never } }, /is not modeled\/validated/],
|
|
752
|
+
];
|
|
753
|
+
|
|
754
|
+
for (const [label, block, expected] of cases) {
|
|
755
|
+
test(`rejects ${label}`, () => {
|
|
756
|
+
expect(() => validateBehaviourBlock("web", block)).toThrow(expected);
|
|
757
|
+
expect(() => behaviourReport(req(["web"]), STAMP, { web: block })).toThrow(expected);
|
|
758
|
+
});
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
test("a failure that names no failure is rejected, where behold would accept it", () => {
|
|
762
|
+
// `"none"` passes behold's non-empty check and renders a confident verdict
|
|
763
|
+
// about nothing — and this module's own doc already said a verdict with no
|
|
764
|
+
// named failure says nothing.
|
|
765
|
+
for (const word of ["none", "None", "n/a", "unknown", "-", "TBD"]) {
|
|
766
|
+
expect(
|
|
767
|
+
() =>
|
|
768
|
+
validateBehaviourBlock("web", {
|
|
769
|
+
...GOOD,
|
|
770
|
+
resilience: { failure: word, verdict: "survives" },
|
|
771
|
+
}),
|
|
772
|
+
`${word} was accepted as a failure`,
|
|
773
|
+
).toThrow(/names no failure/);
|
|
774
|
+
}
|
|
775
|
+
expect(() =>
|
|
776
|
+
validateBehaviourBlock("web", {
|
|
777
|
+
...GOOD,
|
|
778
|
+
resilience: { failure: "primary database failover", verdict: "degrades" },
|
|
779
|
+
}),
|
|
780
|
+
).not.toThrow();
|
|
781
|
+
});
|
|
782
|
+
|
|
783
|
+
test("a tolerance that states no tolerance is rejected, where behold would accept it", () => {
|
|
784
|
+
// behold takes any non-empty string. The epic asks for the engine's STATED
|
|
785
|
+
// tolerance, and "n/a" passes behold's check while defeating its purpose.
|
|
786
|
+
for (const word of ["n/a", "N/A", "none", "unknown", "-", "TBD"]) {
|
|
787
|
+
expect(
|
|
788
|
+
() => validateBehaviourBlock("web", { ...GOOD, provenance: { ...GOOD.provenance, tolerance: word } }),
|
|
789
|
+
`${word} was accepted`,
|
|
790
|
+
).toThrow(/states no tolerance/);
|
|
791
|
+
}
|
|
792
|
+
expect(() =>
|
|
793
|
+
validateBehaviourBlock("web", { ...GOOD, provenance: { ...GOOD.provenance, tolerance: "±80%" } }),
|
|
794
|
+
).not.toThrow();
|
|
795
|
+
});
|
|
796
|
+
|
|
797
|
+
test("a block priced at a level the run did not ask for is refused", () => {
|
|
798
|
+
expect(() =>
|
|
799
|
+
behaviourReport(req(["web"]), STAMP, { web: { ...GOOD, at: { traffic: "1000 rps, p99" } } }),
|
|
800
|
+
).toThrow(/One run, one level/);
|
|
801
|
+
});
|
|
802
|
+
|
|
803
|
+
test("report-level fields are checked too", () => {
|
|
804
|
+
expect(() => behaviourReport(req([]), { ...STAMP, engine: "" }, {})).toThrow(/meta\.engine is missing/);
|
|
805
|
+
expect(() => behaviourReport(req([]), { ...STAMP, version: "" }, {})).toThrow(/meta\.version is missing/);
|
|
806
|
+
expect(() => behaviourReport(req([], ""), STAMP, {})).toThrow(/meta\.at\.traffic is missing/);
|
|
807
|
+
});
|
|
808
|
+
|
|
809
|
+
test("headroom with no axis will not even compile", () => {
|
|
810
|
+
// @ts-expect-error the union requires cpu or latency. Rule 9 is a type as
|
|
811
|
+
// well as a runtime check, and the runtime check is the JavaScript backstop.
|
|
812
|
+
const empty: BehaviourHeadroom = {};
|
|
813
|
+
expect(empty).toBeDefined();
|
|
814
|
+
});
|
|
815
|
+
});
|
|
816
|
+
|
|
817
|
+
describe("the conformance suite refuses to prove nothing (#2356)", () => {
|
|
818
|
+
const refusalOnly = {
|
|
819
|
+
lexicon: "never-predicts",
|
|
820
|
+
scenarios: [
|
|
821
|
+
{ name: "no engine", declared: ["web"], run: () => acmeSim(NOTHING_CONFIGURED)!(REQUEST), expectRefusal: true },
|
|
822
|
+
{ name: "engine down", declared: ["web"], run: () => acmeSim(CONFIGURED_BUT_DOWN)!(REQUEST), expectRefusal: true },
|
|
823
|
+
],
|
|
824
|
+
};
|
|
825
|
+
|
|
826
|
+
test("a lexicon that only ever refuses cannot pass by marking everything a refusal", () => {
|
|
827
|
+
expect(behaviourConformanceGaps(refusalOnly)[0]).toBe(
|
|
828
|
+
"every scenario is a refusal — this suite would pass a lexicon that never predicts anything",
|
|
829
|
+
);
|
|
830
|
+
});
|
|
831
|
+
|
|
832
|
+
test("a predicting scenario that hides the traffic level it asked for is a gap", () => {
|
|
833
|
+
const noLevel = {
|
|
834
|
+
lexicon: "hides-its-level",
|
|
835
|
+
scenarios: [{ name: "up", declared: ["web"], run: () => acmeSim(REACHABLE)!(REQUEST) }],
|
|
836
|
+
};
|
|
837
|
+
expect(behaviourConformanceGaps(noLevel)).toContain(
|
|
838
|
+
'scenario "up" predicts but does not state the traffic level it requested',
|
|
839
|
+
);
|
|
840
|
+
});
|
|
841
|
+
|
|
842
|
+
test("a config the suite can only ask once is a gap — echo is not use", () => {
|
|
843
|
+
// Without `request` and `predict` the suite holds one answer and cannot
|
|
844
|
+
// tell a lexicon that reads `traffic` and `edges` from one that echoes the
|
|
845
|
+
// level and ignores the graph.
|
|
846
|
+
const askOnce = {
|
|
847
|
+
lexicon: "opaque-closure",
|
|
848
|
+
scenarios: [
|
|
849
|
+
{
|
|
850
|
+
name: "up",
|
|
851
|
+
declared: ["web"],
|
|
852
|
+
traffic: "100 rps, p50",
|
|
853
|
+
run: () => acmeSim(REACHABLE)!(REQUEST),
|
|
854
|
+
},
|
|
855
|
+
],
|
|
856
|
+
};
|
|
857
|
+
expect(behaviourConformanceGaps(askOnce).join(" ")).toMatch(/can only ask once/);
|
|
858
|
+
});
|
|
859
|
+
|
|
860
|
+
test("a probe with no second traffic level is a gap — there is no safe default", () => {
|
|
861
|
+
// The old default was `"<level> ×10 (conformance probe)"`, which is a level
|
|
862
|
+
// no engine has agreed to understand. The contract says an engine that
|
|
863
|
+
// cannot understand a level refuses rather than substituting one, so the
|
|
864
|
+
// default failed a contract-correct lexicon for obeying the contract.
|
|
865
|
+
const noSecondLevel = {
|
|
866
|
+
lexicon: "one-level",
|
|
867
|
+
scenarios: [
|
|
868
|
+
{
|
|
869
|
+
name: "up",
|
|
870
|
+
declared: ["web"],
|
|
871
|
+
traffic: "100 rps, p50",
|
|
872
|
+
run: () => acmeSim(REACHABLE)!(REQUEST),
|
|
873
|
+
request: REQUEST,
|
|
874
|
+
predict: (o: PredictBehaviourOptions) => acmeSim(REACHABLE)!(o),
|
|
875
|
+
},
|
|
876
|
+
],
|
|
877
|
+
};
|
|
878
|
+
expect(behaviourConformanceGaps(noSecondLevel).join(" ")).toMatch(/no safe default/);
|
|
879
|
+
});
|
|
880
|
+
|
|
881
|
+
test("a probe whose request cannot exercise the edge probe is a gap", () => {
|
|
882
|
+
// The edge probe drops one edge and requires the answer to change, so a
|
|
883
|
+
// request with fewer than two edges makes it vacuous — and a sibling wiring
|
|
884
|
+
// up a minimal edgeless request would pass "reads the edges it was handed"
|
|
885
|
+
// without the probe ever running.
|
|
886
|
+
const thinGraph = {
|
|
887
|
+
lexicon: "edgeless",
|
|
888
|
+
scenarios: [
|
|
889
|
+
{
|
|
890
|
+
name: "up",
|
|
891
|
+
declared: ["web"],
|
|
892
|
+
traffic: "100 rps, p50",
|
|
893
|
+
otherTraffic: "1000 rps, p50",
|
|
894
|
+
run: () => acmeSim(REACHABLE)!(REQUEST),
|
|
895
|
+
request: { ...REQUEST, edges: [] },
|
|
896
|
+
predict: (o: PredictBehaviourOptions) => acmeSim(REACHABLE)!(o),
|
|
897
|
+
},
|
|
898
|
+
],
|
|
899
|
+
};
|
|
900
|
+
expect(behaviourConformanceGaps(thinGraph).join(" ")).toMatch(/carries 0 edge\(s\)/);
|
|
901
|
+
});
|
|
902
|
+
|
|
903
|
+
test("the fixture's own config has no gaps", () => {
|
|
904
|
+
expect(
|
|
905
|
+
behaviourConformanceGaps({
|
|
906
|
+
lexicon: "acme-sim (fixture)",
|
|
907
|
+
scenarios: [
|
|
908
|
+
{
|
|
909
|
+
name: "up",
|
|
910
|
+
declared: ["web"],
|
|
911
|
+
traffic: "100 rps, p50",
|
|
912
|
+
run: () => acmeSim(REACHABLE)!(REQUEST),
|
|
913
|
+
request: REQUEST,
|
|
914
|
+
predict: (options) => acmeSim(REACHABLE)!(options),
|
|
915
|
+
otherTraffic: "1000 rps, p50",
|
|
916
|
+
},
|
|
917
|
+
{ name: "down", declared: ["web"], run: () => acmeSim(CONFIGURED_BUT_DOWN)!(REQUEST), expectRefusal: true },
|
|
918
|
+
],
|
|
919
|
+
}),
|
|
920
|
+
).toEqual([]);
|
|
921
|
+
});
|
|
922
|
+
});
|
|
923
|
+
|
|
924
|
+
describe("the lazy lexicon the old suite green-lit (#2356)", () => {
|
|
925
|
+
/**
|
|
926
|
+
* A lexicon that satisfies every type on this contract and is useless: it
|
|
927
|
+
* ignores the traffic level it was handed, ignores the edges, prices
|
|
928
|
+
* everything at zero, reports no headroom axis at all, and states `n/a` for
|
|
929
|
+
* its tolerance. Every one of those passed the conformance suite before the
|
|
930
|
+
* behold rule set landed here, and behold would have dropped every block.
|
|
931
|
+
*/
|
|
932
|
+
const lazyBlock = (): PredictedBehaviour =>
|
|
933
|
+
({
|
|
934
|
+
at: { traffic: "whatever" },
|
|
935
|
+
cost: predictedRate(0, "USD"),
|
|
936
|
+
headroom: {},
|
|
937
|
+
errorRate: 0,
|
|
938
|
+
resilience: { failure: "one zone lost", verdict: "survives" },
|
|
939
|
+
provenance: { engine: "lazy", version: "0", tolerance: "n/a", basis: "modeled" },
|
|
940
|
+
}) as unknown as PredictedBehaviour;
|
|
941
|
+
|
|
942
|
+
test("it cannot build a report at all — the validator stops it first", () => {
|
|
943
|
+
expect(() =>
|
|
944
|
+
behaviourReport(req(["web"], "whatever"), { engine: "lazy", version: "0" }, {
|
|
945
|
+
web: lazyBlock(),
|
|
946
|
+
}),
|
|
947
|
+
).toThrow(/neither cpu nor latency/);
|
|
948
|
+
});
|
|
949
|
+
|
|
950
|
+
test("each of its sins is caught by name", () => {
|
|
951
|
+
const withHeadroom = { ...lazyBlock(), headroom: { cpu: 0.5 } };
|
|
952
|
+
expect(() => validateBehaviourBlock("web", withHeadroom)).toThrow(/states no tolerance/);
|
|
953
|
+
|
|
954
|
+
const withTolerance = {
|
|
955
|
+
...withHeadroom,
|
|
956
|
+
provenance: { ...withHeadroom.provenance, tolerance: "±20%" },
|
|
957
|
+
};
|
|
958
|
+
// Zero cost is legal on its own — a free tier is a real answer — so the
|
|
959
|
+
// suite catches an ignored traffic level through `meta.at` instead.
|
|
960
|
+
expect(() => validateBehaviourBlock("web", withTolerance)).not.toThrow();
|
|
961
|
+
expect(() =>
|
|
962
|
+
behaviourReport(req(["web"]), { engine: "lazy", version: "0" }, { web: withTolerance }),
|
|
963
|
+
).toThrow(/One run, one level/);
|
|
964
|
+
});
|
|
965
|
+
});
|
|
966
|
+
|
|
967
|
+
describe("the second lazy lexicon, which passed 28 of 28 (#2356)", () => {
|
|
968
|
+
/**
|
|
969
|
+
* Sharper than the first. Every field is well-formed and every figure is
|
|
970
|
+
* vacuous: nothing costs anything, nothing is under load, nothing ever fails,
|
|
971
|
+
* the tolerance is a real number that happens to be zero, and the traffic
|
|
972
|
+
* level is echoed back without being read. `edges` and `edgeCoverage` are
|
|
973
|
+
* never opened.
|
|
974
|
+
*/
|
|
975
|
+
const lazy = async (options: PredictBehaviourOptions): Promise<BehaviourResult> => {
|
|
976
|
+
const entities: Record<string, PredictedBehaviour> = {};
|
|
977
|
+
for (const name of options.entityNames) {
|
|
978
|
+
entities[name] = {
|
|
979
|
+
at: { traffic: options.traffic },
|
|
980
|
+
cost: predictedRate(0, "USD"),
|
|
981
|
+
headroom: { cpu: 1 },
|
|
982
|
+
errorRate: 0,
|
|
983
|
+
resilience: { failure: "none", verdict: "survives" },
|
|
984
|
+
provenance: { engine: "lazy", version: "1", tolerance: "±0%", basis: "modeled" },
|
|
985
|
+
};
|
|
986
|
+
}
|
|
987
|
+
return behaviourReport(options, { engine: "lazy", version: "1" }, entities);
|
|
988
|
+
};
|
|
989
|
+
|
|
990
|
+
test("its `resilience.failure: \"none\"` is now refused at construction", async () => {
|
|
991
|
+
await expect(lazy(REQUEST)).rejects.toThrow(/names no failure/);
|
|
992
|
+
});
|
|
993
|
+
|
|
994
|
+
test("even with a real failure named, it answers every traffic level identically", async () => {
|
|
995
|
+
const honest = async (options: PredictBehaviourOptions): Promise<BehaviourResult> => {
|
|
996
|
+
const entities: Record<string, PredictedBehaviour> = {};
|
|
997
|
+
for (const name of options.entityNames) {
|
|
998
|
+
entities[name] = {
|
|
999
|
+
at: { traffic: options.traffic },
|
|
1000
|
+
cost: predictedRate(0, "USD"),
|
|
1001
|
+
headroom: { cpu: 1 },
|
|
1002
|
+
errorRate: 0,
|
|
1003
|
+
resilience: { failure: "one zone lost", verdict: "survives" },
|
|
1004
|
+
provenance: { engine: "lazy", version: "1", tolerance: "±0%", basis: "modeled" },
|
|
1005
|
+
};
|
|
1006
|
+
}
|
|
1007
|
+
return behaviourReport(options, { engine: "lazy", version: "1" }, entities);
|
|
1008
|
+
};
|
|
1009
|
+
|
|
1010
|
+
const quiet = await honest(REQUEST);
|
|
1011
|
+
const busy = await honest({ ...REQUEST, traffic: "1000000 rps" });
|
|
1012
|
+
if (isBehaviourRefusalReport(quiet) || isBehaviourRefusalReport(busy)) {
|
|
1013
|
+
throw new Error("expected reports");
|
|
1014
|
+
}
|
|
1015
|
+
// The level is echoed and the figures do not move. This is what the
|
|
1016
|
+
// conformance suite's two-level probe exists to catch, and a suite holding
|
|
1017
|
+
// one opaque `run()` closure could not see it at all.
|
|
1018
|
+
expect(busy.meta.at.traffic).toBe("1000000 rps");
|
|
1019
|
+
expect(quiet.entities.web.cost.perHour).toBe(busy.entities.web.cost.perHour);
|
|
1020
|
+
expect(quiet.entities.web.headroom).toEqual(busy.entities.web.headroom);
|
|
1021
|
+
});
|
|
1022
|
+
|
|
1023
|
+
test("removing every edge changes nothing it says", async () => {
|
|
1024
|
+
const honest = async (options: PredictBehaviourOptions): Promise<BehaviourResult> => {
|
|
1025
|
+
const entities: Record<string, PredictedBehaviour> = {};
|
|
1026
|
+
for (const name of options.entityNames) {
|
|
1027
|
+
entities[name] = {
|
|
1028
|
+
at: { traffic: options.traffic },
|
|
1029
|
+
cost: predictedRate(0, "USD"),
|
|
1030
|
+
headroom: { cpu: 1 },
|
|
1031
|
+
errorRate: 0,
|
|
1032
|
+
resilience: { failure: "one zone lost", verdict: "survives" },
|
|
1033
|
+
provenance: { engine: "lazy", version: "1", tolerance: "±0%", basis: "modeled" },
|
|
1034
|
+
};
|
|
1035
|
+
}
|
|
1036
|
+
return behaviourReport(options, { engine: "lazy", version: "1" }, entities);
|
|
1037
|
+
};
|
|
1038
|
+
const withEdges = await honest(REQUEST);
|
|
1039
|
+
const without = await honest({ ...REQUEST, edges: [], edgeCoverage: { verdict: "unknown" } });
|
|
1040
|
+
if (isBehaviourRefusalReport(withEdges) || isBehaviourRefusalReport(without)) {
|
|
1041
|
+
throw new Error("expected reports");
|
|
1042
|
+
}
|
|
1043
|
+
expect(withEdges.entities.web.headroom).toEqual(without.entities.web.headroom);
|
|
1044
|
+
});
|
|
1045
|
+
|
|
1046
|
+
test("the fixture lexicon, by contrast, moves on both axes", async () => {
|
|
1047
|
+
const quiet = await acmeSim(REACHABLE)!(REQUEST);
|
|
1048
|
+
const busy = await acmeSim(REACHABLE)!({ ...REQUEST, traffic: "1000 rps, p50" });
|
|
1049
|
+
const noEdges = await acmeSim(REACHABLE)!({
|
|
1050
|
+
...REQUEST,
|
|
1051
|
+
edges: [],
|
|
1052
|
+
edgeCoverage: { verdict: "unknown" },
|
|
1053
|
+
});
|
|
1054
|
+
if (
|
|
1055
|
+
isBehaviourRefusalReport(quiet) ||
|
|
1056
|
+
isBehaviourRefusalReport(busy) ||
|
|
1057
|
+
isBehaviourRefusalReport(noEdges)
|
|
1058
|
+
) {
|
|
1059
|
+
throw new Error("expected reports");
|
|
1060
|
+
}
|
|
1061
|
+
expect(busy.entities.web.cost.perHour).not.toBe(quiet.entities.web.cost.perHour);
|
|
1062
|
+
expect(noEdges.entities.web.headroom).not.toEqual(quiet.entities.web.headroom);
|
|
1063
|
+
});
|
|
1064
|
+
});
|
|
1065
|
+
|
|
1066
|
+
describe("the dodge lexicon, which passed 16 of 16 (#2356)", () => {
|
|
1067
|
+
/**
|
|
1068
|
+
* The sharpest of the three. It never reads an edge and prices the estate off
|
|
1069
|
+
* the **character count of the traffic label** — and it passed every
|
|
1070
|
+
* conformance assertion, including "reads the edges it was handed" and
|
|
1071
|
+
* "echoes the edge coverage", by refusing exactly the two probes that would
|
|
1072
|
+
* have caught it. Both probes accepted a refusal as an answer.
|
|
1073
|
+
*/
|
|
1074
|
+
const dodge = async (o: PredictBehaviourOptions): Promise<BehaviourResult> => {
|
|
1075
|
+
if (o.edges.length === 0 || o.edgeCoverage.verdict === "unknown") {
|
|
1076
|
+
return noBehaviourEngineRefusal("dodge");
|
|
1077
|
+
}
|
|
1078
|
+
const entities: Record<string, PredictedBehaviour> = {};
|
|
1079
|
+
for (const name of o.entityNames) {
|
|
1080
|
+
entities[name] = {
|
|
1081
|
+
at: { traffic: o.traffic },
|
|
1082
|
+
// Varies with the traffic *string*, so the level probe is satisfied
|
|
1083
|
+
// while nothing about the level is understood.
|
|
1084
|
+
cost: predictedRate(o.traffic.length / 100, "USD"),
|
|
1085
|
+
headroom: { cpu: 0.5 },
|
|
1086
|
+
errorRate: 0.001,
|
|
1087
|
+
resilience: { failure: "one zone lost", verdict: "survives" },
|
|
1088
|
+
provenance: { engine: "dodge", version: "1", tolerance: "±10%", basis: "modeled" },
|
|
1089
|
+
};
|
|
1090
|
+
}
|
|
1091
|
+
return behaviourReport(o, { engine: "dodge", version: "1" }, entities);
|
|
1092
|
+
};
|
|
1093
|
+
|
|
1094
|
+
// The real probe bodies, run against the dodge. These are the assertions the
|
|
1095
|
+
// conformance suite makes; a weakening of either shows up here as green.
|
|
1096
|
+
test("the coverage probe now complains instead of skipping", async () => {
|
|
1097
|
+
expect(await probeEchoesCoverage(REQUEST, dodge)).toEqual([
|
|
1098
|
+
'refused a request whose graph is intact and whose only change is an "unknown" coverage ' +
|
|
1099
|
+
"verdict (cause: no-engine) — there is nothing here to refuse",
|
|
1100
|
+
]);
|
|
1101
|
+
});
|
|
1102
|
+
|
|
1103
|
+
test("the edge probe leaves it nothing to refuse, and catches that nothing changed", async () => {
|
|
1104
|
+
// Dropping ONE edge keeps the coverage claim true, so the dodge's refusal
|
|
1105
|
+
// condition (`edges.length === 0`) never fires and it has to answer. Its
|
|
1106
|
+
// answer is identical, because it never looked at the graph.
|
|
1107
|
+
expect(await probeReadsEdges(REQUEST, dodge)).toEqual([
|
|
1108
|
+
"removing an edge changed nothing — the graph is being priced as a bag of nodes",
|
|
1109
|
+
]);
|
|
1110
|
+
});
|
|
1111
|
+
|
|
1112
|
+
test("it satisfies the traffic probe, which is why the other two must hold", async () => {
|
|
1113
|
+
// Pricing off `traffic.length` moves the figures between two levels, so the
|
|
1114
|
+
// level probe passes. This is the assertion that shows why the edge and
|
|
1115
|
+
// coverage probes carry the weight.
|
|
1116
|
+
expect(await probeTrafficLevel(REQUEST, dodge, "1000 rps, p50")).toEqual([]);
|
|
1117
|
+
});
|
|
1118
|
+
|
|
1119
|
+
test("the fixture passes all four probes", async () => {
|
|
1120
|
+
const predict = (o: PredictBehaviourOptions): Promise<BehaviourResult> =>
|
|
1121
|
+
acmeSim(REACHABLE)!(o);
|
|
1122
|
+
expect(await probeTrafficLevel(REQUEST, predict, "1000 rps, p50")).toEqual([]);
|
|
1123
|
+
expect(await probeReadsEdges(REQUEST, predict)).toEqual([]);
|
|
1124
|
+
expect(await probeEdgelessConsistency(REQUEST, predict)).toEqual([]);
|
|
1125
|
+
expect(await probeEchoesCoverage(REQUEST, predict)).toEqual([]);
|
|
1126
|
+
});
|
|
1127
|
+
});
|
|
1128
|
+
|
|
1129
|
+
describe("the report's account of its own inputs cannot be edited afterwards (#2360)", () => {
|
|
1130
|
+
test("edgeCoverage is copied, not aliased", () => {
|
|
1131
|
+
// `readonly` is erased at runtime, so assigning the request's object by
|
|
1132
|
+
// reference let a caller mutate `unresolvedKinds` after construction and
|
|
1133
|
+
// change what the report claimed it was computed over.
|
|
1134
|
+
const coverage = {
|
|
1135
|
+
verdict: "partial" as const,
|
|
1136
|
+
unresolvedKinds: ["AWS::SQS::Queue"],
|
|
1137
|
+
dangling: [{ from: "web", path: "vpcId", value: "vpc-1" }],
|
|
1138
|
+
};
|
|
1139
|
+
const report = behaviourReport({ ...req(["web"]), edgeCoverage: coverage }, STAMP, {
|
|
1140
|
+
web: GOOD,
|
|
1141
|
+
});
|
|
1142
|
+
coverage.unresolvedKinds.push("AWS::SNS::Topic");
|
|
1143
|
+
coverage.dangling[0].from = "somewhere-else";
|
|
1144
|
+
|
|
1145
|
+
expect(report.meta.edgeCoverage.unresolvedKinds).toEqual(["AWS::SQS::Queue"]);
|
|
1146
|
+
expect(report.meta.edgeCoverage.dangling?.[0].from).toBe("web");
|
|
1147
|
+
});
|
|
1148
|
+
|
|
1149
|
+
test("the copy is frozen, so a consumer cannot edit it either", () => {
|
|
1150
|
+
const copy = copyEdgeCoverage({ verdict: "partial", unresolvedKinds: ["a"] });
|
|
1151
|
+
expect(Object.isFrozen(copy)).toBe(true);
|
|
1152
|
+
expect(Object.isFrozen(copy.unresolvedKinds)).toBe(true);
|
|
1153
|
+
});
|
|
1154
|
+
});
|
|
1155
|
+
|
|
1156
|
+
/* -------------------------------------------------------------------------- */
|
|
1157
|
+
/* The envelope guard */
|
|
1158
|
+
/* -------------------------------------------------------------------------- */
|
|
1159
|
+
|
|
1160
|
+
describe("isBehaviourResult requires an arm, not just the version (#2356)", () => {
|
|
1161
|
+
test("a bare discriminant is not a result", () => {
|
|
1162
|
+
// It used to pass, then fail isBehaviourRefusalReport, narrow to
|
|
1163
|
+
// BehaviourReport, and hand a consumer `undefined` for `entities`.
|
|
1164
|
+
expect(isBehaviourResult({ behaviour: "v1" })).toBe(false);
|
|
1165
|
+
expect(isBehaviourResult({ behaviour: "v1", entities: {} })).toBe(false);
|
|
1166
|
+
expect(isBehaviourResult({ behaviour: "v1", meta: { engine: "x" } })).toBe(false);
|
|
1167
|
+
});
|
|
1168
|
+
|
|
1169
|
+
test("both real arms are results", () => {
|
|
1170
|
+
expect(isBehaviourResult(behaviourReport(req([]), STAMP, {}))).toBe(true);
|
|
1171
|
+
expect(isBehaviourResult(noBehaviourEngineRefusal("acme"))).toBe(true);
|
|
1172
|
+
});
|
|
1173
|
+
});
|
|
1174
|
+
|
|
1175
|
+
/* -------------------------------------------------------------------------- */
|
|
1176
|
+
/* The runtime witness cannot drift from the type */
|
|
1177
|
+
/* -------------------------------------------------------------------------- */
|
|
1178
|
+
|
|
1179
|
+
describe("the reason list is derived from the type, not written beside it (#2356)", () => {
|
|
1180
|
+
test("every reason the type admits has a runtime witness", () => {
|
|
1181
|
+
// The array is Object.keys of a Record keyed by the union, so this cannot
|
|
1182
|
+
// fail without the module failing to compile — which is the point. Adding a
|
|
1183
|
+
// reason to `UnobservedReason` upstream used to widen this type silently,
|
|
1184
|
+
// leaving the guard rejecting a value the type accepted.
|
|
1185
|
+
const witness: Record<BehaviourUnpredictedReason, true> = {
|
|
1186
|
+
"read-failed": true,
|
|
1187
|
+
"no-binding": true,
|
|
1188
|
+
"unsupported-kind": true,
|
|
1189
|
+
filtered: true,
|
|
1190
|
+
"no-engine": true,
|
|
1191
|
+
"engine-unreachable": true,
|
|
1192
|
+
"engine-out-of-credit": true,
|
|
1193
|
+
"engine-over-quota": true,
|
|
1194
|
+
"credential-in-request": true,
|
|
1195
|
+
};
|
|
1196
|
+
expect([...BEHAVIOUR_UNPREDICTED_REASONS].sort()).toEqual(Object.keys(witness).sort());
|
|
1197
|
+
for (const reason of Object.keys(witness)) {
|
|
1198
|
+
expect(isBehaviourUnpredictedReason(reason), `${reason} has no witness`).toBe(true);
|
|
1199
|
+
}
|
|
1200
|
+
});
|
|
1201
|
+
|
|
1202
|
+
test("the same construction guards the other two closed sets", () => {
|
|
1203
|
+
expect([...BEHAVIOUR_BASES].sort()).toEqual(["modeled", "validated"]);
|
|
1204
|
+
expect([...RESILIENCE_VERDICTS].sort()).toEqual(["degrades", "fails", "survives"]);
|
|
1205
|
+
expect([...EDGE_COVERAGE_VERDICTS].sort()).toEqual(["complete", "partial", "unknown"]);
|
|
1206
|
+
});
|
|
1207
|
+
});
|
|
1208
|
+
|
|
1209
|
+
/* -------------------------------------------------------------------------- */
|
|
1210
|
+
/* Edge coverage */
|
|
1211
|
+
/* -------------------------------------------------------------------------- */
|
|
1212
|
+
|
|
1213
|
+
describe("an empty edge list can say which kind of empty it is (#2360)", () => {
|
|
1214
|
+
test("coverage is required — `[]` alone is not a statement", () => {
|
|
1215
|
+
// @ts-expect-error `edgeCoverage` is required: `edges: []` cannot say
|
|
1216
|
+
// whether nothing references anything or nothing could be reconstructed.
|
|
1217
|
+
const unstated: PredictBehaviourOptions = { ...REQUEST, edgeCoverage: undefined };
|
|
1218
|
+
expect(unstated).toBeDefined();
|
|
1219
|
+
});
|
|
1220
|
+
|
|
1221
|
+
test("a partial rebuild carries what it knows it lost", () => {
|
|
1222
|
+
const partial: PredictBehaviourOptions = {
|
|
1223
|
+
...REQUEST,
|
|
1224
|
+
edges: [],
|
|
1225
|
+
edgeCoverage: {
|
|
1226
|
+
verdict: "partial",
|
|
1227
|
+
dangling: [{ from: "web", path: "vpcId", value: "vpc-0a1b2c3d" }],
|
|
1228
|
+
unresolvedKinds: ["AWS::SQS::Queue"],
|
|
1229
|
+
containmentEdges: [{ from: "web", to: "subnet-a", kind: "ref" }],
|
|
1230
|
+
},
|
|
1231
|
+
};
|
|
1232
|
+
expect(isEdgeCoverageVerdict(partial.edgeCoverage.verdict)).toBe(true);
|
|
1233
|
+
// `from` is the field a flattened string list drops, and it is the one that
|
|
1234
|
+
// says which entity's path leaves the estate.
|
|
1235
|
+
expect(partial.edgeCoverage.dangling?.[0].from).toBe("web");
|
|
1236
|
+
expect(partial.edgeCoverage.dangling?.[0].value).toBe("vpc-0a1b2c3d");
|
|
1237
|
+
// Containment is the one an engine needs for "one zone lost" and `edges`
|
|
1238
|
+
// will never carry, because chant draws it as a boundary rather than a line.
|
|
1239
|
+
expect(partial.edgeCoverage.containmentEdges).toHaveLength(1);
|
|
1240
|
+
});
|
|
1241
|
+
|
|
1242
|
+
test("the coverage reaches the reader, not just the engine", () => {
|
|
1243
|
+
// Held only on the request it never reached a consumer, so a report's
|
|
1244
|
+
// "survives one zone lost" could have been computed over a complete graph
|
|
1245
|
+
// or over one whose builder had no idea what it had missed — a faked number
|
|
1246
|
+
// the shape rendered invisible, which is the failure the refusal arm exists
|
|
1247
|
+
// to prevent, reappearing one level down.
|
|
1248
|
+
const report = behaviourReport(
|
|
1249
|
+
{ ...req(["web"]), edgeCoverage: { verdict: "unknown" } },
|
|
1250
|
+
STAMP,
|
|
1251
|
+
{ web: GOOD },
|
|
1252
|
+
);
|
|
1253
|
+
expect(report.meta.edgeCoverage.verdict).toBe("unknown");
|
|
1254
|
+
});
|
|
1255
|
+
|
|
1256
|
+
test("a `partial` that names no gap is refused", () => {
|
|
1257
|
+
// `partial` means "some references are missing"; naming none of them is
|
|
1258
|
+
// `unknown` wearing a more confident word.
|
|
1259
|
+
expect(() => validateEdgeCoverage({ verdict: "partial" })).toThrow(/names nothing missing/);
|
|
1260
|
+
expect(() =>
|
|
1261
|
+
behaviourReport({ ...req(["web"]), edgeCoverage: { verdict: "partial" } }, STAMP, {
|
|
1262
|
+
web: GOOD,
|
|
1263
|
+
}),
|
|
1264
|
+
).toThrow(/names nothing missing/);
|
|
1265
|
+
|
|
1266
|
+
expect(() =>
|
|
1267
|
+
validateEdgeCoverage({ verdict: "partial", unresolvedKinds: ["AWS::SQS::Queue"] }),
|
|
1268
|
+
).not.toThrow();
|
|
1269
|
+
expect(() =>
|
|
1270
|
+
validateEdgeCoverage({
|
|
1271
|
+
verdict: "partial",
|
|
1272
|
+
dangling: [{ from: "web", path: "vpcId", value: "vpc-1" }],
|
|
1273
|
+
}),
|
|
1274
|
+
).not.toThrow();
|
|
1275
|
+
});
|
|
1276
|
+
|
|
1277
|
+
test("`complete` with a dangling reference is legal, and must stay so", () => {
|
|
1278
|
+
// A dangling ref points outside the named set by definition — a
|
|
1279
|
+
// cross-account VPC, another team's resource — so a builder can have found
|
|
1280
|
+
// every edge among the entities it was asked about and still have
|
|
1281
|
+
// references leaving the estate. Nobody should "fix" this.
|
|
1282
|
+
expect(() =>
|
|
1283
|
+
validateEdgeCoverage({
|
|
1284
|
+
verdict: "complete",
|
|
1285
|
+
dangling: [{ from: "web", path: "vpcId", value: "vpc-elsewhere" }],
|
|
1286
|
+
}),
|
|
1287
|
+
).not.toThrow();
|
|
1288
|
+
});
|
|
1289
|
+
|
|
1290
|
+
test("a bogus verdict is refused", () => {
|
|
1291
|
+
// @ts-expect-error "probably" is not one of the three.
|
|
1292
|
+
expect(() => validateEdgeCoverage({ verdict: "probably" })).toThrow(/is not complete/);
|
|
1293
|
+
});
|
|
1294
|
+
|
|
1295
|
+
test("`unknown` is available for a builder that cannot say", () => {
|
|
1296
|
+
const opaque: PredictBehaviourOptions = {
|
|
1297
|
+
...REQUEST,
|
|
1298
|
+
edges: [],
|
|
1299
|
+
edgeCoverage: { verdict: "unknown" },
|
|
1300
|
+
};
|
|
1301
|
+
expect(opaque.edgeCoverage.verdict).toBe("unknown");
|
|
1302
|
+
});
|
|
1303
|
+
});
|
|
1304
|
+
|
|
1305
|
+
/* -------------------------------------------------------------------------- */
|
|
1306
|
+
/* Nothing in a refusal publishes a credential */
|
|
1307
|
+
/* -------------------------------------------------------------------------- */
|
|
1308
|
+
|
|
1309
|
+
describe("a refusal never prints the engine's credentials (#2358 posts these publicly)", () => {
|
|
1310
|
+
const env = {
|
|
1311
|
+
CHANT_BEHAVIOUR_ENGINE: "https://svc:s3cr3t@engine.internal/predict?key=abc123&sig=zzz",
|
|
1312
|
+
};
|
|
1313
|
+
|
|
1314
|
+
test("userinfo and the query string are gone from the reason and the remedy", () => {
|
|
1315
|
+
const endpoint = behaviourEngineFrom("acme", env)!;
|
|
1316
|
+
const refusal = unreachableBehaviourEngineRefusal("acme", endpoint, "ECONNREFUSED").refusal;
|
|
1317
|
+
for (const text of [refusal.reason, refusal.remedy]) {
|
|
1318
|
+
expect(text, "the password leaked").not.toContain("s3cr3t");
|
|
1319
|
+
expect(text, "the userinfo leaked").not.toContain("svc:");
|
|
1320
|
+
expect(text, "a query token leaked").not.toContain("abc123");
|
|
1321
|
+
expect(text, "a signature leaked").not.toContain("zzz");
|
|
1322
|
+
}
|
|
1323
|
+
// The host survives, because a refusal naming no address cannot be acted on.
|
|
1324
|
+
expect(refusal.reason).toContain("engine.internal");
|
|
1325
|
+
});
|
|
1326
|
+
|
|
1327
|
+
test("the credit and quota refusals redact the same way", () => {
|
|
1328
|
+
const endpoint = behaviourEngineFrom("acme", env)!;
|
|
1329
|
+
for (const refusal of [
|
|
1330
|
+
outOfCreditBehaviourEngineRefusal("acme", endpoint, "balance 0.00").refusal,
|
|
1331
|
+
overQuotaBehaviourEngineRefusal("acme", endpoint, "5000/5000").refusal,
|
|
1332
|
+
]) {
|
|
1333
|
+
expect(refusal.reason).not.toContain("s3cr3t");
|
|
1334
|
+
expect(refusal.reason).not.toContain("abc123");
|
|
1335
|
+
expect(refusal.remedy).not.toContain("s3cr3t");
|
|
1336
|
+
}
|
|
1337
|
+
});
|
|
1338
|
+
|
|
1339
|
+
test("engine-supplied detail is bounded and scrubbed", () => {
|
|
1340
|
+
const endpoint = behaviourEngineFrom("acme", env)!;
|
|
1341
|
+
const chatty =
|
|
1342
|
+
"quota exceeded; see https://billing.acme.example/accounts/9?token=glpat-abcdef123456 " +
|
|
1343
|
+
"for details ".repeat(60);
|
|
1344
|
+
const reason = overQuotaBehaviourEngineRefusal("acme", endpoint, chatty).refusal.reason;
|
|
1345
|
+
expect(reason).not.toContain("glpat-abcdef123456");
|
|
1346
|
+
expect(reason).not.toContain("/accounts/9");
|
|
1347
|
+
expect(reason.length).toBeLessThan(chatty.length);
|
|
1348
|
+
});
|
|
1349
|
+
|
|
1350
|
+
test("an OAuth token in the fragment is gone", () => {
|
|
1351
|
+
// The implicit flow puts the access token after the `#`, where it never
|
|
1352
|
+
// reaches a server and where the URL parse was not looking.
|
|
1353
|
+
const redacted = redactEngineAddress(
|
|
1354
|
+
"https://engine.internal/predict#access_token=Zm9vYmFyYmF6cXV4&state=1",
|
|
1355
|
+
);
|
|
1356
|
+
expect(redacted).not.toContain("Zm9vYmFyYmF6cXV4");
|
|
1357
|
+
expect(redacted).not.toContain("access_token");
|
|
1358
|
+
expect(redacted).toContain("engine.internal");
|
|
1359
|
+
});
|
|
1360
|
+
|
|
1361
|
+
test("the query marker lands beside the fragment marker, not inside it", () => {
|
|
1362
|
+
const redacted = redactEngineAddress("https://engine.internal/p?key=abc#access_token=xyz");
|
|
1363
|
+
expect(redacted).not.toContain("abc");
|
|
1364
|
+
expect(redacted).not.toContain("xyz");
|
|
1365
|
+
// Wire order: query then fragment. Appending `?[redacted]` to a string that
|
|
1366
|
+
// still carried a fragment used to bury it inside the fragment.
|
|
1367
|
+
expect(redacted.indexOf("?")).toBeLessThan(redacted.indexOf("#"));
|
|
1368
|
+
});
|
|
1369
|
+
|
|
1370
|
+
test("an inline flag value on a PATH command is blanked", () => {
|
|
1371
|
+
// An engine may be a command rather than a URL, and a command carries its
|
|
1372
|
+
// credential as an argument — which the URL parse cannot see and which
|
|
1373
|
+
// `redactCredentialMaterial` reads as neither an env value nor a known
|
|
1374
|
+
// token prefix.
|
|
1375
|
+
for (const address of [
|
|
1376
|
+
"engine-cmd --token=hunter2plain --at 100rps",
|
|
1377
|
+
"engine-cmd --api-key hunter2plain",
|
|
1378
|
+
"engine-cmd -p hunter2plain",
|
|
1379
|
+
"engine-cmd --client-secret=hunter2plain",
|
|
1380
|
+
]) {
|
|
1381
|
+
expect(redactEngineAddress(address), address).not.toContain("hunter2plain");
|
|
1382
|
+
expect(redactEngineAddress(address)).toContain("engine-cmd");
|
|
1383
|
+
}
|
|
1384
|
+
});
|
|
1385
|
+
|
|
1386
|
+
test("a path-segment token survives, and the doc says so", () => {
|
|
1387
|
+
// Documented rather than fixed: nothing here can tell a token in a path
|
|
1388
|
+
// segment from a resource id, and an earlier version of the doc claimed
|
|
1389
|
+
// pass 3 caught it. This test pins the honest behaviour so the claim cannot
|
|
1390
|
+
// quietly come back.
|
|
1391
|
+
const opaque = "https://engine.internal/predict/Zm9vYmFyYmF6cXV4/go";
|
|
1392
|
+
expect(redactEngineAddress(opaque)).toContain("Zm9vYmFyYmF6cXV4");
|
|
1393
|
+
});
|
|
1394
|
+
|
|
1395
|
+
test("a socket path or a bare command is left readable", () => {
|
|
1396
|
+
const local = behaviourEngineFrom("acme", { CHANT_BEHAVIOUR_ENGINE: "/var/run/acme.sock" })!;
|
|
1397
|
+
expect(unreachableBehaviourEngineRefusal("acme", local, "no such file").refusal.reason).toContain(
|
|
1398
|
+
"/var/run/acme.sock",
|
|
1399
|
+
);
|
|
1400
|
+
expect(redactEngineAddress("acme-sim")).toBe("acme-sim");
|
|
1401
|
+
});
|
|
1402
|
+
});
|
|
1403
|
+
|
|
1404
|
+
describe("the engine variable chain (#2356)", () => {
|
|
1405
|
+
test("resolves most specific first, and names which variable won", () => {
|
|
1406
|
+
const env = {
|
|
1407
|
+
CHANT_BEHAVIOUR_ENGINE_ACME: "fixture://scoped",
|
|
1408
|
+
CHANT_BEHAVIOUR_ENGINE: "fixture://chant-wide",
|
|
1409
|
+
BEHAVIOUR_ENGINE: "fixture://bare",
|
|
1410
|
+
};
|
|
1411
|
+
expect(behaviourEngineFrom("acme", env)).toEqual({
|
|
1412
|
+
value: "fixture://scoped",
|
|
1413
|
+
source: "CHANT_BEHAVIOUR_ENGINE_ACME",
|
|
1414
|
+
});
|
|
1415
|
+
expect(behaviourEngineFrom("acme", { ...env, CHANT_BEHAVIOUR_ENGINE_ACME: "" })).toEqual({
|
|
1416
|
+
value: "fixture://chant-wide",
|
|
1417
|
+
source: "CHANT_BEHAVIOUR_ENGINE",
|
|
1418
|
+
});
|
|
1419
|
+
expect(behaviourEngineFrom("acme", { BEHAVIOUR_ENGINE: " fixture://bare " })).toEqual({
|
|
1420
|
+
value: "fixture://bare",
|
|
1421
|
+
source: "BEHAVIOUR_ENGINE",
|
|
1422
|
+
});
|
|
1423
|
+
expect(behaviourEngineFrom("acme", {})).toBeUndefined();
|
|
1424
|
+
});
|
|
1425
|
+
|
|
1426
|
+
test("the scoped variable is the lexicon name, upper-cased and underscored", () => {
|
|
1427
|
+
expect(behaviourEngineVariables("gcp-config-connector")[0]).toBe(
|
|
1428
|
+
"CHANT_BEHAVIOUR_ENGINE_GCP_CONFIG_CONNECTOR",
|
|
1429
|
+
);
|
|
1430
|
+
});
|
|
1431
|
+
|
|
1432
|
+
test("the refusal message quotes the chain it just walked", () => {
|
|
1433
|
+
const message = noBehaviourEngineMessage("acme");
|
|
1434
|
+
for (const variable of behaviourEngineVariables("acme")) {
|
|
1435
|
+
expect(message, `the message does not name ${variable}`).toContain(variable);
|
|
1436
|
+
}
|
|
1437
|
+
});
|
|
1438
|
+
});
|
|
1439
|
+
|
|
1440
|
+
/* -------------------------------------------------------------------------- */
|
|
1441
|
+
/* Rule 1: a prediction is never presented as a bill */
|
|
1442
|
+
/* -------------------------------------------------------------------------- */
|
|
1443
|
+
|
|
1444
|
+
describe("rule 1 — a prediction cannot be shaped like a bill (#2356)", () => {
|
|
1445
|
+
test("every cost carries the per-hour discriminant", () => {
|
|
1446
|
+
expect(predictedRate(0.0416, "USD").rate).toBe("per-hour");
|
|
1447
|
+
});
|
|
1448
|
+
|
|
1449
|
+
test("a billing record cannot be assigned where a predicted rate is wanted", () => {
|
|
1450
|
+
interface LineItem {
|
|
1451
|
+
amount: number;
|
|
1452
|
+
currency: string;
|
|
1453
|
+
periodStart: string;
|
|
1454
|
+
periodEnd: string;
|
|
1455
|
+
invoiceId: string;
|
|
1456
|
+
}
|
|
1457
|
+
const charged: LineItem = {
|
|
1458
|
+
amount: 30.37,
|
|
1459
|
+
currency: "USD",
|
|
1460
|
+
periodStart: "2026-08-01",
|
|
1461
|
+
periodEnd: "2026-08-31",
|
|
1462
|
+
invoiceId: "INV-4471",
|
|
1463
|
+
};
|
|
1464
|
+
// @ts-expect-error a charge for an elapsed period is not a rate for an
|
|
1465
|
+
// imagined hour: `rate: "per-hour"` is missing and `amount` is not `perHour`.
|
|
1466
|
+
const asPrediction: PredictedRate = charged;
|
|
1467
|
+
expect(asPrediction).toBeDefined();
|
|
1468
|
+
});
|
|
1469
|
+
|
|
1470
|
+
test("a figure cannot be built without the traffic level it answers", () => {
|
|
1471
|
+
// @ts-expect-error `at` is required — a number with no stated question is
|
|
1472
|
+
// the number most likely to be quoted as money owed.
|
|
1473
|
+
const noLevel: PredictedBehaviour = {
|
|
1474
|
+
cost: predictedRate(0.0416, "USD"),
|
|
1475
|
+
headroom: { cpu: 0.62 },
|
|
1476
|
+
errorRate: 0.001,
|
|
1477
|
+
resilience: { failure: "one zone lost", verdict: "survives" },
|
|
1478
|
+
provenance: { engine: "acme-sim", version: "1.4.2", tolerance: "±15%", basis: "modeled" },
|
|
1479
|
+
};
|
|
1480
|
+
expect(noLevel).toBeDefined();
|
|
1481
|
+
});
|
|
1482
|
+
|
|
1483
|
+
test("a report has nowhere to put a billing period or an account", () => {
|
|
1484
|
+
const asStatement = behaviourReport(
|
|
1485
|
+
req([]),
|
|
1486
|
+
{
|
|
1487
|
+
engine: "acme-sim",
|
|
1488
|
+
version: "1.4.2",
|
|
1489
|
+
// @ts-expect-error the envelope models a run, not a statement: there is
|
|
1490
|
+
// no account to bill and no period that elapsed.
|
|
1491
|
+
accountId: "123456789012",
|
|
1492
|
+
periodStart: "2026-08-01",
|
|
1493
|
+
},
|
|
1494
|
+
{},
|
|
1495
|
+
);
|
|
1496
|
+
expect(asStatement).toBeDefined();
|
|
1497
|
+
});
|
|
1498
|
+
});
|
|
1499
|
+
|
|
1500
|
+
/* -------------------------------------------------------------------------- */
|
|
1501
|
+
/* Rule 2: a missing engine is a named refusal, never a zeroed result */
|
|
1502
|
+
/* -------------------------------------------------------------------------- */
|
|
1503
|
+
|
|
1504
|
+
describe("rule 2 — the refusal arm has no figures on it (#2356)", () => {
|
|
1505
|
+
test("a refusal report cannot carry entities", () => {
|
|
1506
|
+
const refusal = noBehaviourEngineRefusal("acme");
|
|
1507
|
+
// @ts-expect-error `BehaviourRefusalReport` has no `entities` member, so
|
|
1508
|
+
// "zero everything and carry on" is not a shape this contract can express.
|
|
1509
|
+
refusal.entities = {};
|
|
1510
|
+
expect(refusal.refusal.cause).toBe("no-engine");
|
|
1511
|
+
});
|
|
1512
|
+
|
|
1513
|
+
test("`no-engine` and `engine-unreachable` are distinguishable verdicts", () => {
|
|
1514
|
+
expect(isBehaviourUnpredictedReason("no-engine")).toBe(true);
|
|
1515
|
+
expect(isBehaviourUnpredictedReason("engine-unreachable")).toBe(true);
|
|
1516
|
+
expect(noBehaviourEngineRefusal("acme").refusal.cause).not.toBe(
|
|
1517
|
+
unreachableBehaviourEngineRefusal("acme", { value: "x", source: "BEHAVIOUR_ENGINE" }, "down")
|
|
1518
|
+
.refusal.cause,
|
|
1519
|
+
);
|
|
1520
|
+
});
|
|
1521
|
+
});
|
|
1522
|
+
|
|
1523
|
+
/* -------------------------------------------------------------------------- */
|
|
1524
|
+
/* Rule 3: the engine never sees a credential */
|
|
1525
|
+
/* -------------------------------------------------------------------------- */
|
|
1526
|
+
|
|
1527
|
+
describe("rule 3 — no credential can reach the engine (#2356)", () => {
|
|
1528
|
+
test("a credential field is a compile error, by name", () => {
|
|
1529
|
+
// @ts-expect-error `token` is declared `?: never`.
|
|
1530
|
+
const withToken: PredictBehaviourOptions = { ...REQUEST, token: "glpat-xxxx" };
|
|
1531
|
+
// @ts-expect-error `credentials` is declared `?: never`.
|
|
1532
|
+
const withCredentials: PredictBehaviourOptions = { ...REQUEST, credentials: { key: "AKIA" } };
|
|
1533
|
+
// @ts-expect-error `apiKey` is declared `?: never`.
|
|
1534
|
+
const withApiKey: PredictBehaviourOptions = { ...REQUEST, apiKey: "sk-live-1" };
|
|
1535
|
+
// @ts-expect-error `sessionToken` is declared `?: never`.
|
|
1536
|
+
const withSession: PredictBehaviourOptions = { ...REQUEST, sessionToken: "FwoGZ" };
|
|
1537
|
+
expect([withToken, withCredentials, withApiKey, withSession]).toHaveLength(4);
|
|
1538
|
+
});
|
|
1539
|
+
|
|
1540
|
+
test("a name nobody thought of is rejected too, by the excess-property check", () => {
|
|
1541
|
+
// @ts-expect-error the options are sealed: an unlisted key is a compile
|
|
1542
|
+
// error whether or not it is on the credential list.
|
|
1543
|
+
const smuggled: PredictBehaviourOptions = { ...REQUEST, acmeSimSigningSecret: "hunter2" };
|
|
1544
|
+
expect(smuggled).toBeDefined();
|
|
1545
|
+
});
|
|
1546
|
+
|
|
1547
|
+
test("the fixture lexicon screens with the entry point, not the value-only half", async () => {
|
|
1548
|
+
// F1: `assertNoCredentialInOptions` applies one rule of three. A lexicon
|
|
1549
|
+
// calling it — as this fixture did, and as the docs page told #2357 to —
|
|
1550
|
+
// sends the request anyway on a key-name hit and on a walk-depth hit.
|
|
1551
|
+
const leaky = new Map(DECLARED);
|
|
1552
|
+
leaky.set("db", {
|
|
1553
|
+
entityType: "AWS::RDS::DBInstance",
|
|
1554
|
+
props: { awsSecretAccessKey: "wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY" },
|
|
1555
|
+
});
|
|
1556
|
+
const onName = await acmeSim(REACHABLE)!({ ...REQUEST, entities: leaky });
|
|
1557
|
+
expect(isBehaviourRefusalReport(onName), "a key-name hit reached the engine").toBe(true);
|
|
1558
|
+
if (isBehaviourRefusalReport(onName)) {
|
|
1559
|
+
expect(onName.refusal.cause).toBe("credential-in-request");
|
|
1560
|
+
expect(onName.refusal.reason).toContain("awsSecretAccessKey");
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1563
|
+
// And the depth case, which `assertNoCredentialInOptions` also drops.
|
|
1564
|
+
let deep: unknown = "leaf";
|
|
1565
|
+
for (let i = 0; i < 40; i++) deep = { nested: deep };
|
|
1566
|
+
const nested = new Map(DECLARED);
|
|
1567
|
+
nested.set("web", { entityType: "AWS::EC2::Instance", props: { deep } });
|
|
1568
|
+
const onDepth = await acmeSim(REACHABLE)!({ ...REQUEST, entities: nested });
|
|
1569
|
+
expect(isBehaviourRefusalReport(onDepth), "an unread subtree reached the engine").toBe(true);
|
|
1570
|
+
|
|
1571
|
+
// A live token still throws, because that arm is a stop rather than a
|
|
1572
|
+
// degradation — the split the two functions exist to keep.
|
|
1573
|
+
const token = new Map(DECLARED);
|
|
1574
|
+
token.set("web", {
|
|
1575
|
+
entityType: "AWS::EC2::Instance",
|
|
1576
|
+
props: { note: "ghp_abcdefghijklmnop" },
|
|
1577
|
+
});
|
|
1578
|
+
await expect(acmeSim(REACHABLE)!({ ...REQUEST, entities: token })).rejects.toThrow(
|
|
1579
|
+
/a GitHub token/,
|
|
1580
|
+
);
|
|
1581
|
+
});
|
|
1582
|
+
|
|
1583
|
+
test("the depth budget leaves room for a real property tree", () => {
|
|
1584
|
+
// `options → entities → .get(name) → props` spends three levels before a
|
|
1585
|
+
// declared property is reached, and a k8s workload tree
|
|
1586
|
+
// (`spec.template.spec.containers[].env[].valueFrom.secretKeyRef.key`) runs
|
|
1587
|
+
// to a dozen on its own. Exceeding the budget now refuses, so a budget set
|
|
1588
|
+
// too low refuses real estates rather than protecting them.
|
|
1589
|
+
let tree: unknown = { key: "config-value" };
|
|
1590
|
+
for (const level of ["secretKeyRef", "valueFrom", "env", "containers", "spec", "template", "spec"]) {
|
|
1591
|
+
tree = { [level]: [tree] };
|
|
1592
|
+
}
|
|
1593
|
+
const k8s = new Map(DECLARED);
|
|
1594
|
+
k8s.set("web", { entityType: "AWS::EC2::Instance", props: tree as Record<string, unknown> });
|
|
1595
|
+
expect(findCredentialsInOptions({ ...REQUEST, entities: k8s })).toEqual([]);
|
|
1596
|
+
});
|
|
1597
|
+
|
|
1598
|
+
test("a suspicious field name refuses, and does NOT throw", () => {
|
|
1599
|
+
// The blast radius matters. Throwing is the whole-lexicon failure per
|
|
1600
|
+
// lexicon.ts, and the name arm is a heuristic: one `tags: { author: … }`
|
|
1601
|
+
// anywhere in an estate used to kill the entire overlay with a stack trace,
|
|
1602
|
+
// which is precisely the failure constraint 2 was written against.
|
|
1603
|
+
for (const key of ["token", "credentials", "apiKey", "privateKey", "authorization"]) {
|
|
1604
|
+
const options = { ...REQUEST, [key]: "a-value-long-enough-to-suspect" };
|
|
1605
|
+
expect(() => assertNoCredentialInOptions(options), `${key} threw`).not.toThrow();
|
|
1606
|
+
const refusal = screenBehaviourRequest("acme", options);
|
|
1607
|
+
expect(refusal, `${key} was not caught`).toBeDefined();
|
|
1608
|
+
expect(refusal!.refusal.cause).toBe("credential-in-request");
|
|
1609
|
+
expect(refusal!.refusal.reason).toContain(`("${key}")`);
|
|
1610
|
+
}
|
|
1611
|
+
expect(screenBehaviourRequest("acme", REQUEST)).toBeUndefined();
|
|
1612
|
+
});
|
|
1613
|
+
|
|
1614
|
+
test("a key nobody listed is caught too — casing, separators and all", () => {
|
|
1615
|
+
// The reproduction that broke the old 14-name list. Every one of these
|
|
1616
|
+
// compiles clean through a widened variable.
|
|
1617
|
+
const smuggled: Record<string, string> = {
|
|
1618
|
+
xApiKey: "sk-live-DEADBEEF",
|
|
1619
|
+
Authorization: "Bearer abcdefghijklmnop",
|
|
1620
|
+
pat: "glpat-zzzzzzzz",
|
|
1621
|
+
"x-api-key": "whatever-value-here",
|
|
1622
|
+
clientSecret: "s3cr3t-value-here",
|
|
1623
|
+
refreshToken: "r3fr3sh-value-here",
|
|
1624
|
+
awsSecretAccessKey: "wJalrXUtnFEMIK7MDENG",
|
|
1625
|
+
cookie: "session=abcdefgh",
|
|
1626
|
+
};
|
|
1627
|
+
for (const [key, value] of Object.entries(smuggled)) {
|
|
1628
|
+
const found = findCredentialsInOptions({ ...REQUEST, [key]: value });
|
|
1629
|
+
expect(found.length, `${key} was not caught`).toBeGreaterThan(0);
|
|
1630
|
+
}
|
|
1631
|
+
});
|
|
1632
|
+
|
|
1633
|
+
test("a credential in `props` is caught — the channel no type can see", () => {
|
|
1634
|
+
// Props come straight out of the build. A lexicon surfacing a connection
|
|
1635
|
+
// string puts one here without deciding to, which is the realistic leak.
|
|
1636
|
+
const leaky = new Map(DECLARED);
|
|
1637
|
+
leaky.set("db", {
|
|
1638
|
+
entityType: "AWS::RDS::DBInstance",
|
|
1639
|
+
props: { awsSecretAccessKey: "wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY" },
|
|
1640
|
+
});
|
|
1641
|
+
const named = screenBehaviourRequest("acme", { ...REQUEST, entities: leaky });
|
|
1642
|
+
expect(named?.refusal.reason).toMatch(/credential-shaped field name \("awsSecretAccessKey"\)/);
|
|
1643
|
+
|
|
1644
|
+
// A password in a URL is the value arm, so it throws rather than refusing.
|
|
1645
|
+
const urlCreds = new Map(DECLARED);
|
|
1646
|
+
urlCreds.set("db", {
|
|
1647
|
+
entityType: "AWS::RDS::DBInstance",
|
|
1648
|
+
props: { connection: "postgres://app:hunter2@db.internal:5432/prod" },
|
|
1649
|
+
});
|
|
1650
|
+
expect(() => assertNoCredentialInOptions({ ...REQUEST, entities: urlCreds })).toThrow(
|
|
1651
|
+
/a password in a URL's userinfo/,
|
|
1652
|
+
);
|
|
1653
|
+
});
|
|
1654
|
+
|
|
1655
|
+
test("a scheme-less DSN is caught — `new URL()` will not parse one", () => {
|
|
1656
|
+
const dsn = new Map(DECLARED);
|
|
1657
|
+
dsn.set("db", {
|
|
1658
|
+
entityType: "AWS::RDS::DBInstance",
|
|
1659
|
+
props: { dsn: "app:hunter2@db.internal:5432/prod" },
|
|
1660
|
+
});
|
|
1661
|
+
expect(() => assertNoCredentialInOptions({ ...REQUEST, entities: dsn })).toThrow(
|
|
1662
|
+
/a password in a URL's userinfo/,
|
|
1663
|
+
);
|
|
1664
|
+
});
|
|
1665
|
+
|
|
1666
|
+
test("a Map's keys are read, not just its values", () => {
|
|
1667
|
+
// An env block is a Map as often as an object, and walking only values
|
|
1668
|
+
// missed `DB_PASSWORD` entirely.
|
|
1669
|
+
const env = new Map(DECLARED);
|
|
1670
|
+
env.set("web", {
|
|
1671
|
+
entityType: "AWS::EC2::Instance",
|
|
1672
|
+
props: { env: new Map([["DB_PASSWORD", "hunter2plain"]]) },
|
|
1673
|
+
});
|
|
1674
|
+
const refusal = screenBehaviourRequest("acme", { ...REQUEST, entities: env });
|
|
1675
|
+
expect(refusal?.refusal.reason).toMatch(/DB_PASSWORD/);
|
|
1676
|
+
});
|
|
1677
|
+
|
|
1678
|
+
test("a Set's contents are read — `Object.entries` on a Set is empty", () => {
|
|
1679
|
+
const set = new Map(DECLARED);
|
|
1680
|
+
set.set("web", {
|
|
1681
|
+
entityType: "AWS::EC2::Instance",
|
|
1682
|
+
props: { allowed: new Set(["glpat-abcdef123456"]) },
|
|
1683
|
+
});
|
|
1684
|
+
expect(() => assertNoCredentialInOptions({ ...REQUEST, entities: set })).toThrow(
|
|
1685
|
+
/a GitLab personal access token/,
|
|
1686
|
+
);
|
|
1687
|
+
});
|
|
1688
|
+
|
|
1689
|
+
test("a structure deeper than the walk reads is reported, not passed in silence", () => {
|
|
1690
|
+
let deep: unknown = "leaf";
|
|
1691
|
+
for (let i = 0; i < 40; i++) deep = { nested: deep };
|
|
1692
|
+
const nested = new Map(DECLARED);
|
|
1693
|
+
nested.set("web", { entityType: "AWS::EC2::Instance", props: { deep } });
|
|
1694
|
+
const found = findCredentialsInOptions({ ...REQUEST, entities: nested });
|
|
1695
|
+
expect(found.some((f) => f.rule === "walk-depth")).toBe(true);
|
|
1696
|
+
// And it refuses rather than proceeding as though the request were checked.
|
|
1697
|
+
expect(screenBehaviourRequest("acme", { ...REQUEST, entities: nested })).toBeDefined();
|
|
1698
|
+
});
|
|
1699
|
+
|
|
1700
|
+
test("a credential on an edge field is caught", () => {
|
|
1701
|
+
expect(() =>
|
|
1702
|
+
assertNoCredentialInOptions({
|
|
1703
|
+
...REQUEST,
|
|
1704
|
+
edges: [{ from: "web", to: "db", kind: "ref", viaAttr: "token=glpat-abcdef123456" }],
|
|
1705
|
+
}),
|
|
1706
|
+
).toThrow(/a GitLab personal access token/);
|
|
1707
|
+
});
|
|
1708
|
+
|
|
1709
|
+
test("a credential-shaped value is caught wherever it sits, whatever the key", () => {
|
|
1710
|
+
const shapes: Record<string, string> = {
|
|
1711
|
+
note: "ghp_abcdefghijklmnop",
|
|
1712
|
+
label: "AKIAIOSFODNN7EXAMPLE",
|
|
1713
|
+
hint: "xoxb-1234-5678-abcdefg",
|
|
1714
|
+
blob: "-----BEGIN RSA PRIVATE KEY-----\nMIIE\n-----END RSA PRIVATE KEY-----",
|
|
1715
|
+
jwt: "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.abc",
|
|
1716
|
+
};
|
|
1717
|
+
for (const [key, value] of Object.entries(shapes)) {
|
|
1718
|
+
const props = new Map(DECLARED);
|
|
1719
|
+
props.set("web", { entityType: "AWS::EC2::Instance", props: { [key]: value } });
|
|
1720
|
+
expect(
|
|
1721
|
+
() => assertNoCredentialInOptions({ ...REQUEST, entities: props }),
|
|
1722
|
+
`${key} was not caught`,
|
|
1723
|
+
).toThrow(/behaviour engine is never handed a credential/);
|
|
1724
|
+
}
|
|
1725
|
+
});
|
|
1726
|
+
|
|
1727
|
+
test("the guard refuses none of this repo's own generated property keys", () => {
|
|
1728
|
+
// THE regression guard for the name arm, and the reason it can be trusted
|
|
1729
|
+
// enough to keep. An earlier version matched `SECRET|TOKEN|PASSWORD|AUTH`
|
|
1730
|
+
// as a bare substring of the key and refused 406 aws keys, 173 azure, 89
|
|
1731
|
+
// gcp and 7 k8s — `imagePullSecrets`, `secretName`, `secretKeyRef`,
|
|
1732
|
+
// `ClientToken`, `CertificateAuthorityArn`, `authorizedNetworks`,
|
|
1733
|
+
// `passwordPolicy`, `authMode`, `oauthScopes` — every one of them ordinary
|
|
1734
|
+
// in real infrastructure, and every one of them fatal, because the guard
|
|
1735
|
+
// threw. #2357, #2359 and #2360 all build requests from these props.
|
|
1736
|
+
//
|
|
1737
|
+
// Read from the generated `.d.ts` rather than the registry JSON: the JSON
|
|
1738
|
+
// yields 81 names for aws and would prove nothing.
|
|
1739
|
+
const counts: Record<string, number> = {};
|
|
1740
|
+
for (const lexicon of ["aws", "azure", "gcp", "k8s"]) {
|
|
1741
|
+
const dts = join(__dirname, "../../../lexicons", lexicon, "src/generated/index.d.ts");
|
|
1742
|
+
const keys = new Set<string>();
|
|
1743
|
+
for (const m of readFileSync(dts, "utf8").matchAll(
|
|
1744
|
+
/^\s+(?:readonly\s+)?([A-Za-z_]\w*)\??\s*:/gm,
|
|
1745
|
+
)) {
|
|
1746
|
+
keys.add(m[1]);
|
|
1747
|
+
}
|
|
1748
|
+
counts[lexicon] = keys.size;
|
|
1749
|
+
|
|
1750
|
+
const props: Record<string, unknown> = {};
|
|
1751
|
+
for (const key of keys) props[key] = "ok";
|
|
1752
|
+
const entities = new Map(DECLARED);
|
|
1753
|
+
entities.set("web", { entityType: "AWS::EC2::Instance", props });
|
|
1754
|
+
|
|
1755
|
+
const refused = findCredentialsInOptions({ ...REQUEST, entities }).map((f) => f.path);
|
|
1756
|
+
expect(refused, `${lexicon} keys refused with a benign value`).toEqual([]);
|
|
1757
|
+
}
|
|
1758
|
+
|
|
1759
|
+
// A floor, not a pin: this catches a regeneration that SHRINKS the key set
|
|
1760
|
+
// (or an extractor that quietly stops matching), which is the way this test
|
|
1761
|
+
// goes vacuous. A regeneration that adds names is not caught here and does
|
|
1762
|
+
// not need to be — the zero-refusals assertion above already covers every
|
|
1763
|
+
// name present, however many there are.
|
|
1764
|
+
expect(counts.aws).toBeGreaterThan(13000);
|
|
1765
|
+
expect(counts.azure).toBeGreaterThan(4900);
|
|
1766
|
+
expect(counts.gcp).toBeGreaterThan(3300);
|
|
1767
|
+
expect(counts.k8s).toBeGreaterThan(290);
|
|
1768
|
+
});
|
|
1769
|
+
|
|
1770
|
+
test("the reference-by-name family is carved out, by suffix and by name", () => {
|
|
1771
|
+
// Named explicitly so the carve-out list cannot be trimmed without a red.
|
|
1772
|
+
// Every one holds a pointer, an id, a duration or a scope list, never a
|
|
1773
|
+
// secret — and every one is long enough to trip the length gate.
|
|
1774
|
+
const benign = "a-perfectly-ordinary-value";
|
|
1775
|
+
for (const key of [
|
|
1776
|
+
"imagePullSecrets",
|
|
1777
|
+
"secretName",
|
|
1778
|
+
"secretRef",
|
|
1779
|
+
"secretKeyRef",
|
|
1780
|
+
"ClientToken",
|
|
1781
|
+
"AdminPasswordSecretArn",
|
|
1782
|
+
"APIKeyId",
|
|
1783
|
+
"AccessTokenId",
|
|
1784
|
+
"AccessTokenValidity",
|
|
1785
|
+
"ActorTokenScopes",
|
|
1786
|
+
"CertificateAuthorityArn",
|
|
1787
|
+
"authorizedNetworks",
|
|
1788
|
+
"passwordPolicy",
|
|
1789
|
+
"authMode",
|
|
1790
|
+
"oauthScopes",
|
|
1791
|
+
"automountServiceAccountToken",
|
|
1792
|
+
]) {
|
|
1793
|
+
const entities = new Map(DECLARED);
|
|
1794
|
+
entities.set("web", { entityType: "AWS::EC2::Instance", props: { [key]: benign } });
|
|
1795
|
+
expect(
|
|
1796
|
+
findCredentialsInOptions({ ...REQUEST, entities }),
|
|
1797
|
+
`${key} was refused`,
|
|
1798
|
+
).toEqual([]);
|
|
1799
|
+
}
|
|
1800
|
+
});
|
|
1801
|
+
|
|
1802
|
+
test("a genuine credential field holding a reference is not refused; holding a token is", () => {
|
|
1803
|
+
// `AdminPassword` on AWS::DirectoryService::MicrosoftAD is a real
|
|
1804
|
+
// credential field. In chant source it holds a resolve-expression or a
|
|
1805
|
+
// `{ $ref }`, and refusing those would refuse the correct way to write it.
|
|
1806
|
+
const ref = new Map(DECLARED);
|
|
1807
|
+
ref.set("web", {
|
|
1808
|
+
entityType: "AWS::EC2::Instance",
|
|
1809
|
+
props: {
|
|
1810
|
+
AdminPassword: "{{resolve:secretsmanager:prod/ad:SecretString:password}}",
|
|
1811
|
+
AccountPassword: { $ref: "vault.adminPassword" },
|
|
1812
|
+
ADDomainJoinPassword: "${secrets.domainJoin}",
|
|
1813
|
+
},
|
|
1814
|
+
});
|
|
1815
|
+
expect(findCredentialsInOptions({ ...REQUEST, entities: ref })).toEqual([]);
|
|
1816
|
+
|
|
1817
|
+
// A live value in the same field is still caught, by the value arm.
|
|
1818
|
+
const live = new Map(DECLARED);
|
|
1819
|
+
live.set("web", {
|
|
1820
|
+
entityType: "AWS::EC2::Instance",
|
|
1821
|
+
props: { AdminPassword: "ghp_abcdefghijklmnop" },
|
|
1822
|
+
});
|
|
1823
|
+
expect(() => assertNoCredentialInOptions({ ...REQUEST, entities: live })).toThrow(
|
|
1824
|
+
/a GitHub token/,
|
|
1825
|
+
);
|
|
1826
|
+
});
|
|
1827
|
+
|
|
1828
|
+
test("a non-string never trips the name arm", () => {
|
|
1829
|
+
// `{ tokenCount: 4096 }`, `{ AccessControlAllowCredentials: true }`, and a
|
|
1830
|
+
// list of config objects whose parent key contains `auth`.
|
|
1831
|
+
const typed = new Map(DECLARED);
|
|
1832
|
+
typed.set("web", {
|
|
1833
|
+
entityType: "AWS::EC2::Instance",
|
|
1834
|
+
props: {
|
|
1835
|
+
tokenCount: 4096,
|
|
1836
|
+
AccessControlAllowCredentials: true,
|
|
1837
|
+
AdditionalAuthenticationProviders: [{ AuthenticationType: "AWS_IAM" }],
|
|
1838
|
+
tags: { author: "alex" },
|
|
1839
|
+
},
|
|
1840
|
+
});
|
|
1841
|
+
expect(findCredentialsInOptions({ ...REQUEST, entities: typed })).toEqual([]);
|
|
1842
|
+
});
|
|
1843
|
+
|
|
1844
|
+
test("ordinary build output is not refused — ids, ARNs and digests pass", () => {
|
|
1845
|
+
// The other half of the bargain. There is no entropy scoring on purpose:
|
|
1846
|
+
// this walks build output, and a heuristic refusing these would refuse real
|
|
1847
|
+
// projects rather than protect them.
|
|
1848
|
+
const ordinary = new Map(DECLARED);
|
|
1849
|
+
ordinary.set("web", {
|
|
1850
|
+
entityType: "AWS::EC2::Instance",
|
|
1851
|
+
props: {
|
|
1852
|
+
arn: "arn:aws:ec2:us-east-1:123456789012:instance/i-0abcd1234ef567890",
|
|
1853
|
+
digest: "sha256:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
|
|
1854
|
+
accountId: "123456789012",
|
|
1855
|
+
subnet: "subnet-0a1b2c3d4e5f6a7b8",
|
|
1856
|
+
url: "https://api.internal.example.com/v1/things?page=2",
|
|
1857
|
+
},
|
|
1858
|
+
});
|
|
1859
|
+
expect(() => assertNoCredentialInOptions({ ...REQUEST, entities: ordinary })).not.toThrow();
|
|
1860
|
+
});
|
|
1861
|
+
|
|
1862
|
+
test("the lexicon itself refuses a smuggled credential before it resolves an engine", async () => {
|
|
1863
|
+
await expect(
|
|
1864
|
+
// Only a double cast gets a credential this far, which is the type doing
|
|
1865
|
+
// its job; the runtime guard is what catches a caller with no types at all.
|
|
1866
|
+
acmeSim(REACHABLE)!({ ...REQUEST, token: "glpat-xxxx" } as unknown as PredictBehaviourOptions),
|
|
1867
|
+
).rejects.toThrow(/never handed a credential/);
|
|
1868
|
+
});
|
|
1869
|
+
});
|
|
1870
|
+
|
|
1871
|
+
/* -------------------------------------------------------------------------- */
|
|
1872
|
+
/* Rule 4: provenance on every number, with a closed basis */
|
|
1873
|
+
/* -------------------------------------------------------------------------- */
|
|
1874
|
+
|
|
1875
|
+
describe("rule 4 — provenance, and a basis from a closed set (#2356)", () => {
|
|
1876
|
+
test("a prediction cannot be built without provenance", () => {
|
|
1877
|
+
// @ts-expect-error `provenance` is required.
|
|
1878
|
+
const unattributed: PredictedBehaviour = {
|
|
1879
|
+
at: { traffic: "100 rps, p50" },
|
|
1880
|
+
cost: predictedRate(0.0416, "USD"),
|
|
1881
|
+
headroom: { cpu: 0.62 },
|
|
1882
|
+
errorRate: 0.001,
|
|
1883
|
+
resilience: { failure: "one zone lost", verdict: "survives" },
|
|
1884
|
+
};
|
|
1885
|
+
expect(unattributed).toBeDefined();
|
|
1886
|
+
});
|
|
1887
|
+
|
|
1888
|
+
test("provenance cannot be built without a basis", () => {
|
|
1889
|
+
// @ts-expect-error `basis` is required — "which engine" is not enough, the
|
|
1890
|
+
// reader has to know whether a bill was ever involved.
|
|
1891
|
+
const noBasis: PredictedBehaviour["provenance"] = {
|
|
1892
|
+
engine: "acme-sim",
|
|
1893
|
+
version: "1.4.2",
|
|
1894
|
+
tolerance: "±15%",
|
|
1895
|
+
};
|
|
1896
|
+
expect(noBasis).toBeDefined();
|
|
1897
|
+
});
|
|
1898
|
+
|
|
1899
|
+
test("the basis is a closed enum, not a free string", () => {
|
|
1900
|
+
const invented: PredictedBehaviour["provenance"] = {
|
|
1901
|
+
engine: "acme-sim",
|
|
1902
|
+
version: "1.4.2",
|
|
1903
|
+
tolerance: "±15%",
|
|
1904
|
+
// @ts-expect-error "estimated" is not one of the two. A third word would
|
|
1905
|
+
// let a figure dodge the question the enum exists to force.
|
|
1906
|
+
basis: "estimated",
|
|
1907
|
+
};
|
|
1908
|
+
expect(invented).toBeDefined();
|
|
1909
|
+
|
|
1910
|
+
expect(BEHAVIOUR_BASES).toEqual(["modeled", "validated"]);
|
|
1911
|
+
expect(isBehaviourBasis("modeled")).toBe(true);
|
|
1912
|
+
expect(isBehaviourBasis("validated")).toBe(true);
|
|
1913
|
+
expect(isBehaviourBasis("estimated")).toBe(false);
|
|
1914
|
+
expect(isBehaviourBasis("guessed")).toBe(false);
|
|
1915
|
+
});
|
|
1916
|
+
});
|
|
1917
|
+
|
|
1918
|
+
/* -------------------------------------------------------------------------- */
|
|
1919
|
+
/* The tri-state */
|
|
1920
|
+
/* -------------------------------------------------------------------------- */
|
|
1921
|
+
|
|
1922
|
+
describe("the behaviour tri-state (#2356)", () => {
|
|
1923
|
+
test("the reasons are the observation set minus no-credentials, plus the engine and request states", () => {
|
|
1924
|
+
expect([...BEHAVIOUR_UNPREDICTED_REASONS].sort()).toEqual([
|
|
1925
|
+
"credential-in-request",
|
|
1926
|
+
"engine-out-of-credit",
|
|
1927
|
+
"engine-over-quota",
|
|
1928
|
+
"engine-unreachable",
|
|
1929
|
+
"filtered",
|
|
1930
|
+
"no-binding",
|
|
1931
|
+
"no-engine",
|
|
1932
|
+
"read-failed",
|
|
1933
|
+
"unsupported-kind",
|
|
1934
|
+
]);
|
|
1935
|
+
});
|
|
1936
|
+
|
|
1937
|
+
test("each engine state is separately switchable", () => {
|
|
1938
|
+
for (const reason of [
|
|
1939
|
+
"no-engine",
|
|
1940
|
+
"engine-unreachable",
|
|
1941
|
+
"engine-out-of-credit",
|
|
1942
|
+
"engine-over-quota",
|
|
1943
|
+
]) {
|
|
1944
|
+
expect(isBehaviourUnpredictedReason(reason), `${reason} is not a reason`).toBe(true);
|
|
1945
|
+
}
|
|
1946
|
+
});
|
|
1947
|
+
|
|
1948
|
+
test("`no-credentials` is not a behaviour reason — the engine is never handed one", () => {
|
|
1949
|
+
expect(isBehaviourUnpredictedReason("no-credentials")).toBe(false);
|
|
1950
|
+
// @ts-expect-error and it is not assignable either, so a lexicon cannot
|
|
1951
|
+
// send an operator hunting for a variable this contract forbids.
|
|
1952
|
+
const borrowed: (typeof BEHAVIOUR_UNPREDICTED_REASONS)[number] = "no-credentials";
|
|
1953
|
+
expect(borrowed).toBe("no-credentials");
|
|
1954
|
+
});
|
|
1955
|
+
|
|
1956
|
+
test("the four shared verdicts keep the spelling `UnobservedReason` gave them", () => {
|
|
1957
|
+
for (const reason of ["read-failed", "no-binding", "unsupported-kind", "filtered"]) {
|
|
1958
|
+
expect(isBehaviourUnpredictedReason(reason)).toBe(true);
|
|
1959
|
+
}
|
|
1960
|
+
});
|
|
1961
|
+
});
|