@intentius/chant 0.65.0 → 0.66.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/behaviour-delta.d.ts +181 -0
- package/dist/behaviour-delta.d.ts.map +1 -0
- package/dist/behaviour-http.d.ts +106 -0
- package/dist/behaviour-http.d.ts.map +1 -0
- package/dist/behaviour-overlay.d.ts +61 -0
- package/dist/behaviour-overlay.d.ts.map +1 -0
- package/dist/behaviour.d.ts +178 -3
- package/dist/behaviour.d.ts.map +1 -1
- package/dist/cli/handlers/scenario.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/lifecycle/scenario-eval.d.ts +23 -5
- package/dist/lifecycle/scenario-eval.d.ts.map +1 -1
- package/dist/lifecycle/scenario.d.ts +45 -3
- package/dist/lifecycle/scenario.d.ts.map +1 -1
- package/dist/lifecycle/types.d.ts +17 -0
- package/dist/lifecycle/types.d.ts.map +1 -1
- package/dist/op/activities/activity-contracts.d.ts +42 -0
- package/dist/op/activities/activity-contracts.d.ts.map +1 -1
- package/dist/op/activities/index.d.ts +2 -0
- package/dist/op/activities/index.d.ts.map +1 -1
- package/dist/op/activities/predict-behaviour.d.ts +207 -0
- package/dist/op/activities/predict-behaviour.d.ts.map +1 -0
- package/dist/op/activities/reconcile.d.ts +7 -0
- package/dist/op/activities/reconcile.d.ts.map +1 -1
- package/dist/op/composites/behaviour-op.d.ts +57 -0
- package/dist/op/composites/behaviour-op.d.ts.map +1 -0
- package/dist/op/composites/index.d.ts +2 -0
- package/dist/op/composites/index.d.ts.map +1 -1
- package/dist/op/index.d.ts +2 -2
- package/dist/op/index.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/behaviour-delta.test.ts +331 -0
- package/src/behaviour-delta.ts +564 -0
- package/src/behaviour-http.test.ts +456 -0
- package/src/behaviour-http.ts +252 -0
- package/src/behaviour-overlay.test.ts +149 -0
- package/src/behaviour-overlay.ts +76 -0
- package/src/behaviour.test.ts +50 -0
- package/src/behaviour.ts +255 -3
- package/src/cli/handlers/scenario.test.ts +108 -0
- package/src/cli/handlers/scenario.ts +63 -16
- package/src/index.ts +2 -0
- package/src/lifecycle/scenario-cost.test.ts +183 -0
- package/src/lifecycle/scenario-eval.ts +133 -6
- package/src/lifecycle/scenario.ts +72 -4
- package/src/lifecycle/types.ts +17 -0
- package/src/lint/rules/op/ops012-activity-contract.test.ts +31 -0
- package/src/op/activities/activity-contracts.ts +49 -0
- package/src/op/activities/index.ts +24 -0
- package/src/op/activities/predict-behaviour.test.ts +255 -0
- package/src/op/activities/predict-behaviour.ts +468 -0
- package/src/op/activities/reconcile.ts +7 -2
- package/src/op/activity-contract-registry.test.ts +3 -0
- package/src/op/composites/behaviour-op.test.ts +56 -0
- package/src/op/composites/behaviour-op.ts +99 -0
- package/src/op/composites/index.ts +2 -0
- package/src/op/index.ts +2 -0
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `cost` clause on a scenario (#2358): its shape at declaration, and its
|
|
3
|
+
* evaluation against the fixture's recorded prediction.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { describe, expect, test } from "vitest";
|
|
7
|
+
import { EXPECT_KEYS, Scenario, snapshot } from "./scenario";
|
|
8
|
+
import { evaluateScenario, type ScenarioBehaviourFixture } from "./scenario-eval";
|
|
9
|
+
import type { ChangeSet } from "./change-set";
|
|
10
|
+
import {
|
|
11
|
+
behaviourReport,
|
|
12
|
+
noBehaviourEngineRefusal,
|
|
13
|
+
outOfCreditBehaviourEngineRefusal,
|
|
14
|
+
predictedRate,
|
|
15
|
+
type PredictedBehaviour,
|
|
16
|
+
type UnpredictedEntity,
|
|
17
|
+
} from "../behaviour";
|
|
18
|
+
|
|
19
|
+
const EMPTY: ChangeSet = { env: "prod", entries: [] };
|
|
20
|
+
const TRAFFIC = "100 rps, p50";
|
|
21
|
+
|
|
22
|
+
function figure(perHour: number, currency = "USD"): PredictedBehaviour {
|
|
23
|
+
return {
|
|
24
|
+
at: { traffic: TRAFFIC },
|
|
25
|
+
cost: predictedRate(perHour, currency),
|
|
26
|
+
headroom: { cpu: 0.5 },
|
|
27
|
+
errorRate: 0.001,
|
|
28
|
+
resilience: { failure: "one zone lost", verdict: "survives" },
|
|
29
|
+
provenance: { engine: "acme-sim", version: "1.4.2", tolerance: "±15%", basis: "modeled" },
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function fixture(
|
|
34
|
+
entities: Record<string, PredictedBehaviour>,
|
|
35
|
+
unpredicted: Record<string, UnpredictedEntity> = {},
|
|
36
|
+
total?: { perHour: number; currency: string },
|
|
37
|
+
): ScenarioBehaviourFixture {
|
|
38
|
+
return {
|
|
39
|
+
result: behaviourReport(
|
|
40
|
+
{ entityNames: [...Object.keys(entities), ...Object.keys(unpredicted)], traffic: TRAFFIC, edgeCoverage: { verdict: "unknown" } },
|
|
41
|
+
{ engine: "acme-sim", version: "1.4.2", ...(total ? { total: predictedRate(total.perHour, total.currency) } : {}) },
|
|
42
|
+
entities,
|
|
43
|
+
unpredicted,
|
|
44
|
+
),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function cost(verdict: ReturnType<typeof evaluateScenario>) {
|
|
49
|
+
return verdict.checks.find((c) => c.clause === "cost")!;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
describe("Scenario — the cost clause's shape", () => {
|
|
53
|
+
test("cost is a recognized expect key", () => {
|
|
54
|
+
expect(EXPECT_KEYS).toContain("cost");
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test("accepts a bound with a currency, and an optional entity", () => {
|
|
58
|
+
const s = Scenario("s", { given: snapshot("fixtures/prod.json"), expect: { cost: { maxPerHour: 12.5, currency: "USD" } } });
|
|
59
|
+
expect(s.expect.cost).toEqual({ maxPerHour: 12.5, currency: "USD" });
|
|
60
|
+
const e = Scenario("s", { given: snapshot("prod"), expect: { cost: { maxPerHour: 1, currency: " EUR ", entity: "db" } } });
|
|
61
|
+
expect(e.expect.cost).toEqual({ maxPerHour: 1, currency: "EUR", entity: "db" });
|
|
62
|
+
expect(Object.isFrozen(e.expect.cost)).toBe(true);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test("composes with the other clauses", () => {
|
|
66
|
+
const s = Scenario("s", { given: snapshot("prod"), expect: { noop: true, cost: { maxPerHour: 1, currency: "USD" } } });
|
|
67
|
+
expect(Object.keys(s.expect).sort()).toEqual(["cost", "noop"]);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
test("rejects a bound with no currency — chant converts nothing", () => {
|
|
71
|
+
expect(() =>
|
|
72
|
+
// @ts-expect-error deliberately omitting currency
|
|
73
|
+
Scenario("s", { given: snapshot("prod"), expect: { cost: { maxPerHour: 1 } } }),
|
|
74
|
+
).toThrow(/expect.cost.currency/);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test("rejects a negative, non-finite or non-numeric bound", () => {
|
|
78
|
+
expect(() => Scenario("s", { given: snapshot("prod"), expect: { cost: { maxPerHour: -1, currency: "USD" } } })).toThrow(/non-negative finite/);
|
|
79
|
+
expect(() => Scenario("s", { given: snapshot("prod"), expect: { cost: { maxPerHour: Number.NaN, currency: "USD" } } })).toThrow(/non-negative finite/);
|
|
80
|
+
expect(() =>
|
|
81
|
+
// @ts-expect-error deliberately passing a string
|
|
82
|
+
Scenario("s", { given: snapshot("prod"), expect: { cost: { maxPerHour: "12", currency: "USD" } } }),
|
|
83
|
+
).toThrow(/non-negative finite/);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test("rejects an unknown field inside cost, and a non-object cost", () => {
|
|
87
|
+
expect(() =>
|
|
88
|
+
// @ts-expect-error deliberately passing an unrecognized field
|
|
89
|
+
Scenario("s", { given: snapshot("prod"), expect: { cost: { maxPerHour: 1, currency: "USD", maxPerMonth: 700 } } }),
|
|
90
|
+
).toThrow(/unknown `expect.cost` field "maxPerMonth"/);
|
|
91
|
+
expect(() =>
|
|
92
|
+
// @ts-expect-error deliberately passing a number
|
|
93
|
+
Scenario("s", { given: snapshot("prod"), expect: { cost: 12 } }),
|
|
94
|
+
).toThrow(/must be \{ maxPerHour, currency, entity\? \}/);
|
|
95
|
+
});
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
describe("evaluateScenario — cost against the fixture's recorded prediction", () => {
|
|
99
|
+
test("a fixture with no behaviour block fails the clause by name — a bound cannot be checked against no figure", () => {
|
|
100
|
+
const verdict = evaluateScenario(EMPTY, { cost: { maxPerHour: 1, currency: "USD" } }, { missing: "given fixtures/prod.json carries no `behaviour` block" });
|
|
101
|
+
expect(verdict.pass).toBe(false);
|
|
102
|
+
expect(cost(verdict).detail).toMatch(/carries no `behaviour` block.*Record one/);
|
|
103
|
+
// And with nothing handed over at all.
|
|
104
|
+
expect(evaluateScenario(EMPTY, { cost: { maxPerHour: 1, currency: "USD" } }).pass).toBe(false);
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
test("a fixture whose prediction is a refusal fails with the refusal's reason, never a pass on nothing", () => {
|
|
108
|
+
const verdict = evaluateScenario(EMPTY, { cost: { maxPerHour: 1e9, currency: "USD" } }, { result: noBehaviourEngineRefusal("augur") });
|
|
109
|
+
expect(verdict.pass).toBe(false);
|
|
110
|
+
expect(cost(verdict).detail).toContain("the fixture's prediction is a refusal (no-engine)");
|
|
111
|
+
expect(cost(verdict).detail).toContain("Set CHANT_BEHAVIOUR_ENGINE to the engine's address.");
|
|
112
|
+
|
|
113
|
+
const broke = outOfCreditBehaviourEngineRefusal("augur", { value: "engine", source: "CHANT_BEHAVIOUR_ENGINE" }, "balance 0");
|
|
114
|
+
expect(cost(evaluateScenario(EMPTY, { cost: { maxPerHour: 1e9, currency: "USD" } }, { result: broke })).detail).toContain("engine-out-of-credit");
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
test("turns red when the fixture's figure exceeds the bound, naming both rates and the level", () => {
|
|
118
|
+
const verdict = evaluateScenario(EMPTY, { cost: { maxPerHour: 0.25, currency: "USD" } }, fixture({ db: figure(0.272) }));
|
|
119
|
+
expect(verdict.pass).toBe(false);
|
|
120
|
+
expect(cost(verdict).detail).toBe(
|
|
121
|
+
'exceeded: 0.272 USD/hour at "100 rps, p50" against a bound of 0.25 USD/hour; read from chant\'s own sum over 1 predicted entity (the engine states no total)',
|
|
122
|
+
);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
test("passes at or under the bound, and says on the pass which figure it read", () => {
|
|
126
|
+
const verdict = evaluateScenario(EMPTY, { cost: { maxPerHour: 0.272, currency: "USD" } }, fixture({ db: figure(0.272) }));
|
|
127
|
+
expect(verdict.pass).toBe(true);
|
|
128
|
+
expect(cost(verdict).detail).toContain("read from chant's own sum");
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
test("reads the engine's own total when it states one, rather than summing", () => {
|
|
132
|
+
const verdict = evaluateScenario(EMPTY, { cost: { maxPerHour: 1, currency: "USD" } }, fixture({ db: figure(0.272), web: figure(0.04) }, {}, { perHour: 1.5, currency: "USD" }));
|
|
133
|
+
expect(verdict.pass).toBe(false);
|
|
134
|
+
expect(cost(verdict).detail).toContain("1.5 USD/hour");
|
|
135
|
+
expect(cost(verdict).detail).toContain("read from the engine's own estate total (acme-sim 1.4.2)");
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
test("chant's sum names every declined entity, because those are in the estate and not in the sum", () => {
|
|
139
|
+
const verdict = evaluateScenario(
|
|
140
|
+
EMPTY,
|
|
141
|
+
{ cost: { maxPerHour: 1, currency: "USD" } },
|
|
142
|
+
fixture({ db: figure(0.272) }, { role: { type: "AWS::IAM::Role", reason: "unsupported-kind", detail: "a role is a grant" } }),
|
|
143
|
+
);
|
|
144
|
+
expect(verdict.pass).toBe(true);
|
|
145
|
+
expect(cost(verdict).detail).toContain("1 declined and not in the figure: role (unsupported-kind: a role is a grant)");
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test("bounds one entity's own rate when named, with its provenance", () => {
|
|
149
|
+
const fx = fixture({ db: figure(0.272), web: figure(5) });
|
|
150
|
+
expect(evaluateScenario(EMPTY, { cost: { maxPerHour: 0.3, currency: "USD", entity: "db" } }, fx).pass).toBe(true);
|
|
151
|
+
const red = evaluateScenario(EMPTY, { cost: { maxPerHour: 0.3, currency: "USD", entity: "web" } }, fx);
|
|
152
|
+
expect(red.pass).toBe(false);
|
|
153
|
+
expect(cost(red).detail).toContain("read from web's own rate (acme-sim 1.4.2, ±15%, modeled)");
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
test("a named entity the engine declined, or never asked about, fails with the reason", () => {
|
|
157
|
+
const fx = fixture({ db: figure(0.272) }, { role: { reason: "unsupported-kind", detail: "a role is a grant" } });
|
|
158
|
+
expect(cost(evaluateScenario(EMPTY, { cost: { maxPerHour: 1, currency: "USD", entity: "role" } }, fx)).detail).toContain(
|
|
159
|
+
'"role" was declined by the engine (unsupported-kind: a role is a grant)',
|
|
160
|
+
);
|
|
161
|
+
expect(cost(evaluateScenario(EMPTY, { cost: { maxPerHour: 1, currency: "USD", entity: "ghost" } }, fx)).detail).toContain(
|
|
162
|
+
'"ghost" is in neither the fixture\'s figures nor its declined entities',
|
|
163
|
+
);
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
test("a bound in another currency fails — chant converts nothing", () => {
|
|
167
|
+
const verdict = evaluateScenario(EMPTY, { cost: { maxPerHour: 1, currency: "EUR" } }, fixture({ db: figure(0.272) }));
|
|
168
|
+
expect(verdict.pass).toBe(false);
|
|
169
|
+
expect(cost(verdict).detail).toContain("the bound is in EUR and the figure is 0.272 USD/hour");
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
test("entities priced in two currencies with no engine total cannot be summed", () => {
|
|
173
|
+
const verdict = evaluateScenario(EMPTY, { cost: { maxPerHour: 1, currency: "USD" } }, fixture({ db: figure(0.272), eu: figure(0.1, "EUR") }));
|
|
174
|
+
expect(verdict.pass).toBe(false);
|
|
175
|
+
expect(cost(verdict).detail).toContain("priced in EUR, USD");
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
test("the cost clause sits beside the others in declaration order and does not touch them", () => {
|
|
179
|
+
const verdict = evaluateScenario(EMPTY, { noop: true, cost: { maxPerHour: 1, currency: "USD" } }, fixture({ db: figure(0.272) }));
|
|
180
|
+
expect(verdict.checks.map((c) => c.clause)).toEqual(["noop", "cost"]);
|
|
181
|
+
expect(verdict.pass).toBe(true);
|
|
182
|
+
});
|
|
183
|
+
});
|
|
@@ -17,7 +17,24 @@
|
|
|
17
17
|
import { summarize, type ChangeSet } from "./change-set";
|
|
18
18
|
import { evaluateUnobservedGate, type UnobservedGateFinding } from "./unobserved-gate";
|
|
19
19
|
import { unobservedReasonText } from "../observation";
|
|
20
|
-
import
|
|
20
|
+
import { isBehaviourRefusalReport, renderBehaviourRefusal, type BehaviourResult } from "../behaviour";
|
|
21
|
+
import { formatPerHour } from "../behaviour-delta";
|
|
22
|
+
import type {
|
|
23
|
+
ScenarioCostExpectation,
|
|
24
|
+
ScenarioDeleteExpectation,
|
|
25
|
+
ScenarioExpect,
|
|
26
|
+
ScenarioUnobservedPolicy,
|
|
27
|
+
} from "./scenario";
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* What stands in for the engine's answer when a scenario carries a `cost`
|
|
31
|
+
* clause (#2358): the fixture's `behaviour` block, or the reason there is
|
|
32
|
+
* none. The handler resolves it from the same fixture every other clause
|
|
33
|
+
* reads; this module only evaluates it.
|
|
34
|
+
*/
|
|
35
|
+
export type ScenarioBehaviourFixture =
|
|
36
|
+
| { readonly result: BehaviourResult }
|
|
37
|
+
| { readonly missing: string };
|
|
21
38
|
|
|
22
39
|
/**
|
|
23
40
|
* One clause's verdict — always present in {@link ScenarioVerdict.checks}, in
|
|
@@ -28,7 +45,11 @@ export interface ScenarioCheckResult {
|
|
|
28
45
|
/** Which `expect` clause this is (`"noop"`, `"create"`, `"deletes"`, …). */
|
|
29
46
|
clause: string;
|
|
30
47
|
pass: boolean;
|
|
31
|
-
/** Present when `pass` is false — what was expected vs what the plan
|
|
48
|
+
/** Present when `pass` is false — what was expected vs what the plan
|
|
49
|
+
* proposes, naming resources for delete/ownership failures. The `cost`
|
|
50
|
+
* clause carries it on a pass too, saying which figure was bounded and
|
|
51
|
+
* what the engine declined, so a passing bound is never read as a bound
|
|
52
|
+
* over the whole estate when it was not. */
|
|
32
53
|
detail?: string;
|
|
33
54
|
}
|
|
34
55
|
|
|
@@ -40,11 +61,17 @@ export interface ScenarioVerdict {
|
|
|
40
61
|
}
|
|
41
62
|
|
|
42
63
|
/**
|
|
43
|
-
* Evaluate `expect` against `cs`. Pure: reads `cs` and
|
|
44
|
-
*
|
|
45
|
-
*
|
|
64
|
+
* Evaluate `expect` against `cs`. Pure: reads `cs`, `expect` and, for a
|
|
65
|
+
* `cost` clause, the fixture's `behaviour` block handed over as `behaviour`;
|
|
66
|
+
* computes nothing else. Every clause present on `expect` is checked
|
|
67
|
+
* independently and every one contributes to `checks`; `pass` is true only
|
|
68
|
+
* when all of them are.
|
|
46
69
|
*/
|
|
47
|
-
export function evaluateScenario(
|
|
70
|
+
export function evaluateScenario(
|
|
71
|
+
cs: ChangeSet,
|
|
72
|
+
expect: ScenarioExpect,
|
|
73
|
+
behaviour?: ScenarioBehaviourFixture,
|
|
74
|
+
): ScenarioVerdict {
|
|
48
75
|
const counts = summarize(cs);
|
|
49
76
|
const checks: ScenarioCheckResult[] = [];
|
|
50
77
|
|
|
@@ -91,9 +118,109 @@ export function evaluateScenario(cs: ChangeSet, expect: ScenarioExpect): Scenari
|
|
|
91
118
|
checks.push(evaluateUnobservedClause(cs, expect.unobserved));
|
|
92
119
|
}
|
|
93
120
|
|
|
121
|
+
if (expect.cost !== undefined) {
|
|
122
|
+
checks.push(evaluateCostClause(expect.cost, behaviour));
|
|
123
|
+
}
|
|
124
|
+
|
|
94
125
|
return { pass: checks.every((c) => c.pass), checks };
|
|
95
126
|
}
|
|
96
127
|
|
|
128
|
+
/**
|
|
129
|
+
* The `cost` clause (#2358). Three refusals before any number is read, each
|
|
130
|
+
* naming why, because the alternative to each is a bound that passes on
|
|
131
|
+
* nothing:
|
|
132
|
+
*
|
|
133
|
+
* - no `behaviour` block in the fixture — nothing was predicted;
|
|
134
|
+
* - the block is a refusal — the engine was absent, unreachable, out of
|
|
135
|
+
* credit, over quota, or refused a credential, and the fixture carries
|
|
136
|
+
* that refusal's own text;
|
|
137
|
+
* - the named entity is not in the fixture's figures — declined with a
|
|
138
|
+
* reason, or never asked about.
|
|
139
|
+
*
|
|
140
|
+
* The figure is then one of three, and the detail says which: the named
|
|
141
|
+
* entity's own rate, the engine's stated total, or chant's sum over every
|
|
142
|
+
* predicted entity when the engine states no total. The sum is chant's own
|
|
143
|
+
* arithmetic and is labelled so; it also names every entity the engine
|
|
144
|
+
* declined, because those are in the estate and not in the sum.
|
|
145
|
+
*/
|
|
146
|
+
function evaluateCostClause(
|
|
147
|
+
bound: ScenarioCostExpectation,
|
|
148
|
+
behaviour: ScenarioBehaviourFixture | undefined,
|
|
149
|
+
): ScenarioCheckResult {
|
|
150
|
+
const fail = (detail: string): ScenarioCheckResult => ({ clause: "cost", pass: false, detail });
|
|
151
|
+
|
|
152
|
+
if (behaviour === undefined || "missing" in behaviour) {
|
|
153
|
+
return fail(
|
|
154
|
+
`${behaviour?.missing ?? "the fixture carries no `behaviour` block"} — a cost bound is checked against the ` +
|
|
155
|
+
"fixture's recorded prediction, and there is none. Record one on the snapshot, or drop the clause.",
|
|
156
|
+
);
|
|
157
|
+
}
|
|
158
|
+
const result = behaviour.result;
|
|
159
|
+
if (isBehaviourRefusalReport(result)) {
|
|
160
|
+
return fail(
|
|
161
|
+
`the fixture's prediction is a refusal (${result.refusal.cause}), and a bound cannot be checked against no ` +
|
|
162
|
+
`figure: ${renderBehaviourRefusal(result.refusal, { color: false }).replace(/\n\s*/g, " ")}`,
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const declined = Object.entries(result.unpredicted ?? {}).map(
|
|
167
|
+
([name, u]) => `${name} (${u.reason}${u.detail ? `: ${u.detail}` : ""})`,
|
|
168
|
+
);
|
|
169
|
+
const declinedNote = declined.length > 0 ? `; ${declined.length} declined and not in the figure: ${declined.join(", ")}` : "";
|
|
170
|
+
|
|
171
|
+
let perHour: number;
|
|
172
|
+
let currency: string;
|
|
173
|
+
let which: string;
|
|
174
|
+
if (bound.entity !== undefined) {
|
|
175
|
+
const figure = Object.prototype.hasOwnProperty.call(result.entities, bound.entity)
|
|
176
|
+
? result.entities[bound.entity]
|
|
177
|
+
: undefined;
|
|
178
|
+
if (!figure) {
|
|
179
|
+
const hole = result.unpredicted?.[bound.entity];
|
|
180
|
+
return fail(
|
|
181
|
+
hole
|
|
182
|
+
? `"${bound.entity}" was declined by the engine (${hole.reason}${hole.detail ? `: ${hole.detail}` : ""}), so it has no figure to bound`
|
|
183
|
+
: `"${bound.entity}" is in neither the fixture's figures nor its declined entities — the prediction never asked about it`,
|
|
184
|
+
);
|
|
185
|
+
}
|
|
186
|
+
perHour = figure.cost.perHour;
|
|
187
|
+
currency = figure.cost.currency;
|
|
188
|
+
which = `${bound.entity}'s own rate (${figure.provenance.engine} ${figure.provenance.version}, ${figure.provenance.tolerance}, ${figure.provenance.basis})`;
|
|
189
|
+
} else if (result.meta.total) {
|
|
190
|
+
perHour = result.meta.total.perHour;
|
|
191
|
+
currency = result.meta.total.currency;
|
|
192
|
+
which = `the engine's own estate total (${result.meta.engine} ${result.meta.version})${declinedNote}`;
|
|
193
|
+
} else {
|
|
194
|
+
const names = Object.keys(result.entities);
|
|
195
|
+
const currencies = new Set(names.map((n) => result.entities[n].cost.currency));
|
|
196
|
+
if (currencies.size > 1) {
|
|
197
|
+
return fail(
|
|
198
|
+
`the engine states no total and the entities are priced in ${[...currencies].sort().join(", ")} — chant converts ` +
|
|
199
|
+
"nothing, so there is no one sum to bound",
|
|
200
|
+
);
|
|
201
|
+
}
|
|
202
|
+
if (names.length === 0) {
|
|
203
|
+
return fail(`the engine states no total and predicted no entity${declinedNote} — nothing to bound`);
|
|
204
|
+
}
|
|
205
|
+
perHour = names.reduce((sum, n) => sum + result.entities[n].cost.perHour, 0);
|
|
206
|
+
currency = [...currencies][0];
|
|
207
|
+
which = `chant's own sum over ${names.length} predicted entit${names.length === 1 ? "y" : "ies"} (the engine states no total)${declinedNote}`;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const level = result.meta.at.traffic;
|
|
211
|
+
if (currency !== bound.currency) {
|
|
212
|
+
return fail(
|
|
213
|
+
`the bound is in ${bound.currency} and the figure is ${formatPerHour(perHour)} ${currency}/hour at "${level}" — ` +
|
|
214
|
+
`chant converts nothing; read from ${which}`,
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
const pass = perHour <= bound.maxPerHour;
|
|
218
|
+
const detail =
|
|
219
|
+
`${pass ? "" : "exceeded: "}${formatPerHour(perHour)} ${currency}/hour at "${level}" against a bound of ` +
|
|
220
|
+
`${formatPerHour(bound.maxPerHour)} ${bound.currency}/hour; read from ${which}`;
|
|
221
|
+
return { clause: "cost", pass, detail };
|
|
222
|
+
}
|
|
223
|
+
|
|
97
224
|
/** Names (with type) of every entry matching one of `actions`, for a legible failure message. */
|
|
98
225
|
function nameList(cs: ChangeSet, actions: readonly string[]): string {
|
|
99
226
|
const names = cs.entries
|
|
@@ -107,12 +107,48 @@ export interface ScenarioDeleteExpectation {
|
|
|
107
107
|
*/
|
|
108
108
|
export type ScenarioUnobservedPolicy = "refuse" | { readonly allow: readonly string[] };
|
|
109
109
|
|
|
110
|
+
/**
|
|
111
|
+
* A bound on one predicted figure in the fixture (#2358).
|
|
112
|
+
*
|
|
113
|
+
* A scenario has one fixture, not a before-and-after pair, so this bounds a
|
|
114
|
+
* **single** predicted rate rather than a delta: the fixture's own
|
|
115
|
+
* `behaviour` block (a `BehaviourResult` recorded on the `LifecycleSnapshot`,
|
|
116
|
+
* ../lifecycle/types.ts) stands in for the engine's answer the way the rest
|
|
117
|
+
* of the snapshot stands in for a live read, and the clause asks whether one
|
|
118
|
+
* figure in it is at or under `maxPerHour`.
|
|
119
|
+
*
|
|
120
|
+
* Which figure: `entity`'s own `cost.perHour` when named; otherwise the
|
|
121
|
+
* estate's — the engine's own `meta.total` when it states one, and chant's
|
|
122
|
+
* sum over every predicted entity when it does not. The verdict's detail says
|
|
123
|
+
* which of the three it read, and names every entity the engine declined,
|
|
124
|
+
* because a sum over an estate with declined entities is a sum over less
|
|
125
|
+
* than the estate and the reader should know it.
|
|
126
|
+
*
|
|
127
|
+
* The bound is a rate for one hypothetical hour at the fixture's stated
|
|
128
|
+
* traffic level, never an amount; `currency` is required and compared
|
|
129
|
+
* exactly, because chant converts nothing and a bound in USD says nothing
|
|
130
|
+
* about a figure in EUR.
|
|
131
|
+
*
|
|
132
|
+
* A fixture whose `behaviour` block is a refusal, or that carries none, fails
|
|
133
|
+
* the clause with the refusal's reason. A bound cannot be checked against no
|
|
134
|
+
* figure, and passing would be the faked number the epic forbids.
|
|
135
|
+
*/
|
|
136
|
+
export interface ScenarioCostExpectation {
|
|
137
|
+
/** The figure must be at or under this many `currency` per hour. */
|
|
138
|
+
readonly maxPerHour: number;
|
|
139
|
+
/** ISO 4217 code the figure must be stated in. Compared exactly. */
|
|
140
|
+
readonly currency: string;
|
|
141
|
+
/** Bound one entity's figure instead of the estate's. */
|
|
142
|
+
readonly entity?: string;
|
|
143
|
+
}
|
|
144
|
+
|
|
110
145
|
/**
|
|
111
146
|
* The assertion vocabulary (#1292 research, settled shape). Every clause is
|
|
112
147
|
* independently optional and clauses compose within one `expect` object — a
|
|
113
|
-
* scenario can assert `noop`, exact counts, specific named deletes,
|
|
114
|
-
* unobserved policy together. At least one clause must be
|
|
115
|
-
* `expect` asserts nothing and is refused as a likely
|
|
148
|
+
* scenario can assert `noop`, exact counts, specific named deletes, an
|
|
149
|
+
* unobserved policy and a cost bound together. At least one clause must be
|
|
150
|
+
* present; an empty `expect` asserts nothing and is refused as a likely
|
|
151
|
+
* mistake.
|
|
116
152
|
*/
|
|
117
153
|
export interface ScenarioExpect {
|
|
118
154
|
/** The plan proposes no create, update, or delete, and no declared effect
|
|
@@ -135,11 +171,15 @@ export interface ScenarioExpect {
|
|
|
135
171
|
/** How the scenario treats unobserved rows. Omitted = unconstrained (an
|
|
136
172
|
* unobserved entity neither passes nor fails the scenario on its own). */
|
|
137
173
|
readonly unobserved?: ScenarioUnobservedPolicy;
|
|
174
|
+
/** One predicted figure in the fixture's `behaviour` block is at or under
|
|
175
|
+
* a stated rate per hour (#2358). See {@link ScenarioCostExpectation} for
|
|
176
|
+
* which figure, and for why a refusal in the fixture fails it. */
|
|
177
|
+
readonly cost?: ScenarioCostExpectation;
|
|
138
178
|
}
|
|
139
179
|
|
|
140
180
|
/** Every recognized `expect` clause key — for the excess-key check below and
|
|
141
181
|
* for exhaustiveness at call sites. */
|
|
142
|
-
const EXPECT_KEYS = ["noop", "create", "update", "delete", "deletes", "unobserved"] as const;
|
|
182
|
+
export const EXPECT_KEYS = ["noop", "create", "update", "delete", "deletes", "unobserved", "cost"] as const;
|
|
143
183
|
|
|
144
184
|
const OWNERSHIP_VALUES: readonly Ownership[] = ["owned", "foreign", "unknown"];
|
|
145
185
|
|
|
@@ -293,6 +333,34 @@ function validateExpect(name: string, expect: unknown): ScenarioExpect {
|
|
|
293
333
|
);
|
|
294
334
|
}
|
|
295
335
|
}
|
|
336
|
+
if ("cost" in e) {
|
|
337
|
+
const c = e.cost;
|
|
338
|
+
if (typeof c !== "object" || c === null || Array.isArray(c)) {
|
|
339
|
+
throw new Error(`Scenario("${name}"): \`expect.cost\` must be { maxPerHour, currency, entity? }`);
|
|
340
|
+
}
|
|
341
|
+
const { maxPerHour, currency, entity } = c as Record<string, unknown>;
|
|
342
|
+
if (typeof maxPerHour !== "number" || !Number.isFinite(maxPerHour) || maxPerHour < 0) {
|
|
343
|
+
throw new Error(`Scenario("${name}"): \`expect.cost.maxPerHour\` must be a non-negative finite number`);
|
|
344
|
+
}
|
|
345
|
+
if (typeof currency !== "string" || currency.trim() === "") {
|
|
346
|
+
throw new Error(
|
|
347
|
+
`Scenario("${name}"): \`expect.cost.currency\` must be a non-empty ISO 4217 code — chant converts nothing, so a bound needs its currency`,
|
|
348
|
+
);
|
|
349
|
+
}
|
|
350
|
+
if (entity !== undefined && (typeof entity !== "string" || entity.length === 0)) {
|
|
351
|
+
throw new Error(`Scenario("${name}"): \`expect.cost.entity\`, when present, must be a non-empty string`);
|
|
352
|
+
}
|
|
353
|
+
for (const key of Object.keys(c as object)) {
|
|
354
|
+
if (!["maxPerHour", "currency", "entity"].includes(key)) {
|
|
355
|
+
throw new Error(`Scenario("${name}"): unknown \`expect.cost\` field "${key}" — expected maxPerHour, currency, entity`);
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
out.cost = Object.freeze({
|
|
359
|
+
maxPerHour,
|
|
360
|
+
currency: currency.trim(),
|
|
361
|
+
...(entity !== undefined ? { entity } : {}),
|
|
362
|
+
});
|
|
363
|
+
}
|
|
296
364
|
|
|
297
365
|
return Object.freeze(out) as ScenarioExpect;
|
|
298
366
|
}
|
package/src/lifecycle/types.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { ResourceMetadata, ArtifactMetadata } from "../lexicon";
|
|
2
2
|
import type { UnobservedEntity } from "../observation";
|
|
3
3
|
import type { DeepResourceObservation } from "../deep-observation";
|
|
4
|
+
import type { BehaviourResult } from "../behaviour";
|
|
4
5
|
import type { IREdge } from "../graph-ir";
|
|
5
6
|
|
|
6
7
|
export type { ResourceMetadata, ArtifactMetadata } from "../lexicon";
|
|
@@ -84,6 +85,22 @@ export interface LifecycleSnapshot {
|
|
|
84
85
|
stackExports?: Record<string, Record<string, unknown>>;
|
|
85
86
|
/** Build digest at snapshot time — what was declared when this snapshot was taken */
|
|
86
87
|
digest?: BuildDigest;
|
|
88
|
+
/**
|
|
89
|
+
* What a behaviour engine said about this estate, when one was asked
|
|
90
|
+
* (#2358): the whole `BehaviourResult`, a report or a named refusal,
|
|
91
|
+
* exactly as `predictBehaviour()` returned it.
|
|
92
|
+
*
|
|
93
|
+
* Carried on the snapshot rather than in a fixture of its own because a
|
|
94
|
+
* `Scenario` has one `given`, and that one fixture stands in for everything
|
|
95
|
+
* a live read would have answered — presence, ownership, and now the
|
|
96
|
+
* engine's figures. A scenario's `expect.cost` clause reads this block;
|
|
97
|
+
* `chant scenario check` refuses the clause, naming the reason, when the
|
|
98
|
+
* block is absent or is a refusal, because a bound checked against no figure
|
|
99
|
+
* would pass on nothing. Absent on every snapshot written before this and
|
|
100
|
+
* on every snapshot recorded without an engine, which is read as "no
|
|
101
|
+
* prediction was made" rather than as a prediction of nothing.
|
|
102
|
+
*/
|
|
103
|
+
behaviour?: BehaviourResult;
|
|
87
104
|
}
|
|
88
105
|
|
|
89
106
|
/**
|
|
@@ -95,6 +95,37 @@ describe("OPS012: activity-contract", () => {
|
|
|
95
95
|
expect(diags[0].entity).toBe("op");
|
|
96
96
|
});
|
|
97
97
|
|
|
98
|
+
test("validates a step calling predictBehaviour or behaviourFinding against core's own table (#2358)", () => {
|
|
99
|
+
const ok = makeCtxFromEntities(new Map([
|
|
100
|
+
["op", opEntity("pr-behaviour", [
|
|
101
|
+
{ kind: "activity", fn: "predictBehaviour", args: { environment: "prod", traffic: "1000 rps, p99" } },
|
|
102
|
+
{
|
|
103
|
+
kind: "activity",
|
|
104
|
+
fn: "behaviourFinding",
|
|
105
|
+
args: { environment: "prod", traffic: "1000 rps, p99", op: "pr-behaviour", mode: "comment" },
|
|
106
|
+
outcomeAttribute: { name: "Comment", from: "commentUrl" },
|
|
107
|
+
},
|
|
108
|
+
])],
|
|
109
|
+
]));
|
|
110
|
+
expect(ops012.check(ok)).toHaveLength(0);
|
|
111
|
+
|
|
112
|
+
const typo = makeCtxFromEntities(new Map([
|
|
113
|
+
["op", opEntity("pr-behaviour", [
|
|
114
|
+
{ kind: "activity", fn: "behaviourFinding", args: { enviroment: "prod", traffic: "1000 rps, p99", op: "pr-behaviour" } },
|
|
115
|
+
])],
|
|
116
|
+
]));
|
|
117
|
+
const diags = ops012.check(typo);
|
|
118
|
+
expect(diags.length).toBeGreaterThan(0);
|
|
119
|
+
expect(diags.some((d) => d.message.includes("enviroment"))).toBe(true);
|
|
120
|
+
|
|
121
|
+
const badMode = makeCtxFromEntities(new Map([
|
|
122
|
+
["op", opEntity("pr-behaviour", [
|
|
123
|
+
{ kind: "activity", fn: "behaviourFinding", args: { environment: "prod", traffic: "1000 rps, p99", op: "pr-behaviour", mode: "issue" } },
|
|
124
|
+
])],
|
|
125
|
+
]));
|
|
126
|
+
expect(ops012.check(badMode).some((d) => d.message.includes("mode"))).toBe(true);
|
|
127
|
+
});
|
|
128
|
+
|
|
98
129
|
test("errors on an outcomeAttribute.from path that doesn't exist on the declared return type", () => {
|
|
99
130
|
const ctx = makeCtxFromEntities(new Map([
|
|
100
131
|
["op", opEntity("deploy", [
|
|
@@ -64,3 +64,52 @@ export const chantTeardownContract = activityContract(
|
|
|
64
64
|
"chantTeardown",
|
|
65
65
|
z.strictObject({ path: z.string() }),
|
|
66
66
|
);
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The prediction (#2358). `args` mirrors `PredictBehaviourArgs` field for
|
|
70
|
+
* field; the return is the contract's `BehaviourResult`, a report or a
|
|
71
|
+
* refusal, so `returns` is the envelope both arms share and a step's
|
|
72
|
+
* `outcomeAttribute.from` may name `behaviour` or `refusal`.
|
|
73
|
+
*/
|
|
74
|
+
export const predictBehaviourContract = activityContract(
|
|
75
|
+
"predictBehaviour",
|
|
76
|
+
z.strictObject({
|
|
77
|
+
environment: z.string(),
|
|
78
|
+
traffic: z.string(),
|
|
79
|
+
stack: z.string().optional(),
|
|
80
|
+
region: z.string().optional(),
|
|
81
|
+
owned: z.boolean().optional(),
|
|
82
|
+
}),
|
|
83
|
+
z.object({ behaviour: z.literal("v1"), refusal: z.unknown().optional() }),
|
|
84
|
+
);
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* The pull-request finding (#2358). Same inputs as the prediction plus the
|
|
88
|
+
* Op's name (the marker's key, #2319), the mode, and an optional explicit
|
|
89
|
+
* base branch. `returns` names what a step reads as an outcome — the posted
|
|
90
|
+
* URL, and whether either side refused.
|
|
91
|
+
*/
|
|
92
|
+
export const behaviourFindingContract = activityContract(
|
|
93
|
+
"behaviourFinding",
|
|
94
|
+
z.strictObject({
|
|
95
|
+
environment: z.string(),
|
|
96
|
+
traffic: z.string(),
|
|
97
|
+
op: z.string(),
|
|
98
|
+
mode: z.enum(["comment", "report"]).optional(),
|
|
99
|
+
base: z.string().optional(),
|
|
100
|
+
title: z.string().optional(),
|
|
101
|
+
stack: z.string().optional(),
|
|
102
|
+
region: z.string().optional(),
|
|
103
|
+
owned: z.boolean().optional(),
|
|
104
|
+
}),
|
|
105
|
+
z.object({
|
|
106
|
+
mode: z.enum(["comment", "report"]),
|
|
107
|
+
base: z.string(),
|
|
108
|
+
head: z.string(),
|
|
109
|
+
refused: z.boolean(),
|
|
110
|
+
summary: z.string(),
|
|
111
|
+
commentUrl: z.string().optional(),
|
|
112
|
+
pullRequest: z.string().optional(),
|
|
113
|
+
mergeRequest: z.string().optional(),
|
|
114
|
+
}),
|
|
115
|
+
);
|
|
@@ -124,3 +124,27 @@ export type {
|
|
|
124
124
|
GhRunner,
|
|
125
125
|
ApplyBumpFn,
|
|
126
126
|
} from "./lexicon-upgrade";
|
|
127
|
+
|
|
128
|
+
// The behaviour prediction and the pull-request finding built on it (#2358).
|
|
129
|
+
// Only the two activities and the pure helpers are re-exported: the registry
|
|
130
|
+
// collects every exported function as an activity, so `createBehaviourFinding`
|
|
131
|
+
// and `checkoutBase` stay on the module.
|
|
132
|
+
export {
|
|
133
|
+
predictBehaviour,
|
|
134
|
+
behaviourFinding,
|
|
135
|
+
behaviourFindingMarker,
|
|
136
|
+
declaredEdgeCoverage,
|
|
137
|
+
baseRefFrom,
|
|
138
|
+
headRefFrom,
|
|
139
|
+
noBaseRefMessage,
|
|
140
|
+
noPredictingLexiconMessage,
|
|
141
|
+
ambiguousPredictorMessage,
|
|
142
|
+
} from "./predict-behaviour";
|
|
143
|
+
export type {
|
|
144
|
+
PredictBehaviourArgs,
|
|
145
|
+
BehaviourFindingArgs,
|
|
146
|
+
BehaviourFindingResult,
|
|
147
|
+
BehaviourFindingMode,
|
|
148
|
+
BehaviourFindingDeps,
|
|
149
|
+
BaseCheckout,
|
|
150
|
+
} from "./predict-behaviour";
|