@sdods/core 0.2.2 → 0.3.0

package/dist/steps/perf.steps.d.ts

@@ -0,0 +1,248 @@
+import './params.js';
+import type { Page, TestInfo } from '@playwright/test';
+import type { EnvConfig, PerformanceMetrics } from '@sdods/contracts';
+import type { ApiClient, HttpMethod } from '../api/client.js';
+import type { ApiContext } from '../fixtures/api-context.js';
+import type { ScenarioMeta } from '../fixtures/scenario.js';
+/**
+ * Performance budgets.
+ *
+ * `PerfBudgetsSchema` (pageLoadMs / lcpMs / fcpMs / ttfbMs / apiP95Ms), the `perf.budgets` block in
+ * the project and env yaml, the `perfBudgets: true` release gate and `attachmentNames.perf` all
+ * existed before this file; nothing read any of them. These steps close that loop.
+ *
+ * Two rules run through every step here, both learned from hand-written project steps that got
+ * them wrong:
+ *
+ *  1. A BUDGET IS READ FROM CONFIG, NEVER FROM THE FEATURE FILE. A number typed into a `.feature`
+ *     is a number that drifts from the environment it runs on — staging and a laptop do not share
+ *     a page-load budget. `Then the response time should be under {int} ms` (api.steps.ts) is the
+ *     literal-threshold form and stays exactly as it is; the budget-driven forms below are new
+ *     patterns, not a widening of it. A local literal is still reachable where a scenario genuinely
+ *     needs one — `should be under {int} ms` — so the escape hatch is visible in the step text.
+ *  2. AN UNMEASURED VITAL FAILS. LCP does not fire on a page with no contentful paint, and WebKit
+ *     does not implement it at all; paint timings and the navigation entry can be missing too.
+ *     Recording those as `0` makes every budget assertion pass — a whole perf module
+ *     goes green while measuring nothing. So a vital that did not fire is recorded as `null`
+ *     together with the reason, and asserting on it FAILS naming that reason. A real navigation
+ *     cannot produce a genuine 0 ms vital (they are all offsets from navigation start), so treating
+ *     "absent or 0" as "not measured" loses no honest signal.
+ *
+ * A budget that is not configured is likewise a hard `CONFIG_INVALID` naming the missing key, not
+ * a skip: silently skipping is the same vacuous green in a different costume.
+ *
+ * The two comparison operators are deliberately different and deliberately visible in the step
+ * text: `under {int} ms` is strict `<` (matching `the response time should be under {int} ms` in
+ * api.steps.ts), `within its configured budget` is `<=` — a budget is a ceiling you may sit on.
+ */
+/** The page vitals SDODS records. Each name is also its key in `perf.budgets`. */
+export declare const VITAL_KEYS: readonly ["pageLoadMs", "lcpMs", "fcpMs", "ttfbMs"];
+export type VitalKey = (typeof VITAL_KEYS)[number];
+/** The `perf.budgets` key a sampled API latency distribution is judged against. */
+export declare const API_P95_BUDGET_KEY = "apiP95Ms";
+type BudgetLayer = {
+    budgets?: Record<string, number | undefined>;
+} | undefined;
+/** The slice of a resolved config these steps read. Nothing here writes config. */
+export interface BudgetConfig {
+    project: {
+        perf?: BudgetLayer;
+    };
+    env: {
+        name?: string;
+        perf?: BudgetLayer;
+    };
+}
+/**
+ * Effective budgets, env overriding project per key.
+ *
+ * `resolveConfig` already deep-merges `env.perf` into `config.project.perf`, so on a resolved
+ * config the project side is the whole answer. Merging the env side again is idempotent and keeps
+ * this correct for a config assembled by hand. Non-numeric values are dropped rather than spread —
+ * an explicit `undefined` in the env layer would otherwise erase a real project budget and turn a
+ * configured assertion into an unconfigured one.
+ */
+export declare function resolvedBudgets(config: BudgetConfig): Record<string, number>;
+/**
+ * The configured budget for one key, or a hard error naming it.
+ *
+ * PROVES the assertion that follows is gated on something real. A missing budget must never
+ * degrade to a skip or to `Infinity`: an unconfigured budget is the single failure mode that lets
+ * an entire perf suite report success while asserting nothing at all.
+ */
+export declare function requireBudget(config: BudgetConfig, key: string): number;
+/** One `When I record the page vitals`. `null` in `vitals` means NOT MEASURED, never "0 ms". */
+export interface VitalsRecording {
+    url: string;
+    vitals: Record<VitalKey, number | null>;
+    /** Why a vital is null, keyed by vital. Surfaced verbatim in the failure message. */
+    unmeasured: Partial<Record<VitalKey, string>>;
+    /** The contract shape written to `sdods/perf/<step>` and ingested into `steps.perf_json`. */
+    metrics: PerformanceMetrics;
+}
+export interface LatencySample {
+    status: number;
+    ms: number;
+}
+export interface LatencySampleSet {
+    method: string;
+    path: string;
+    samples: LatencySample[];
+    p95Ms: number;
+}
+interface PerfStore {
+    recordings: VitalsRecording[];
+    sampleSets: LatencySampleSet[];
+}
+export declare function perfStore(key: object): PerfStore;
+/**
+ * Nearest-rank p95 — deterministic for a given set of samples, with no interpolation to argue
+ * about. Below 20 samples the nearest rank IS the maximum; the sample count travels in the
+ * attachment and in every failure message so a reader can see when "p95" means "slowest of five".
+ *
+ * Throws rather than returning 0 on an empty sample: a p95 of 0 slides under every budget.
+ */
+export declare function p95Of(msValues: readonly number[]): number;
+/** What the page hands back. Every timing is `number | null`; null always means "did not fire". */
+export interface RawVitals {
+    hasNavigationEntry: boolean;
+    navigationStartEpochMs: number | null;
+    pageLoadMs: number | null;
+    ttfbMs: number | null;
+    fcpMs: number | null;
+    lcpMs: number | null;
+    domContentLoadedMs: number | null;
+    totalResources: number;
+    totalResourceSizeKB: number;
+    lcpSupported: boolean;
+    fcpSupported: boolean;
+}
+/**
+ * Runs inside the page. Serialised by Playwright, so it closes over nothing but its argument — the
+ * type aliases above are erased at compile time and are safe to reference; the constants are not,
+ * which is why the settle window is passed in.
+ */
+export declare function collectRawVitals(settleMs: number): Promise<RawVitals>;
+/**
+ * Turns the page's answer into a recording, capturing WHY a vital is null while the browser's
+ * capabilities are still in hand. By the time a `Then` step fails, "lcpMs was not measured" on its
+ * own sends the reader hunting for a bug in a page that is merely running in a browser that does
+ * not implement LCP (WebKit). Support is read from `PerformanceObserver.supportedEntryTypes`, never
+ * assumed from the browser name.
+ */
+export declare function toRecording(raw: RawVitals, url: string, browserName?: string): VitalsRecording;
+interface VitalsFixtures {
+    page: Page;
+    config: BudgetConfig;
+    scenario: ScenarioMeta;
+    $bddContext: {
+        stepIndex: number;
+    };
+    $testInfo: TestInfo;
+}
+/**
+ * PROVES that the current navigation produced real page vitals, and pins them to this step so a
+ * later assertion has something that can actually fail. Records LCP / FCP / TTFB / load plus the
+ * resource count and weight, as the `PerformanceMetrics` contract, under `sdods/perf/<step>` — the
+ * name `attachmentNames.perf` has always defined and the DB ingest has always read into
+ * `steps.perf_json`, and which nothing had ever written.
+ *
+ * TRAP: recording before anything navigated. `performance.getEntriesByType('navigation')` is empty
+ * on a page that has not loaded a document, so every vital comes back null. That is step misuse
+ * rather than a slow application, so it fails here as `RUN_FAILED` naming the fix, instead of as
+ * four confusing assertion failures later.
+ */
+export declare const recordPageVitals: ({ page, config, scenario, $bddContext, $testInfo, }: VitalsFixtures) => Promise<void>;
+interface AssertFixtures {
+    scenario: ScenarioMeta;
+    config: BudgetConfig;
+    apiContext: ApiContext;
+    env: EnvConfig;
+}
+/**
+ * PROVES a recorded vital sits inside the ceiling THIS environment declares. The budget is read
+ * from `perf.budgets` and is never a step argument: a threshold written into a feature file ships
+ * unchanged to every environment, which is exactly what a per-environment budget exists to prevent.
+ *
+ * Ordered so the cheapest fix surfaces first: forgot the recording → typo'd the vital name →
+ * budget missing from config → vital never fired → the comparison itself.
+ */
+export declare const assertVitalWithinBudget: ({ scenario, config, apiContext, env }: AssertFixtures, metricName: string) => Promise<void>;
+/**
+ * PROVES a recorded vital sits under a limit this scenario owns. The visible escape hatch from the
+ * configured budget, for a page whose limit the environment-wide budget cannot express. Strictly
+ * `<`, matching `the response time should be under {int} ms`.
+ */
+export declare const assertVitalUnderThreshold: ({ scenario, apiContext, env }: AssertFixtures, metricName: string, maxMs: number) => Promise<void>;
+/**
+ * PROVES a second navigation is not slower than the first — the caching / warm-path claim.
+ * Compares the last two recordings of one vital, so a scenario records, acts, and records again.
+ *
+ * Carries an explicit tolerance because both sides are live measurements: the zero-tolerance form
+ * (`plus 0 ms`) is available, but it is the author's decision in the feature file rather than a
+ * hidden fudge factor here. Fails outright with fewer than two recordings, because comparing a
+ * recording with itself always passes.
+ */
+export declare const assertVitalNoWorseThanPrevious: ({ scenario, apiContext, env }: AssertFixtures, metricName: string, toleranceMs: number) => Promise<void>;
+/**
+ * PROVES the budgets a perf suite depends on actually exist for the environment being run.
+ *
+ * This step guards the other steps. A missing or partial `perf.budgets` block is invisible
+ * everywhere else in a run: the release gate reads `perfBudgets: true`, the suite goes green, and
+ * nobody learns the ceiling was never set. Put this in the first scenario of a perf module and the
+ * gap fails loudly, once, naming the key.
+ */
+export declare const assertBudgetsConfigured: ({ config, apiContext, env }: AssertFixtures, names: string) => Promise<void>;
+interface SampleFixtures {
+    api: ApiClient;
+    apiContext: ApiContext;
+    env: EnvConfig;
+    scenario: ScenarioMeta;
+    $bddContext: {
+        stepIndex: number;
+    };
+    $testInfo: TestInfo;
+}
+/**
+ * PROVES an endpoint's latency DISTRIBUTION rather than one lucky request. `the response time
+ * should be under {int} ms` (api.steps.ts) times a single call — the measurement most likely to be
+ * a cold start or a cache hit. A p95 needs a sample, and SDODS had no way to take one.
+ */
+export declare const sampleLatency: ({ api, apiContext, env, scenario, $bddContext, $testInfo }: SampleFixtures, method: HttpMethod, path: string, times: number) => Promise<void>;
+/** PROVES the same for a write path, where the latency that matters belongs to a real payload. */
+export declare const sampleLatencyWithBody: ({ api, apiContext, env, scenario, $bddContext, $testInfo }: SampleFixtures, method: HttpMethod, path: string, times: number, body: string) => Promise<void>;
+/**
+ * PROVES the sampled p95 sits inside the environment's `apiP95Ms` ceiling. The budget key is fixed
+ * and read from config — nothing to mistype in the feature file, nothing to drift.
+ *
+ * Says nothing about whether the responses were correct: a p95 over twenty fast 500s passes this,
+ * and should. Pair it with `every latency sample should have returned status 200`, which is the
+ * step that proves the server was doing the work being timed.
+ */
+export declare const assertP95WithinBudget: ({ scenario, config, }: {
+    scenario: ScenarioMeta;
+    config: BudgetConfig;
+}) => Promise<void>;
+/** PROVES the sampled p95 sits under a limit this scenario owns. Strictly `<`, as with the vitals. */
+export declare const assertP95UnderThreshold: ({ scenario }: {
+    scenario: ScenarioMeta;
+}, maxMs: number) => Promise<void>;
+/**
+ * PROVES that whatever changed between two samples did not make the endpoint slower — a warmed
+ * cache, a new index, a refusal being cheaper to serve than a response. Compares the last two
+ * sample sets in this scenario; fails outright with only one, because comparing a sample with
+ * itself always passes.
+ */
+export declare const assertP95NoWorseThanPrevious: ({ scenario }: {
+    scenario: ScenarioMeta;
+}, toleranceMs: number) => Promise<void>;
+/**
+ * PROVES the timed responses were the ones the scenario meant to time. Without it a latency budget
+ * is happily met by an endpoint that 500s in 3 ms, or that 401s because the sample lost the
+ * scenario's auth. Lists the offending statuses, because "not all 200" sends nobody anywhere.
+ */
+export declare const assertEverySampleStatus: ({ scenario }: {
+    scenario: ScenarioMeta;
+}, status: number) => Promise<void>;
+export {};
+//# sourceMappingURL=perf.steps.d.ts.map