@opice/harness 0.6.1 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/reporter.ts CHANGED
@@ -8,7 +8,7 @@
8
8
  * The CLI handles end-of-run finalization: the reporter writes a
9
9
  * handoff file under $TMPDIR with the runId and credentials, the
10
10
  * `opice test` wrapper picks it up after `bun test` exits and POSTs
11
- * /api/v1/runs/<id>/finish so the dashboard sees the run as completed.
11
+ * /api/v1/<slug>/runs/<id>/finish so the dashboard sees the run as completed.
12
12
  *
13
13
  * When env vars aren't configured, the reporter falls back to a no-op so
14
14
  * harness behavior matches the bindx prototype.
@@ -19,6 +19,7 @@ import { mkdirSync, writeFileSync } from 'node:fs'
19
19
  import { tmpdir } from 'node:os'
20
20
  import path from 'node:path'
21
21
  import { parseOpiceDsn } from './dsn.js'
22
+ import { resolveSelectedTier } from './tier.js'
22
23
 
23
24
  /** Per-request cap, so a hung connection can't stall a scenario's afterAll. */
24
25
  const REQUEST_TIMEOUT_MS = 10_000
@@ -28,11 +29,37 @@ const FLUSH_BUDGET_MS = 15_000
28
29
  export interface ReporterConfig {
29
30
  endpoint: string
30
31
  projectId: string
31
- apiKey: string
32
+ /** Service-token credentials (the OPICE_DSN userinfo / OPICE_CLIENT_ID+SECRET). */
33
+ clientId: string
34
+ clientSecret: string
32
35
  branch?: string
33
36
  commit?: string
34
37
  /** 'ci' for runs from automation, 'local' for opted-in dev runs. */
35
38
  source?: 'ci' | 'local'
39
+ /**
40
+ * The tier this run SELECTED (from `OPICE_TIER`) — recorded on the run so the
41
+ * dashboard can explain why scenarios were skipped. Omitted when no tier
42
+ * filter was set (the run ran everything).
43
+ */
44
+ tier?: string
45
+ }
46
+
47
+ /**
48
+ * Strict reporting policy, resolved once from the env in {@link configureFromEnv}.
49
+ *
50
+ * Reporting is best-effort by design — a flaky uplink or a dashboard outage must
51
+ * never redden an otherwise-green test run. But that decoupling hides a real
52
+ * failure mode: a misconfigured token or an unreachable endpoint means the run
53
+ * is silently NOT recorded, while CI stays green. Strict mode (opt in via
54
+ * `OPICE_REPORT_STRICT` / `opice test --fail-on-report-error`) makes that loud —
55
+ * any reporting failure fails the run (the harness throws from a scenario's
56
+ * `afterAll`; the CLI escalates a failed `POST /finish` to a non-zero exit).
57
+ */
58
+ let strictReporting = false
59
+
60
+ /** Whether strict reporting is active (see {@link strictReporting}). */
61
+ export function isStrictReporting(): boolean {
62
+ return strictReporting
36
63
  }
37
64
 
38
65
  export interface StepEvent {
@@ -86,6 +113,17 @@ export interface ScenarioStart {
86
113
  seeds?: string[]
87
114
  /** Identities / roles the scenario acts as. */
88
115
  roles?: string[]
116
+ /** Declared tier (critical | standard | extended) — when it runs. */
117
+ tier?: string
118
+ }
119
+
120
+ /**
121
+ * A scenario the tier filter excluded from this run — registered for the record
122
+ * but never executed. Reported `skipped`, carrying a `reason` (which tier it
123
+ * declared vs the selected one) so the dashboard can explain the absence.
124
+ */
125
+ export interface ScenarioSkip extends ScenarioStart {
126
+ reason?: string
89
127
  }
90
128
 
91
129
  export interface ScenarioFinish {
@@ -101,18 +139,30 @@ export interface ScenarioFinish {
101
139
 
102
140
  export interface Reporter {
103
141
  startScenario(input: ScenarioStart): Promise<string>
142
+ /** Record a scenario the tier filter skipped (created already-finished as `skipped`). */
143
+ skipScenario(input: ScenarioSkip): Promise<void>
104
144
  recordStep(event: StepEvent): Promise<void>
105
145
  finishScenario(input: ScenarioFinish): Promise<void>
106
146
  flush(): Promise<void>
147
+ /**
148
+ * True if any report to the platform failed (network error or non-2xx). Used
149
+ * by the harness to fail the run under strict reporting — see
150
+ * {@link isStrictReporting}. Always false for the no-op reporter.
151
+ */
152
+ hadFailures(): boolean
107
153
  }
108
154
 
109
155
  class NoopReporter implements Reporter {
110
156
  async startScenario(input: ScenarioStart): Promise<string> {
111
157
  return `noop-${input.name}-${Date.now()}`
112
158
  }
159
+ async skipScenario(_input: ScenarioSkip): Promise<void> {}
113
160
  async recordStep(_event: StepEvent): Promise<void> {}
114
161
  async finishScenario(_input: ScenarioFinish): Promise<void> {}
115
162
  async flush(): Promise<void> {}
163
+ hadFailures(): boolean {
164
+ return false
165
+ }
116
166
  }
117
167
 
118
168
  export const HANDOFF_DIR = path.join(tmpdir(), 'opice-handoffs')
@@ -123,7 +173,11 @@ function handoffPath(pid = process.pid): string {
123
173
 
124
174
  export interface RunHandoff {
125
175
  endpoint: string
126
- apiKey: string
176
+ /** Project slug — the CLI builds /api/v1/<project>/runs/<id>/finish from it. */
177
+ project: string
178
+ /** Service-token credentials so the CLI can POST /finish with the CF-Access-Client-* headers. */
179
+ clientId: string
180
+ clientSecret: string
127
181
  runId: string
128
182
  }
129
183
 
@@ -131,9 +185,15 @@ class HttpReporter implements Reporter {
131
185
  private runIdPromise: Promise<string> | null = null
132
186
  private readonly pending: Set<Promise<unknown>> = new Set()
133
187
  private warnedUnreachable = false
188
+ /** Count of failed reports (network error or non-2xx). Drives strict mode. */
189
+ private failures = 0
134
190
 
135
191
  constructor(private readonly config: ReporterConfig) {}
136
192
 
193
+ hadFailures(): boolean {
194
+ return this.failures > 0
195
+ }
196
+
137
197
  private async ensureRun(): Promise<string> {
138
198
  if (!this.runIdPromise) {
139
199
  this.runIdPromise = this.startRun()
@@ -142,17 +202,24 @@ class HttpReporter implements Reporter {
142
202
  }
143
203
 
144
204
  private async startRun(): Promise<string> {
145
- const response = await this.fetch('POST', '/api/v1/runs', {
205
+ const response = await this.fetch('POST', `/api/v1/${this.config.projectId}/runs`, {
146
206
  branch: this.config.branch,
147
207
  commit: this.config.commit,
148
208
  source: this.config.source,
209
+ tier: this.config.tier,
149
210
  })
150
211
  const runId = response['runId'] as string
151
212
  // Synchronous write so the CLI can pick this up even if the test
152
213
  // process exits abruptly (process.on('exit') runs sync).
153
214
  try {
154
215
  mkdirSync(HANDOFF_DIR, { recursive: true })
155
- const handoff: RunHandoff = { endpoint: this.config.endpoint, apiKey: this.config.apiKey, runId }
216
+ const handoff: RunHandoff = {
217
+ endpoint: this.config.endpoint,
218
+ project: this.config.projectId,
219
+ clientId: this.config.clientId,
220
+ clientSecret: this.config.clientSecret,
221
+ runId,
222
+ }
156
223
  writeFileSync(handoffPath(), JSON.stringify(handoff), 'utf-8')
157
224
  } catch {
158
225
  // best-effort
@@ -162,17 +229,35 @@ class HttpReporter implements Reporter {
162
229
 
163
230
  async startScenario(input: ScenarioStart): Promise<string> {
164
231
  const runId = await this.ensureRun()
165
- const response = await this.fetch('POST', `/api/v1/runs/${runId}/scenarios`, {
232
+ const response = await this.fetch('POST', `/api/v1/${this.config.projectId}/runs/${runId}/scenarios`, {
166
233
  name: input.name,
167
234
  hash: input.hash,
168
235
  testFile: input.testFile,
169
236
  feature: input.feature,
170
237
  seeds: input.seeds,
171
238
  roles: input.roles,
239
+ tier: input.tier,
172
240
  })
173
241
  return response['scenarioId'] as string
174
242
  }
175
243
 
244
+ async skipScenario(input: ScenarioSkip): Promise<void> {
245
+ const runId = await this.ensureRun()
246
+ // A skipped scenario is created already-finished on the platform — no
247
+ // steps follow, so we don't keep the returned id.
248
+ await this.fetch('POST', `/api/v1/${this.config.projectId}/runs/${runId}/scenarios`, {
249
+ name: input.name,
250
+ hash: input.hash,
251
+ testFile: input.testFile,
252
+ feature: input.feature,
253
+ seeds: input.seeds,
254
+ roles: input.roles,
255
+ tier: input.tier,
256
+ skipped: true,
257
+ reason: input.reason,
258
+ })
259
+ }
260
+
176
261
  recordStep(event: StepEvent): Promise<void> {
177
262
  // Track synchronously so flush() awaits the entire pipeline (including
178
263
  // encodeScreenshot's fs.readFile and the upload), not just whatever
@@ -187,7 +272,7 @@ class HttpReporter implements Reporter {
187
272
  const screenshot = event.screenshotPath
188
273
  ? await this.encodeScreenshot(event.screenshotPath)
189
274
  : undefined
190
- await this.fetch('POST', `/api/v1/runs/${runId}/scenarios/${event.scenarioId}/steps`, {
275
+ await this.fetch('POST', `/api/v1/${this.config.projectId}/runs/${runId}/scenarios/${event.scenarioId}/steps`, {
191
276
  attempt: event.attempt,
192
277
  sequence: event.sequence,
193
278
  kind: event.kind,
@@ -206,7 +291,7 @@ class HttpReporter implements Reporter {
206
291
  const runId = await this.ensureRun()
207
292
  // Awaited inline so the scenario status is committed before the
208
293
  // bun:test afterAll returns.
209
- await this.fetch('PATCH', `/api/v1/runs/${runId}/scenarios/${input.scenarioId}`, {
294
+ await this.fetch('PATCH', `/api/v1/${this.config.projectId}/runs/${runId}/scenarios/${input.scenarioId}`, {
210
295
  status: input.status,
211
296
  durationMs: input.durationMs,
212
297
  attempts: input.attempts,
@@ -245,7 +330,9 @@ class HttpReporter implements Reporter {
245
330
  response = await fetch(this.config.endpoint + path, {
246
331
  method,
247
332
  headers: {
248
- 'authorization': `Bearer ${this.config.apiKey}`,
333
+ // Cloudflare Access service-token pair — validated at the edge, never the origin.
334
+ 'cf-access-client-id': this.config.clientId,
335
+ 'cf-access-client-secret': this.config.clientSecret,
249
336
  'content-type': 'application/json',
250
337
  },
251
338
  body: body == null ? undefined : JSON.stringify(body),
@@ -257,25 +344,34 @@ class HttpReporter implements Reporter {
257
344
  // DOM and routes fetch through a same-origin policy). Callers swallow
258
345
  // reporter errors so the test still runs, so this is the one place the
259
346
  // failure is visible — make it loud and actionable.
260
- this.warnUnreachable(`${method} ${path}`, err instanceof Error ? err.message : String(err))
347
+ this.noteFailure(`${method} ${path}`, err instanceof Error ? err.message : String(err))
261
348
  throw err
262
349
  }
263
350
  if (!response.ok) {
264
351
  const detail = `${response.status} ${await response.text()}`.trim()
265
- this.warnUnreachable(`${method} ${path}`, detail)
352
+ this.noteFailure(`${method} ${path}`, detail)
266
353
  throw new Error(`opice reporter ${method} ${path} failed: ${detail}`)
267
354
  }
268
355
  return (await response.json()) as Record<string, unknown>
269
356
  }
270
357
 
271
358
  /**
272
- * A configured reporter that can't reach the platform means the run is
273
- * silently NOT recorded the most confusing failure mode in onboarding
274
- * (the test passes, but nothing shows on the dashboard). Surface it once,
275
- * with the usual culprits, instead of letting the swallowed throw vanish.
359
+ * Record a reporting failure and surface it. Callers swallow reporter errors
360
+ * so the test still runs (reporting is best-effort), which makes this the one
361
+ * place a failure is visible so every failure is logged to stderr (a
362
+ * configured reporter that can't reach the platform means the run is silently
363
+ * NOT recorded, the most confusing failure mode in onboarding: the test
364
+ * passes but nothing shows on the dashboard). The first failure prints the
365
+ * full hint with the usual culprits; the rest a concise one-liner so a
366
+ * recurring failure is visible without flooding the log. Counts toward
367
+ * {@link hadFailures}, which strict mode fails the run on.
276
368
  */
277
- private warnUnreachable(call: string, detail: string): void {
278
- if (this.warnedUnreachable) return
369
+ private noteFailure(call: string, detail: string): void {
370
+ this.failures++
371
+ if (this.warnedUnreachable) {
372
+ console.error(`[opice] reporter error (${call}): ${detail} — this report was NOT recorded.`)
373
+ return
374
+ }
279
375
  this.warnedUnreachable = true
280
376
  console.error(
281
377
  `[opice] reporter could not reach the platform (${call}: ${detail}). `
@@ -284,7 +380,8 @@ class HttpReporter implements Reporter {
284
380
  + `[opice] - the test runner's global setup installs a DOM (happy-dom/jsdom) or mocks\n`
285
381
  + `[opice] fetch, so the cross-origin POST is blocked (look for "Cross-Origin Request\n`
286
382
  + `[opice] Blocked" / an OPTIONS … 401). Scope that setup so it skips the e2e dir.\n`
287
- + `[opice] - a missing / expired OPICE_DSN api key (401), or an unreachable endpoint.`,
383
+ + `[opice] - a missing / expired OPICE_DSN api key (401), or an unreachable endpoint.\n`
384
+ + `[opice] (set OPICE_REPORT_STRICT=1 / opice test --fail-on-report-error to fail the run on this.)`,
288
385
  )
289
386
  }
290
387
  }
@@ -299,13 +396,39 @@ export function setReporter(reporter: Reporter): void {
299
396
  active = reporter
300
397
  }
301
398
 
399
+ function isTruthy(value: string | undefined): boolean {
400
+ if (!value) return false
401
+ const v = value.toLowerCase()
402
+ return v === '1' || v === 'true' || v === 'yes' || v === 'on'
403
+ }
404
+
405
+ /**
406
+ * Strict reporting is requested but the reporter is a no-op — it can never fail,
407
+ * so strict mode has nothing to enforce. Warn rather than silently ignoring it:
408
+ * the user asked for "fail if reporting fails" and is instead getting no
409
+ * reporting at all, which strict can't catch.
410
+ */
411
+ function warnStrictNoop(why: string): void {
412
+ console.error(
413
+ `[opice] OPICE_REPORT_STRICT is set but ${why} — strict reporting has no effect `
414
+ + `(there is nothing to report, so nothing can fail).`,
415
+ )
416
+ }
417
+
302
418
  export function configureFromEnv(env: NodeJS.ProcessEnv = process.env): Reporter {
419
+ // Strict reporting: fail the run if any report to the platform fails. Opt-in
420
+ // (default best-effort is locked design), resolved once here for the whole
421
+ // process. The CLI's `--fail-on-report-error` sets OPICE_REPORT_STRICT in the
422
+ // child env, so a bare `bun test` honours it too.
423
+ strictReporting = isTruthy(env['OPICE_REPORT_STRICT'])
303
424
  // Individual vars win; OPICE_DSN fills any gaps (see dsn.ts).
304
425
  const dsn = parseOpiceDsn(env['OPICE_DSN'])
305
426
  const endpoint = env['OPICE_ENDPOINT'] ?? dsn?.endpoint
306
427
  const projectId = env['OPICE_PROJECT'] ?? dsn?.project
307
- const apiKey = env['OPICE_API_KEY'] ?? dsn?.apiKey
308
- if (!endpoint || !projectId || !apiKey) {
428
+ const clientId = env['OPICE_CLIENT_ID'] ?? dsn?.clientId
429
+ const clientSecret = env['OPICE_CLIENT_SECRET'] ?? dsn?.clientSecret
430
+ if (!endpoint || !projectId || !clientId || !clientSecret) {
431
+ if (strictReporting) warnStrictNoop('reporter credentials are not configured (no OPICE_DSN / OPICE_* vars)')
309
432
  return new NoopReporter()
310
433
  }
311
434
  // Reporting is opt-in outside CI. A local `bun test` while authoring would
@@ -317,15 +440,20 @@ export function configureFromEnv(env: NodeJS.ProcessEnv = process.env): Reporter
317
440
  const mode = (env['OPICE_REPORT'] ?? 'auto').toLowerCase()
318
441
  const shouldReport = mode === 'never' ? false : mode === 'always' ? true : isCI
319
442
  if (!shouldReport) {
443
+ if (strictReporting) warnStrictNoop(`reporting is disabled here (OPICE_REPORT=${mode}, CI=${isCI})`)
320
444
  return new NoopReporter()
321
445
  }
322
446
  const reporter = new HttpReporter({
323
447
  endpoint,
324
448
  projectId,
325
- apiKey,
449
+ clientId,
450
+ clientSecret,
326
451
  branch: env['OPICE_BRANCH'] ?? env['GITHUB_REF_NAME'],
327
452
  commit: env['OPICE_COMMIT'] ?? env['GITHUB_SHA'],
328
453
  source: isCI ? 'ci' : 'local',
454
+ // Record the selected tier only when one was explicitly requested — a run
455
+ // with no OPICE_TIER ran everything and carries no tier filter.
456
+ tier: env['OPICE_TIER'] ? resolveSelectedTier(env) : undefined,
329
457
  })
330
458
  setReporter(reporter)
331
459
  return reporter
package/src/scenario.ts CHANGED
@@ -2,8 +2,9 @@ import { createRequire } from 'node:module'
2
2
  import path from 'node:path'
3
3
  import { closePage, getContext, launchPage } from './context.js'
4
4
  import { screenshot } from './element.js'
5
- import { getReporter, type Reporter } from './reporter.js'
5
+ import { getReporter, isStrictReporting, type Reporter } from './reporter.js'
6
6
  import { loadUserSetup } from './setup.js'
7
+ import { isTierSkipped, normalizeTier, parseSelectedTier, type Tier, TIER_ORDER } from './tier.js'
7
8
 
8
9
  /**
9
10
  * `bun:test` is resolved lazily, at the moment `browserTest` registers a
@@ -44,6 +45,18 @@ export interface BrowserTestMeta {
44
45
  hash?: string
45
46
  /** Feature / requirement id this scenario covers (e.g. `'F-SML-03a'`). */
46
47
  feature?: string
48
+ /**
49
+ * Test tier — *when* this scenario runs (critical < standard < extended).
50
+ * A run selects a tier via `OPICE_TIER` / `opice test --tier`; selection is a
51
+ * threshold (running `standard` runs critical + standard). A scenario above
52
+ * the selected tier is **skipped**: reported as `skipped` (so it still shows
53
+ * on the dashboard) but never opens a browser. Defaults to `standard`.
54
+ *
55
+ * critical — the must-pass core, every push
56
+ * standard — the normal suite (default), PRs / merges
57
+ * extended — slow / edge / expensive, nightly or on demand
58
+ */
59
+ tier?: Tier
47
60
  /**
48
61
  * Seeds that must be loaded for this scenario to run — machine-checkable
49
62
  * preconditions, not prose. e.g. `['initial-data', 'crm-master-data']`.
@@ -107,6 +120,40 @@ function captureTestFile(): string | undefined {
107
120
  return undefined
108
121
  }
109
122
 
123
+ // The tier this run selected (OPICE_TIER), resolved + cached once. An
124
+ // unrecognized value warns once and falls back to running everything.
125
+ let cachedSelectedTier: Tier | undefined
126
+ function getSelectedTier(): Tier {
127
+ if (cachedSelectedTier === undefined) {
128
+ const parsed = parseSelectedTier()
129
+ if (!parsed.recognized) {
130
+ console.warn(
131
+ `[opice] unknown OPICE_TIER="${process.env['OPICE_TIER']}" — running all tiers `
132
+ + `(use one of: ${TIER_ORDER.join(', ')}).`,
133
+ )
134
+ }
135
+ cachedSelectedTier = parsed.tier
136
+ }
137
+ return cachedSelectedTier
138
+ }
139
+
140
+ // Names of scenarios skipped by the tier filter, summarized once at process
141
+ // exit so a tiered run prints "N skipped" instead of a line per scenario.
142
+ const skippedScenarioNames: string[] = []
143
+ let skipSummaryHooked = false
144
+ function noteSkipped(name: string): void {
145
+ skippedScenarioNames.push(name)
146
+ if (skipSummaryHooked) return
147
+ skipSummaryHooked = true
148
+ process.on('exit', () => {
149
+ if (skippedScenarioNames.length === 0) return
150
+ console.warn(
151
+ `[opice] ${skippedScenarioNames.length} scenario(s) skipped — above the selected `
152
+ + `tier '${getSelectedTier()}' (OPICE_TIER).`,
153
+ )
154
+ })
155
+ }
156
+
110
157
  let currentScenarioId: string | null = null
111
158
  let currentScenarioStart: number = 0
112
159
  let currentScenarioFailures = 0
@@ -165,6 +212,14 @@ export function browserTest(meta: BrowserTestMeta, fn: () => void | Promise<void
165
212
  // fn is the legacy registrar (it registers its own test()/hooks).
166
213
  const isBody = fn.constructor.name === 'AsyncFunction'
167
214
 
215
+ // Tier gate: a scenario above the selected tier is registered but not run —
216
+ // reported `skipped` so the dashboard shows the full inventory.
217
+ const scenarioTier = normalizeTier(meta.tier)
218
+ if (isTierSkipped(scenarioTier, getSelectedTier())) {
219
+ registerSkipped(meta, scenarioTier, testFile, reporter)
220
+ return
221
+ }
222
+
168
223
  describe(meta.name, () => {
169
224
  beforeAll(async () => {
170
225
  currentScenarioStart = Date.now()
@@ -180,6 +235,7 @@ export function browserTest(meta: BrowserTestMeta, fn: () => void | Promise<void
180
235
  feature: meta.feature,
181
236
  seeds: meta.seeds,
182
237
  roles: meta.roles,
238
+ tier: scenarioTier,
183
239
  })
184
240
  } catch {
185
241
  currentScenarioId = null
@@ -250,6 +306,18 @@ export function browserTest(meta: BrowserTestMeta, fn: () => void | Promise<void
250
306
  }
251
307
  }
252
308
  currentScenarioId = null
309
+ // Strict reporting: a swallowed report failure (here or in any earlier
310
+ // hook/step) must turn the run red. afterAll always runs (beforeAll
311
+ // catches its own report failures), so throwing here is enough to make
312
+ // bun exit non-zero even when every assertion passed. The detail was
313
+ // already logged at the point of failure (reporter.noteFailure).
314
+ if (isStrictReporting() && reporter.hadFailures()) {
315
+ throw new Error(
316
+ `[opice] reporting to the platform failed and strict reporting is on `
317
+ + `(OPICE_REPORT_STRICT / opice test --fail-on-report-error) — failing the run. `
318
+ + `See the [opice] reporter error(s) above for the cause.`,
319
+ )
320
+ }
253
321
  }, 30_000)
254
322
 
255
323
  if (isBody) {
@@ -285,6 +353,38 @@ export function browserTest(meta: BrowserTestMeta, fn: () => void | Promise<void
285
353
  })
286
354
  }
287
355
 
356
+ /**
357
+ * Register a scenario the tier filter excluded. It's reported to the platform as
358
+ * `skipped` (so the dashboard shows it alongside what ran) but never opens a
359
+ * browser. The report runs in a real — instant, browser-free — `test`, not a
360
+ * `test.skip`: bun won't run a describe's hooks if every test in it is skipped,
361
+ * so a skip body is the only place left to POST from. With reporting off (local
362
+ * authoring), the body is a no-op against the NoopReporter.
363
+ */
364
+ function registerSkipped(meta: BrowserTestMeta, tier: Tier, testFile: string | undefined, reporter: Reporter): void {
365
+ noteSkipped(meta.name)
366
+ const { describe, test } = bunTest()
367
+ const reason = `tier '${tier}' above the selected tier '${getSelectedTier()}'`
368
+ describe(meta.name, () => {
369
+ test('skipped (tier)', async () => {
370
+ try {
371
+ await reporter.skipScenario({
372
+ name: meta.name,
373
+ hash: meta.hash,
374
+ testFile,
375
+ feature: meta.feature,
376
+ seeds: meta.seeds,
377
+ roles: meta.roles,
378
+ tier,
379
+ reason,
380
+ })
381
+ } catch {
382
+ // Best-effort: reporting a skip must never fail the run.
383
+ }
384
+ })
385
+ })
386
+ }
387
+
288
388
  /**
289
389
  * Open a fresh isolated browser context + page for `meta` and navigate to its
290
390
  * scenario URL. `launchPage()` closes any previous context first, so calling
package/src/tier.ts ADDED
@@ -0,0 +1,67 @@
1
+ /**
2
+ * Test tiers — an ordered hierarchy for *when* a scenario runs.
3
+ *
4
+ * A scenario declares its tier in `browserTest` meta (default `standard`); a run
5
+ * selects a tier via `OPICE_TIER` (the `opice test --tier` flag). Selection is a
6
+ * THRESHOLD: running a tier runs every scenario AT OR BELOW it — the standard
7
+ * `smoke ⊂ regression ⊂ full` model. Scenarios above the selected tier are
8
+ * *skipped*: still registered and reported as `skipped` (so the dashboard shows
9
+ * the full inventory, not just what ran), but they never open a browser.
10
+ *
11
+ * critical — the must-pass core. Run on every push.
12
+ * standard — the normal suite (the default for an untagged scenario). PRs / merges.
13
+ * extended — slow / edge / expensive. Run nightly or on demand.
14
+ *
15
+ * OPICE_TIER=critical → critical only
16
+ * OPICE_TIER=standard → critical + standard
17
+ * OPICE_TIER=extended → everything (also the default when OPICE_TIER is unset)
18
+ */
19
+ export type Tier = 'critical' | 'standard' | 'extended'
20
+
21
+ /** Tiers low → high. A scenario's index is its level; it runs when level <= selected level. */
22
+ export const TIER_ORDER: readonly Tier[] = ['critical', 'standard', 'extended']
23
+
24
+ /** A scenario with no declared tier sits in the middle `standard` tier. */
25
+ export const DEFAULT_SCENARIO_TIER: Tier = 'standard'
26
+
27
+ /** With no `OPICE_TIER` set, run everything — select the widest tier. */
28
+ export const DEFAULT_SELECTED_TIER: Tier = 'extended'
29
+
30
+ function isTier(value: string | undefined): value is Tier {
31
+ return value === 'critical' || value === 'standard' || value === 'extended'
32
+ }
33
+
34
+ /** Normalize a scenario's declared tier, defaulting (and tolerating junk) to `standard`. */
35
+ export function normalizeTier(tier: string | undefined): Tier {
36
+ return isTier(tier) ? tier : DEFAULT_SCENARIO_TIER
37
+ }
38
+
39
+ export interface SelectedTier {
40
+ tier: Tier
41
+ /** false when `OPICE_TIER` held an unrecognized value (the caller may warn). */
42
+ recognized: boolean
43
+ }
44
+
45
+ /**
46
+ * The tier selected for this run, parsed from `OPICE_TIER`. Unset → run
47
+ * everything. `all`/`full` are friendly aliases for the widest tier. An
48
+ * unrecognized value resolves to "run everything" (`recognized: false`) rather
49
+ * than silently dropping tests — better to over-run than to skip on a typo.
50
+ */
51
+ export function parseSelectedTier(env: NodeJS.ProcessEnv = process.env): SelectedTier {
52
+ const raw = env['OPICE_TIER']?.trim().toLowerCase()
53
+ if (!raw) return { tier: DEFAULT_SELECTED_TIER, recognized: true }
54
+ if (isTier(raw)) return { tier: raw, recognized: true }
55
+ if (raw === 'all' || raw === 'full') return { tier: 'extended', recognized: true }
56
+ return { tier: DEFAULT_SELECTED_TIER, recognized: false }
57
+ }
58
+
59
+ /** Convenience: the resolved selected tier, ignoring whether the value was recognized. */
60
+ export function resolveSelectedTier(env: NodeJS.ProcessEnv = process.env): Tier {
61
+ return parseSelectedTier(env).tier
62
+ }
63
+
64
+ /** A scenario is skipped when its tier sits ABOVE the selected one. */
65
+ export function isTierSkipped(scenarioTier: Tier, selectedTier: Tier): boolean {
66
+ return TIER_ORDER.indexOf(scenarioTier) > TIER_ORDER.indexOf(selectedTier)
67
+ }