@opice/harness 0.6.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/dsn.d.ts +8 -6
- package/dist/dsn.d.ts.map +1 -1
- package/dist/dsn.js +4 -3
- package/dist/dsn.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/reporter.d.ts +35 -3
- package/dist/reporter.d.ts.map +1 -1
- package/dist/reporter.js +110 -19
- package/dist/reporter.js.map +1 -1
- package/dist/scenario.d.ts +13 -0
- package/dist/scenario.d.ts.map +1 -1
- package/dist/scenario.js +82 -1
- package/dist/scenario.js.map +1 -1
- package/dist/tier.d.ts +44 -0
- package/dist/tier.d.ts.map +1 -0
- package/dist/tier.js +38 -0
- package/dist/tier.js.map +1 -0
- package/package.json +1 -1
- package/src/dsn.ts +12 -9
- package/src/index.ts +12 -1
- package/src/reporter.ts +149 -21
- package/src/scenario.ts +101 -1
- package/src/tier.ts +67 -0
package/src/reporter.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* The CLI handles end-of-run finalization: the reporter writes a
|
|
9
9
|
* handoff file under $TMPDIR with the runId and credentials, the
|
|
10
10
|
* `opice test` wrapper picks it up after `bun test` exits and POSTs
|
|
11
|
-
* /api/v1
|
|
11
|
+
* /api/v1/<slug>/runs/<id>/finish so the dashboard sees the run as completed.
|
|
12
12
|
*
|
|
13
13
|
* When env vars aren't configured, the reporter falls back to a no-op so
|
|
14
14
|
* harness behavior matches the bindx prototype.
|
|
@@ -19,6 +19,7 @@ import { mkdirSync, writeFileSync } from 'node:fs'
|
|
|
19
19
|
import { tmpdir } from 'node:os'
|
|
20
20
|
import path from 'node:path'
|
|
21
21
|
import { parseOpiceDsn } from './dsn.js'
|
|
22
|
+
import { resolveSelectedTier } from './tier.js'
|
|
22
23
|
|
|
23
24
|
/** Per-request cap, so a hung connection can't stall a scenario's afterAll. */
|
|
24
25
|
const REQUEST_TIMEOUT_MS = 10_000
|
|
@@ -28,11 +29,37 @@ const FLUSH_BUDGET_MS = 15_000
|
|
|
28
29
|
export interface ReporterConfig {
|
|
29
30
|
endpoint: string
|
|
30
31
|
projectId: string
|
|
31
|
-
|
|
32
|
+
/** Service-token credentials (the OPICE_DSN userinfo / OPICE_CLIENT_ID+SECRET). */
|
|
33
|
+
clientId: string
|
|
34
|
+
clientSecret: string
|
|
32
35
|
branch?: string
|
|
33
36
|
commit?: string
|
|
34
37
|
/** 'ci' for runs from automation, 'local' for opted-in dev runs. */
|
|
35
38
|
source?: 'ci' | 'local'
|
|
39
|
+
/**
|
|
40
|
+
* The tier this run SELECTED (from `OPICE_TIER`) — recorded on the run so the
|
|
41
|
+
* dashboard can explain why scenarios were skipped. Omitted when no tier
|
|
42
|
+
* filter was set (the run ran everything).
|
|
43
|
+
*/
|
|
44
|
+
tier?: string
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Strict reporting policy, resolved once from the env in {@link configureFromEnv}.
|
|
49
|
+
*
|
|
50
|
+
* Reporting is best-effort by design — a flaky uplink or a dashboard outage must
|
|
51
|
+
* never redden an otherwise-green test run. But that decoupling hides a real
|
|
52
|
+
* failure mode: a misconfigured token or an unreachable endpoint means the run
|
|
53
|
+
* is silently NOT recorded, while CI stays green. Strict mode (opt in via
|
|
54
|
+
* `OPICE_REPORT_STRICT` / `opice test --fail-on-report-error`) makes that loud —
|
|
55
|
+
* any reporting failure fails the run (the harness throws from a scenario's
|
|
56
|
+
* `afterAll`; the CLI escalates a failed `POST /finish` to a non-zero exit).
|
|
57
|
+
*/
|
|
58
|
+
let strictReporting = false
|
|
59
|
+
|
|
60
|
+
/** Whether strict reporting is active (see {@link strictReporting}). */
|
|
61
|
+
export function isStrictReporting(): boolean {
|
|
62
|
+
return strictReporting
|
|
36
63
|
}
|
|
37
64
|
|
|
38
65
|
export interface StepEvent {
|
|
@@ -86,6 +113,17 @@ export interface ScenarioStart {
|
|
|
86
113
|
seeds?: string[]
|
|
87
114
|
/** Identities / roles the scenario acts as. */
|
|
88
115
|
roles?: string[]
|
|
116
|
+
/** Declared tier (critical | standard | extended) — when it runs. */
|
|
117
|
+
tier?: string
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* A scenario the tier filter excluded from this run — registered for the record
|
|
122
|
+
* but never executed. Reported `skipped`, carrying a `reason` (which tier it
|
|
123
|
+
* declared vs the selected one) so the dashboard can explain the absence.
|
|
124
|
+
*/
|
|
125
|
+
export interface ScenarioSkip extends ScenarioStart {
|
|
126
|
+
reason?: string
|
|
89
127
|
}
|
|
90
128
|
|
|
91
129
|
export interface ScenarioFinish {
|
|
@@ -101,18 +139,30 @@ export interface ScenarioFinish {
|
|
|
101
139
|
|
|
102
140
|
export interface Reporter {
|
|
103
141
|
startScenario(input: ScenarioStart): Promise<string>
|
|
142
|
+
/** Record a scenario the tier filter skipped (created already-finished as `skipped`). */
|
|
143
|
+
skipScenario(input: ScenarioSkip): Promise<void>
|
|
104
144
|
recordStep(event: StepEvent): Promise<void>
|
|
105
145
|
finishScenario(input: ScenarioFinish): Promise<void>
|
|
106
146
|
flush(): Promise<void>
|
|
147
|
+
/**
|
|
148
|
+
* True if any report to the platform failed (network error or non-2xx). Used
|
|
149
|
+
* by the harness to fail the run under strict reporting — see
|
|
150
|
+
* {@link isStrictReporting}. Always false for the no-op reporter.
|
|
151
|
+
*/
|
|
152
|
+
hadFailures(): boolean
|
|
107
153
|
}
|
|
108
154
|
|
|
109
155
|
class NoopReporter implements Reporter {
|
|
110
156
|
async startScenario(input: ScenarioStart): Promise<string> {
|
|
111
157
|
return `noop-${input.name}-${Date.now()}`
|
|
112
158
|
}
|
|
159
|
+
async skipScenario(_input: ScenarioSkip): Promise<void> {}
|
|
113
160
|
async recordStep(_event: StepEvent): Promise<void> {}
|
|
114
161
|
async finishScenario(_input: ScenarioFinish): Promise<void> {}
|
|
115
162
|
async flush(): Promise<void> {}
|
|
163
|
+
hadFailures(): boolean {
|
|
164
|
+
return false
|
|
165
|
+
}
|
|
116
166
|
}
|
|
117
167
|
|
|
118
168
|
export const HANDOFF_DIR = path.join(tmpdir(), 'opice-handoffs')
|
|
@@ -123,7 +173,11 @@ function handoffPath(pid = process.pid): string {
|
|
|
123
173
|
|
|
124
174
|
export interface RunHandoff {
|
|
125
175
|
endpoint: string
|
|
126
|
-
|
|
176
|
+
/** Project slug — the CLI builds /api/v1/<project>/runs/<id>/finish from it. */
|
|
177
|
+
project: string
|
|
178
|
+
/** Service-token credentials so the CLI can POST /finish with the CF-Access-Client-* headers. */
|
|
179
|
+
clientId: string
|
|
180
|
+
clientSecret: string
|
|
127
181
|
runId: string
|
|
128
182
|
}
|
|
129
183
|
|
|
@@ -131,9 +185,15 @@ class HttpReporter implements Reporter {
|
|
|
131
185
|
private runIdPromise: Promise<string> | null = null
|
|
132
186
|
private readonly pending: Set<Promise<unknown>> = new Set()
|
|
133
187
|
private warnedUnreachable = false
|
|
188
|
+
/** Count of failed reports (network error or non-2xx). Drives strict mode. */
|
|
189
|
+
private failures = 0
|
|
134
190
|
|
|
135
191
|
constructor(private readonly config: ReporterConfig) {}
|
|
136
192
|
|
|
193
|
+
hadFailures(): boolean {
|
|
194
|
+
return this.failures > 0
|
|
195
|
+
}
|
|
196
|
+
|
|
137
197
|
private async ensureRun(): Promise<string> {
|
|
138
198
|
if (!this.runIdPromise) {
|
|
139
199
|
this.runIdPromise = this.startRun()
|
|
@@ -142,17 +202,24 @@ class HttpReporter implements Reporter {
|
|
|
142
202
|
}
|
|
143
203
|
|
|
144
204
|
private async startRun(): Promise<string> {
|
|
145
|
-
const response = await this.fetch('POST',
|
|
205
|
+
const response = await this.fetch('POST', `/api/v1/${this.config.projectId}/runs`, {
|
|
146
206
|
branch: this.config.branch,
|
|
147
207
|
commit: this.config.commit,
|
|
148
208
|
source: this.config.source,
|
|
209
|
+
tier: this.config.tier,
|
|
149
210
|
})
|
|
150
211
|
const runId = response['runId'] as string
|
|
151
212
|
// Synchronous write so the CLI can pick this up even if the test
|
|
152
213
|
// process exits abruptly (process.on('exit') runs sync).
|
|
153
214
|
try {
|
|
154
215
|
mkdirSync(HANDOFF_DIR, { recursive: true })
|
|
155
|
-
const handoff: RunHandoff = {
|
|
216
|
+
const handoff: RunHandoff = {
|
|
217
|
+
endpoint: this.config.endpoint,
|
|
218
|
+
project: this.config.projectId,
|
|
219
|
+
clientId: this.config.clientId,
|
|
220
|
+
clientSecret: this.config.clientSecret,
|
|
221
|
+
runId,
|
|
222
|
+
}
|
|
156
223
|
writeFileSync(handoffPath(), JSON.stringify(handoff), 'utf-8')
|
|
157
224
|
} catch {
|
|
158
225
|
// best-effort
|
|
@@ -162,17 +229,35 @@ class HttpReporter implements Reporter {
|
|
|
162
229
|
|
|
163
230
|
async startScenario(input: ScenarioStart): Promise<string> {
|
|
164
231
|
const runId = await this.ensureRun()
|
|
165
|
-
const response = await this.fetch('POST', `/api/v1/runs/${runId}/scenarios`, {
|
|
232
|
+
const response = await this.fetch('POST', `/api/v1/${this.config.projectId}/runs/${runId}/scenarios`, {
|
|
166
233
|
name: input.name,
|
|
167
234
|
hash: input.hash,
|
|
168
235
|
testFile: input.testFile,
|
|
169
236
|
feature: input.feature,
|
|
170
237
|
seeds: input.seeds,
|
|
171
238
|
roles: input.roles,
|
|
239
|
+
tier: input.tier,
|
|
172
240
|
})
|
|
173
241
|
return response['scenarioId'] as string
|
|
174
242
|
}
|
|
175
243
|
|
|
244
|
+
async skipScenario(input: ScenarioSkip): Promise<void> {
|
|
245
|
+
const runId = await this.ensureRun()
|
|
246
|
+
// A skipped scenario is created already-finished on the platform — no
|
|
247
|
+
// steps follow, so we don't keep the returned id.
|
|
248
|
+
await this.fetch('POST', `/api/v1/${this.config.projectId}/runs/${runId}/scenarios`, {
|
|
249
|
+
name: input.name,
|
|
250
|
+
hash: input.hash,
|
|
251
|
+
testFile: input.testFile,
|
|
252
|
+
feature: input.feature,
|
|
253
|
+
seeds: input.seeds,
|
|
254
|
+
roles: input.roles,
|
|
255
|
+
tier: input.tier,
|
|
256
|
+
skipped: true,
|
|
257
|
+
reason: input.reason,
|
|
258
|
+
})
|
|
259
|
+
}
|
|
260
|
+
|
|
176
261
|
recordStep(event: StepEvent): Promise<void> {
|
|
177
262
|
// Track synchronously so flush() awaits the entire pipeline (including
|
|
178
263
|
// encodeScreenshot's fs.readFile and the upload), not just whatever
|
|
@@ -187,7 +272,7 @@ class HttpReporter implements Reporter {
|
|
|
187
272
|
const screenshot = event.screenshotPath
|
|
188
273
|
? await this.encodeScreenshot(event.screenshotPath)
|
|
189
274
|
: undefined
|
|
190
|
-
await this.fetch('POST', `/api/v1/runs/${runId}/scenarios/${event.scenarioId}/steps`, {
|
|
275
|
+
await this.fetch('POST', `/api/v1/${this.config.projectId}/runs/${runId}/scenarios/${event.scenarioId}/steps`, {
|
|
191
276
|
attempt: event.attempt,
|
|
192
277
|
sequence: event.sequence,
|
|
193
278
|
kind: event.kind,
|
|
@@ -206,7 +291,7 @@ class HttpReporter implements Reporter {
|
|
|
206
291
|
const runId = await this.ensureRun()
|
|
207
292
|
// Awaited inline so the scenario status is committed before the
|
|
208
293
|
// bun:test afterAll returns.
|
|
209
|
-
await this.fetch('PATCH', `/api/v1/runs/${runId}/scenarios/${input.scenarioId}`, {
|
|
294
|
+
await this.fetch('PATCH', `/api/v1/${this.config.projectId}/runs/${runId}/scenarios/${input.scenarioId}`, {
|
|
210
295
|
status: input.status,
|
|
211
296
|
durationMs: input.durationMs,
|
|
212
297
|
attempts: input.attempts,
|
|
@@ -245,7 +330,9 @@ class HttpReporter implements Reporter {
|
|
|
245
330
|
response = await fetch(this.config.endpoint + path, {
|
|
246
331
|
method,
|
|
247
332
|
headers: {
|
|
248
|
-
|
|
333
|
+
// Cloudflare Access service-token pair — validated at the edge, never the origin.
|
|
334
|
+
'cf-access-client-id': this.config.clientId,
|
|
335
|
+
'cf-access-client-secret': this.config.clientSecret,
|
|
249
336
|
'content-type': 'application/json',
|
|
250
337
|
},
|
|
251
338
|
body: body == null ? undefined : JSON.stringify(body),
|
|
@@ -257,25 +344,34 @@ class HttpReporter implements Reporter {
|
|
|
257
344
|
// DOM and routes fetch through a same-origin policy). Callers swallow
|
|
258
345
|
// reporter errors so the test still runs, so this is the one place the
|
|
259
346
|
// failure is visible — make it loud and actionable.
|
|
260
|
-
this.
|
|
347
|
+
this.noteFailure(`${method} ${path}`, err instanceof Error ? err.message : String(err))
|
|
261
348
|
throw err
|
|
262
349
|
}
|
|
263
350
|
if (!response.ok) {
|
|
264
351
|
const detail = `${response.status} ${await response.text()}`.trim()
|
|
265
|
-
this.
|
|
352
|
+
this.noteFailure(`${method} ${path}`, detail)
|
|
266
353
|
throw new Error(`opice reporter ${method} ${path} failed: ${detail}`)
|
|
267
354
|
}
|
|
268
355
|
return (await response.json()) as Record<string, unknown>
|
|
269
356
|
}
|
|
270
357
|
|
|
271
358
|
/**
|
|
272
|
-
*
|
|
273
|
-
*
|
|
274
|
-
*
|
|
275
|
-
*
|
|
359
|
+
* Record a reporting failure and surface it. Callers swallow reporter errors
|
|
360
|
+
* so the test still runs (reporting is best-effort), which makes this the one
|
|
361
|
+
* place a failure is visible — so every failure is logged to stderr (a
|
|
362
|
+
* configured reporter that can't reach the platform means the run is silently
|
|
363
|
+
* NOT recorded, the most confusing failure mode in onboarding: the test
|
|
364
|
+
* passes but nothing shows on the dashboard). The first failure prints the
|
|
365
|
+
* full hint with the usual culprits; the rest a concise one-liner so a
|
|
366
|
+
* recurring failure is visible without flooding the log. Counts toward
|
|
367
|
+
* {@link hadFailures}, which strict mode fails the run on.
|
|
276
368
|
*/
|
|
277
|
-
private
|
|
278
|
-
|
|
369
|
+
private noteFailure(call: string, detail: string): void {
|
|
370
|
+
this.failures++
|
|
371
|
+
if (this.warnedUnreachable) {
|
|
372
|
+
console.error(`[opice] reporter error (${call}): ${detail} — this report was NOT recorded.`)
|
|
373
|
+
return
|
|
374
|
+
}
|
|
279
375
|
this.warnedUnreachable = true
|
|
280
376
|
console.error(
|
|
281
377
|
`[opice] reporter could not reach the platform (${call}: ${detail}). `
|
|
@@ -284,7 +380,8 @@ class HttpReporter implements Reporter {
|
|
|
284
380
|
+ `[opice] - the test runner's global setup installs a DOM (happy-dom/jsdom) or mocks\n`
|
|
285
381
|
+ `[opice] fetch, so the cross-origin POST is blocked (look for "Cross-Origin Request\n`
|
|
286
382
|
+ `[opice] Blocked" / an OPTIONS … 401). Scope that setup so it skips the e2e dir.\n`
|
|
287
|
-
+ `[opice] - a missing / expired OPICE_DSN api key (401), or an unreachable endpoint
|
|
383
|
+
+ `[opice] - a missing / expired OPICE_DSN api key (401), or an unreachable endpoint.\n`
|
|
384
|
+
+ `[opice] (set OPICE_REPORT_STRICT=1 / opice test --fail-on-report-error to fail the run on this.)`,
|
|
288
385
|
)
|
|
289
386
|
}
|
|
290
387
|
}
|
|
@@ -299,13 +396,39 @@ export function setReporter(reporter: Reporter): void {
|
|
|
299
396
|
active = reporter
|
|
300
397
|
}
|
|
301
398
|
|
|
399
|
+
function isTruthy(value: string | undefined): boolean {
|
|
400
|
+
if (!value) return false
|
|
401
|
+
const v = value.toLowerCase()
|
|
402
|
+
return v === '1' || v === 'true' || v === 'yes' || v === 'on'
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/**
|
|
406
|
+
* Strict reporting is requested but the reporter is a no-op — it can never fail,
|
|
407
|
+
* so strict mode has nothing to enforce. Warn rather than silently ignoring it:
|
|
408
|
+
* the user asked for "fail if reporting fails" and is instead getting no
|
|
409
|
+
* reporting at all, which strict can't catch.
|
|
410
|
+
*/
|
|
411
|
+
function warnStrictNoop(why: string): void {
|
|
412
|
+
console.error(
|
|
413
|
+
`[opice] OPICE_REPORT_STRICT is set but ${why} — strict reporting has no effect `
|
|
414
|
+
+ `(there is nothing to report, so nothing can fail).`,
|
|
415
|
+
)
|
|
416
|
+
}
|
|
417
|
+
|
|
302
418
|
export function configureFromEnv(env: NodeJS.ProcessEnv = process.env): Reporter {
|
|
419
|
+
// Strict reporting: fail the run if any report to the platform fails. Opt-in
|
|
420
|
+
// (default best-effort is locked design), resolved once here for the whole
|
|
421
|
+
// process. The CLI's `--fail-on-report-error` sets OPICE_REPORT_STRICT in the
|
|
422
|
+
// child env, so a bare `bun test` honours it too.
|
|
423
|
+
strictReporting = isTruthy(env['OPICE_REPORT_STRICT'])
|
|
303
424
|
// Individual vars win; OPICE_DSN fills any gaps (see dsn.ts).
|
|
304
425
|
const dsn = parseOpiceDsn(env['OPICE_DSN'])
|
|
305
426
|
const endpoint = env['OPICE_ENDPOINT'] ?? dsn?.endpoint
|
|
306
427
|
const projectId = env['OPICE_PROJECT'] ?? dsn?.project
|
|
307
|
-
const
|
|
308
|
-
|
|
428
|
+
const clientId = env['OPICE_CLIENT_ID'] ?? dsn?.clientId
|
|
429
|
+
const clientSecret = env['OPICE_CLIENT_SECRET'] ?? dsn?.clientSecret
|
|
430
|
+
if (!endpoint || !projectId || !clientId || !clientSecret) {
|
|
431
|
+
if (strictReporting) warnStrictNoop('reporter credentials are not configured (no OPICE_DSN / OPICE_* vars)')
|
|
309
432
|
return new NoopReporter()
|
|
310
433
|
}
|
|
311
434
|
// Reporting is opt-in outside CI. A local `bun test` while authoring would
|
|
@@ -317,15 +440,20 @@ export function configureFromEnv(env: NodeJS.ProcessEnv = process.env): Reporter
|
|
|
317
440
|
const mode = (env['OPICE_REPORT'] ?? 'auto').toLowerCase()
|
|
318
441
|
const shouldReport = mode === 'never' ? false : mode === 'always' ? true : isCI
|
|
319
442
|
if (!shouldReport) {
|
|
443
|
+
if (strictReporting) warnStrictNoop(`reporting is disabled here (OPICE_REPORT=${mode}, CI=${isCI})`)
|
|
320
444
|
return new NoopReporter()
|
|
321
445
|
}
|
|
322
446
|
const reporter = new HttpReporter({
|
|
323
447
|
endpoint,
|
|
324
448
|
projectId,
|
|
325
|
-
|
|
449
|
+
clientId,
|
|
450
|
+
clientSecret,
|
|
326
451
|
branch: env['OPICE_BRANCH'] ?? env['GITHUB_REF_NAME'],
|
|
327
452
|
commit: env['OPICE_COMMIT'] ?? env['GITHUB_SHA'],
|
|
328
453
|
source: isCI ? 'ci' : 'local',
|
|
454
|
+
// Record the selected tier only when one was explicitly requested — a run
|
|
455
|
+
// with no OPICE_TIER ran everything and carries no tier filter.
|
|
456
|
+
tier: env['OPICE_TIER'] ? resolveSelectedTier(env) : undefined,
|
|
329
457
|
})
|
|
330
458
|
setReporter(reporter)
|
|
331
459
|
return reporter
|
package/src/scenario.ts
CHANGED
|
@@ -2,8 +2,9 @@ import { createRequire } from 'node:module'
|
|
|
2
2
|
import path from 'node:path'
|
|
3
3
|
import { closePage, getContext, launchPage } from './context.js'
|
|
4
4
|
import { screenshot } from './element.js'
|
|
5
|
-
import { getReporter, type Reporter } from './reporter.js'
|
|
5
|
+
import { getReporter, isStrictReporting, type Reporter } from './reporter.js'
|
|
6
6
|
import { loadUserSetup } from './setup.js'
|
|
7
|
+
import { isTierSkipped, normalizeTier, parseSelectedTier, type Tier, TIER_ORDER } from './tier.js'
|
|
7
8
|
|
|
8
9
|
/**
|
|
9
10
|
* `bun:test` is resolved lazily, at the moment `browserTest` registers a
|
|
@@ -44,6 +45,18 @@ export interface BrowserTestMeta {
|
|
|
44
45
|
hash?: string
|
|
45
46
|
/** Feature / requirement id this scenario covers (e.g. `'F-SML-03a'`). */
|
|
46
47
|
feature?: string
|
|
48
|
+
/**
|
|
49
|
+
* Test tier — *when* this scenario runs (critical < standard < extended).
|
|
50
|
+
* A run selects a tier via `OPICE_TIER` / `opice test --tier`; selection is a
|
|
51
|
+
* threshold (running `standard` runs critical + standard). A scenario above
|
|
52
|
+
* the selected tier is **skipped**: reported as `skipped` (so it still shows
|
|
53
|
+
* on the dashboard) but never opens a browser. Defaults to `standard`.
|
|
54
|
+
*
|
|
55
|
+
* critical — the must-pass core, every push
|
|
56
|
+
* standard — the normal suite (default), PRs / merges
|
|
57
|
+
* extended — slow / edge / expensive, nightly or on demand
|
|
58
|
+
*/
|
|
59
|
+
tier?: Tier
|
|
47
60
|
/**
|
|
48
61
|
* Seeds that must be loaded for this scenario to run — machine-checkable
|
|
49
62
|
* preconditions, not prose. e.g. `['initial-data', 'crm-master-data']`.
|
|
@@ -107,6 +120,40 @@ function captureTestFile(): string | undefined {
|
|
|
107
120
|
return undefined
|
|
108
121
|
}
|
|
109
122
|
|
|
123
|
+
// The tier this run selected (OPICE_TIER), resolved + cached once. An
|
|
124
|
+
// unrecognized value warns once and falls back to running everything.
|
|
125
|
+
let cachedSelectedTier: Tier | undefined
|
|
126
|
+
function getSelectedTier(): Tier {
|
|
127
|
+
if (cachedSelectedTier === undefined) {
|
|
128
|
+
const parsed = parseSelectedTier()
|
|
129
|
+
if (!parsed.recognized) {
|
|
130
|
+
console.warn(
|
|
131
|
+
`[opice] unknown OPICE_TIER="${process.env['OPICE_TIER']}" — running all tiers `
|
|
132
|
+
+ `(use one of: ${TIER_ORDER.join(', ')}).`,
|
|
133
|
+
)
|
|
134
|
+
}
|
|
135
|
+
cachedSelectedTier = parsed.tier
|
|
136
|
+
}
|
|
137
|
+
return cachedSelectedTier
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Names of scenarios skipped by the tier filter, summarized once at process
|
|
141
|
+
// exit so a tiered run prints "N skipped" instead of a line per scenario.
|
|
142
|
+
const skippedScenarioNames: string[] = []
|
|
143
|
+
let skipSummaryHooked = false
|
|
144
|
+
function noteSkipped(name: string): void {
|
|
145
|
+
skippedScenarioNames.push(name)
|
|
146
|
+
if (skipSummaryHooked) return
|
|
147
|
+
skipSummaryHooked = true
|
|
148
|
+
process.on('exit', () => {
|
|
149
|
+
if (skippedScenarioNames.length === 0) return
|
|
150
|
+
console.warn(
|
|
151
|
+
`[opice] ${skippedScenarioNames.length} scenario(s) skipped — above the selected `
|
|
152
|
+
+ `tier '${getSelectedTier()}' (OPICE_TIER).`,
|
|
153
|
+
)
|
|
154
|
+
})
|
|
155
|
+
}
|
|
156
|
+
|
|
110
157
|
let currentScenarioId: string | null = null
|
|
111
158
|
let currentScenarioStart: number = 0
|
|
112
159
|
let currentScenarioFailures = 0
|
|
@@ -165,6 +212,14 @@ export function browserTest(meta: BrowserTestMeta, fn: () => void | Promise<void
|
|
|
165
212
|
// fn is the legacy registrar (it registers its own test()/hooks).
|
|
166
213
|
const isBody = fn.constructor.name === 'AsyncFunction'
|
|
167
214
|
|
|
215
|
+
// Tier gate: a scenario above the selected tier is registered but not run —
|
|
216
|
+
// reported `skipped` so the dashboard shows the full inventory.
|
|
217
|
+
const scenarioTier = normalizeTier(meta.tier)
|
|
218
|
+
if (isTierSkipped(scenarioTier, getSelectedTier())) {
|
|
219
|
+
registerSkipped(meta, scenarioTier, testFile, reporter)
|
|
220
|
+
return
|
|
221
|
+
}
|
|
222
|
+
|
|
168
223
|
describe(meta.name, () => {
|
|
169
224
|
beforeAll(async () => {
|
|
170
225
|
currentScenarioStart = Date.now()
|
|
@@ -180,6 +235,7 @@ export function browserTest(meta: BrowserTestMeta, fn: () => void | Promise<void
|
|
|
180
235
|
feature: meta.feature,
|
|
181
236
|
seeds: meta.seeds,
|
|
182
237
|
roles: meta.roles,
|
|
238
|
+
tier: scenarioTier,
|
|
183
239
|
})
|
|
184
240
|
} catch {
|
|
185
241
|
currentScenarioId = null
|
|
@@ -250,6 +306,18 @@ export function browserTest(meta: BrowserTestMeta, fn: () => void | Promise<void
|
|
|
250
306
|
}
|
|
251
307
|
}
|
|
252
308
|
currentScenarioId = null
|
|
309
|
+
// Strict reporting: a swallowed report failure (here or in any earlier
|
|
310
|
+
// hook/step) must turn the run red. afterAll always runs (beforeAll
|
|
311
|
+
// catches its own report failures), so throwing here is enough to make
|
|
312
|
+
// bun exit non-zero even when every assertion passed. The detail was
|
|
313
|
+
// already logged at the point of failure (reporter.noteFailure).
|
|
314
|
+
if (isStrictReporting() && reporter.hadFailures()) {
|
|
315
|
+
throw new Error(
|
|
316
|
+
`[opice] reporting to the platform failed and strict reporting is on `
|
|
317
|
+
+ `(OPICE_REPORT_STRICT / opice test --fail-on-report-error) — failing the run. `
|
|
318
|
+
+ `See the [opice] reporter error(s) above for the cause.`,
|
|
319
|
+
)
|
|
320
|
+
}
|
|
253
321
|
}, 30_000)
|
|
254
322
|
|
|
255
323
|
if (isBody) {
|
|
@@ -285,6 +353,38 @@ export function browserTest(meta: BrowserTestMeta, fn: () => void | Promise<void
|
|
|
285
353
|
})
|
|
286
354
|
}
|
|
287
355
|
|
|
356
|
+
/**
|
|
357
|
+
* Register a scenario the tier filter excluded. It's reported to the platform as
|
|
358
|
+
* `skipped` (so the dashboard shows it alongside what ran) but never opens a
|
|
359
|
+
* browser. The report runs in a real — instant, browser-free — `test`, not a
|
|
360
|
+
* `test.skip`: bun won't run a describe's hooks if every test in it is skipped,
|
|
361
|
+
* so a skip body is the only place left to POST from. With reporting off (local
|
|
362
|
+
* authoring), the body is a no-op against the NoopReporter.
|
|
363
|
+
*/
|
|
364
|
+
function registerSkipped(meta: BrowserTestMeta, tier: Tier, testFile: string | undefined, reporter: Reporter): void {
|
|
365
|
+
noteSkipped(meta.name)
|
|
366
|
+
const { describe, test } = bunTest()
|
|
367
|
+
const reason = `tier '${tier}' above the selected tier '${getSelectedTier()}'`
|
|
368
|
+
describe(meta.name, () => {
|
|
369
|
+
test('skipped (tier)', async () => {
|
|
370
|
+
try {
|
|
371
|
+
await reporter.skipScenario({
|
|
372
|
+
name: meta.name,
|
|
373
|
+
hash: meta.hash,
|
|
374
|
+
testFile,
|
|
375
|
+
feature: meta.feature,
|
|
376
|
+
seeds: meta.seeds,
|
|
377
|
+
roles: meta.roles,
|
|
378
|
+
tier,
|
|
379
|
+
reason,
|
|
380
|
+
})
|
|
381
|
+
} catch {
|
|
382
|
+
// Best-effort: reporting a skip must never fail the run.
|
|
383
|
+
}
|
|
384
|
+
})
|
|
385
|
+
})
|
|
386
|
+
}
|
|
387
|
+
|
|
288
388
|
/**
|
|
289
389
|
* Open a fresh isolated browser context + page for `meta` and navigate to its
|
|
290
390
|
* scenario URL. `launchPage()` closes any previous context first, so calling
|
package/src/tier.ts
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test tiers — an ordered hierarchy for *when* a scenario runs.
|
|
3
|
+
*
|
|
4
|
+
* A scenario declares its tier in `browserTest` meta (default `standard`); a run
|
|
5
|
+
* selects a tier via `OPICE_TIER` (the `opice test --tier` flag). Selection is a
|
|
6
|
+
* THRESHOLD: running a tier runs every scenario AT OR BELOW it — the standard
|
|
7
|
+
* `smoke ⊂ regression ⊂ full` model. Scenarios above the selected tier are
|
|
8
|
+
* *skipped*: still registered and reported as `skipped` (so the dashboard shows
|
|
9
|
+
* the full inventory, not just what ran), but they never open a browser.
|
|
10
|
+
*
|
|
11
|
+
* critical — the must-pass core. Run on every push.
|
|
12
|
+
* standard — the normal suite (the default for an untagged scenario). PRs / merges.
|
|
13
|
+
* extended — slow / edge / expensive. Run nightly or on demand.
|
|
14
|
+
*
|
|
15
|
+
* OPICE_TIER=critical → critical only
|
|
16
|
+
* OPICE_TIER=standard → critical + standard
|
|
17
|
+
* OPICE_TIER=extended → everything (also the default when OPICE_TIER is unset)
|
|
18
|
+
*/
|
|
19
|
+
export type Tier = 'critical' | 'standard' | 'extended'
|
|
20
|
+
|
|
21
|
+
/** Tiers low → high. A scenario's index is its level; it runs when level <= selected level. */
|
|
22
|
+
export const TIER_ORDER: readonly Tier[] = ['critical', 'standard', 'extended']
|
|
23
|
+
|
|
24
|
+
/** A scenario with no declared tier sits in the middle `standard` tier. */
|
|
25
|
+
export const DEFAULT_SCENARIO_TIER: Tier = 'standard'
|
|
26
|
+
|
|
27
|
+
/** With no `OPICE_TIER` set, run everything — select the widest tier. */
|
|
28
|
+
export const DEFAULT_SELECTED_TIER: Tier = 'extended'
|
|
29
|
+
|
|
30
|
+
function isTier(value: string | undefined): value is Tier {
|
|
31
|
+
return value === 'critical' || value === 'standard' || value === 'extended'
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Normalize a scenario's declared tier, defaulting (and tolerating junk) to `standard`. */
|
|
35
|
+
export function normalizeTier(tier: string | undefined): Tier {
|
|
36
|
+
return isTier(tier) ? tier : DEFAULT_SCENARIO_TIER
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface SelectedTier {
|
|
40
|
+
tier: Tier
|
|
41
|
+
/** false when `OPICE_TIER` held an unrecognized value (the caller may warn). */
|
|
42
|
+
recognized: boolean
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* The tier selected for this run, parsed from `OPICE_TIER`. Unset → run
|
|
47
|
+
* everything. `all`/`full` are friendly aliases for the widest tier. An
|
|
48
|
+
* unrecognized value resolves to "run everything" (`recognized: false`) rather
|
|
49
|
+
* than silently dropping tests — better to over-run than to skip on a typo.
|
|
50
|
+
*/
|
|
51
|
+
export function parseSelectedTier(env: NodeJS.ProcessEnv = process.env): SelectedTier {
|
|
52
|
+
const raw = env['OPICE_TIER']?.trim().toLowerCase()
|
|
53
|
+
if (!raw) return { tier: DEFAULT_SELECTED_TIER, recognized: true }
|
|
54
|
+
if (isTier(raw)) return { tier: raw, recognized: true }
|
|
55
|
+
if (raw === 'all' || raw === 'full') return { tier: 'extended', recognized: true }
|
|
56
|
+
return { tier: DEFAULT_SELECTED_TIER, recognized: false }
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Convenience: the resolved selected tier, ignoring whether the value was recognized. */
|
|
60
|
+
export function resolveSelectedTier(env: NodeJS.ProcessEnv = process.env): Tier {
|
|
61
|
+
return parseSelectedTier(env).tier
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** A scenario is skipped when its tier sits ABOVE the selected one. */
|
|
65
|
+
export function isTierSkipped(scenarioTier: Tier, selectedTier: Tier): boolean {
|
|
66
|
+
return TIER_ORDER.indexOf(scenarioTier) > TIER_ORDER.indexOf(selectedTier)
|
|
67
|
+
}
|