autokap 1.9.2 → 1.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-runner.js +90 -3
- package/dist/execution-types.d.ts +3 -2
- package/dist/opcode-runner.js +84 -9
- package/dist/postcondition.js +10 -3
- package/package.json +1 -1
package/dist/cli-runner.js
CHANGED
|
@@ -217,12 +217,51 @@ export async function runCapture(options) {
|
|
|
217
217
|
// badge on the preset while it captures. Best-effort + local-only: skip dry
|
|
218
218
|
// runs and cloud runs (AUTOKAP_RUN_ID is set on cloud runners, which own
|
|
219
219
|
// their own capture_runs row). A failure here must never block the capture.
|
|
220
|
-
|
|
220
|
+
const isLocalRegisteredRun = !options.dryRun && !process.env.AUTOKAP_RUN_ID;
|
|
221
|
+
if (isLocalRegisteredRun) {
|
|
221
222
|
await postRunStart(config, runId, program.presetId, program.variants.length, options.env);
|
|
222
223
|
}
|
|
224
|
+
// Ctrl-C handling: a local run that registered itself server-side must never
|
|
225
|
+
// leave the preset stuck on the "En cours" badge. On SIGINT/SIGTERM we abort
|
|
226
|
+
// the in-flight capture (so the finally-block cleanup runs) and best-effort
|
|
227
|
+
// tell the server the run was interrupted — otherwise the capture_runs row
|
|
228
|
+
// sits in `running` and the preset stays "capturing" until the 15-min stale
|
|
229
|
+
// cutoff. A caller-supplied abortSignal is forwarded into the same controller
|
|
230
|
+
// so both paths converge on one cancellation.
|
|
231
|
+
const abortController = new AbortController();
|
|
232
|
+
if (options.abortSignal) {
|
|
233
|
+
if (options.abortSignal.aborted) {
|
|
234
|
+
abortController.abort(options.abortSignal.reason);
|
|
235
|
+
}
|
|
236
|
+
else {
|
|
237
|
+
options.abortSignal.addEventListener('abort', () => abortController.abort(options.abortSignal.reason), { once: true });
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
let interrupted = false;
|
|
241
|
+
let abortNotified = false;
|
|
242
|
+
const notifyAborted = async () => {
|
|
243
|
+
if (abortNotified || !isLocalRegisteredRun)
|
|
244
|
+
return;
|
|
245
|
+
abortNotified = true;
|
|
246
|
+
await postRunAborted(config, runId, program.presetId, options.env);
|
|
247
|
+
};
|
|
248
|
+
const onInterrupt = (signal) => {
|
|
249
|
+
if (interrupted) {
|
|
250
|
+
// Second interrupt: the user wants out now. Hard-exit with the
|
|
251
|
+
// conventional 128+SIGINT(2) code; the best-effort notify already fired.
|
|
252
|
+
process.exit(130);
|
|
253
|
+
}
|
|
254
|
+
interrupted = true;
|
|
255
|
+
logger.warn(`[capture] ${signal} received — stopping the run and notifying AutoKap…`);
|
|
256
|
+
abortController.abort(new Error('User interrupted (Ctrl-C)'));
|
|
257
|
+
};
|
|
258
|
+
if (isLocalRegisteredRun) {
|
|
259
|
+
process.on('SIGINT', onInterrupt);
|
|
260
|
+
process.on('SIGTERM', onInterrupt);
|
|
261
|
+
}
|
|
223
262
|
const runOptions = {
|
|
224
263
|
recoveryChain,
|
|
225
|
-
abortSignal:
|
|
264
|
+
abortSignal: abortController.signal,
|
|
226
265
|
maxParallelVariants,
|
|
227
266
|
llmConfig,
|
|
228
267
|
presetName: program.presetId,
|
|
@@ -292,15 +331,30 @@ export async function runCapture(options) {
|
|
|
292
331
|
};
|
|
293
332
|
let runResult;
|
|
294
333
|
let cliResult;
|
|
334
|
+
let runAborted = false;
|
|
295
335
|
try {
|
|
296
336
|
runResult = await executeProgram(program, createAdapter, runOptions);
|
|
337
|
+
runAborted = interrupted || runResult.error === 'aborted';
|
|
297
338
|
if (runResult.success) {
|
|
298
339
|
logger.info(`[capture] Run completed successfully — ${runResult.telemetry.totalOpcodes} opcodes, ${runResult.telemetry.recoveredOpcodes} recovered, ${runResult.totalDurationMs}ms`);
|
|
299
340
|
}
|
|
341
|
+
else if (runAborted) {
|
|
342
|
+
logger.warn('[capture] Run interrupted by the user — skipping artifact upload.');
|
|
343
|
+
}
|
|
300
344
|
else {
|
|
301
345
|
logger.error(`[capture] Run failed: ${runResult.error}`);
|
|
302
346
|
}
|
|
303
|
-
if (
|
|
347
|
+
if (runAborted) {
|
|
348
|
+
// Don't upload partial artifacts for a cancelled run; the server is told
|
|
349
|
+
// separately (notifyAborted) so the preset leaves the "capturing" state.
|
|
350
|
+
cliResult = {
|
|
351
|
+
success: false,
|
|
352
|
+
runId,
|
|
353
|
+
runResult,
|
|
354
|
+
error: 'Capture interrupted (Ctrl-C)',
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
else if (options.dryRun) {
|
|
304
358
|
logger.info(`[capture] DRY RUN complete — ${runResult.telemetry.totalOpcodes} opcodes executed, 0 captures, 0 credits charged`);
|
|
305
359
|
cliResult = { success: runResult.success, runId, runResult };
|
|
306
360
|
}
|
|
@@ -354,6 +408,16 @@ export async function runCapture(options) {
|
|
|
354
408
|
}
|
|
355
409
|
}
|
|
356
410
|
finally {
|
|
411
|
+
if (isLocalRegisteredRun) {
|
|
412
|
+
process.off('SIGINT', onInterrupt);
|
|
413
|
+
process.off('SIGTERM', onInterrupt);
|
|
414
|
+
}
|
|
415
|
+
// On interruption, mark the run terminal server-side regardless of the
|
|
416
|
+
// user's debug-log preference, so the preset never stays stuck "capturing".
|
|
417
|
+
// Idempotent with the error-log flush below (same capture_failed dedupeKey).
|
|
418
|
+
if (runAborted) {
|
|
419
|
+
await notifyAborted();
|
|
420
|
+
}
|
|
357
421
|
// AUT-149: export structured debug logs to AutoKap on capture failure.
|
|
358
422
|
// Best-effort — the LogCollector swallows network errors.
|
|
359
423
|
const shouldExport = options.exportDebugLogs !== false
|
|
@@ -693,6 +757,29 @@ async function postRunStart(config, runId, presetId, variantCount, env) {
|
|
|
693
757
|
logger.warn(`[capture] Run registration error: ${message}`);
|
|
694
758
|
}
|
|
695
759
|
}
|
|
760
|
+
// Best-effort terminal notification when a local run is interrupted (Ctrl-C).
|
|
761
|
+
// Marks the run failed server-side and clears the preset's "capturing" badge,
|
|
762
|
+
// so an aborted capture can't leave the preset stuck "En cours". Never throws.
|
|
763
|
+
async function postRunAborted(config, runId, presetId, env) {
|
|
764
|
+
try {
|
|
765
|
+
const response = await fetch(`${config.apiBaseUrl}/api/cli/runs`, {
|
|
766
|
+
method: 'PATCH',
|
|
767
|
+
headers: {
|
|
768
|
+
'Authorization': `Bearer ${config.apiKey}`,
|
|
769
|
+
'Content-Type': 'application/json',
|
|
770
|
+
},
|
|
771
|
+
body: JSON.stringify({ runId, presetId, status: 'aborted', env }),
|
|
772
|
+
signal: AbortSignal.timeout(10_000),
|
|
773
|
+
});
|
|
774
|
+
if (!response.ok) {
|
|
775
|
+
logger.warn(`[capture] Failed to report interruption (HTTP ${response.status}); the preset may show "in progress" until the stale cutoff`);
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
catch (err) {
|
|
779
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
780
|
+
logger.warn(`[capture] Interruption report error: ${message}`);
|
|
781
|
+
}
|
|
782
|
+
}
|
|
696
783
|
async function uploadResults(config, program, result, runId, sessionId, provenance) {
|
|
697
784
|
const artifactJobs = result.variantResults.flatMap((variant) => {
|
|
698
785
|
const variantSpec = program.variants.find((entry) => entry.id === variant.variantId);
|
|
@@ -600,10 +600,11 @@ export interface ExecutionProgram {
|
|
|
600
600
|
*/
|
|
601
601
|
deviceConfigs?: Record<string, DeviceConfig>;
|
|
602
602
|
/**
|
|
603
|
-
* Project-level
|
|
603
|
+
* Project-level decorative URL used to decorate browser mockups. The CLI
|
|
604
604
|
* substitutes the captured origin (typically a local dev server) with this
|
|
605
605
|
* value via `transformBrowserUrl` before baking it into the browser bar.
|
|
606
|
-
*
|
|
606
|
+
* AUT-269: derived automatically from the project's prod environment base
|
|
607
|
+
* URL (absent when no prod environment is configured), not a separate field.
|
|
607
608
|
*/
|
|
608
609
|
publicUrl?: string;
|
|
609
610
|
/**
|
package/dist/opcode-runner.js
CHANGED
|
@@ -156,8 +156,31 @@ export async function executeProgram(program, createAdapter, options = {}) {
|
|
|
156
156
|
});
|
|
157
157
|
await Promise.all(workers);
|
|
158
158
|
const completedVariantResults = variantResults.filter((result) => Boolean(result));
|
|
159
|
-
const aborted = options.abortSignal?.aborted && completedVariantResults.length < program.variants.length;
|
|
160
|
-
|
|
159
|
+
const aborted = Boolean(options.abortSignal?.aborted) && completedVariantResults.length < program.variants.length;
|
|
160
|
+
// Fail-closed on incomplete delivery. Previously `success` was computed purely
|
|
161
|
+
// from the SURVIVING variant results, so a run that silently dropped variants
|
|
162
|
+
// (a variant finishing "ok" while persisting no artifact) was recorded as a
|
|
163
|
+
// clean success — a 4-of-24 capture looked identical to a full run, got billed,
|
|
164
|
+
// was never retried, and surfaced nothing. Require that every variant ran,
|
|
165
|
+
// succeeded, AND produced its expected number of artifacts; name the deficient
|
|
166
|
+
// variants so the failure is actionable and the runtime cause is diagnosable
|
|
167
|
+
// straight from the run log. Dry runs intentionally skip capture opcodes (and
|
|
168
|
+
// programs with no capture points have nothing to enforce), so they are exempt
|
|
169
|
+
// from the artifact check but still require every variant to succeed.
|
|
170
|
+
const expectedArtifactsPerVariant = program.steps.filter((step) => isArtifactProducingOpcode(step.kind)).length;
|
|
171
|
+
const enforceArtifactCompleteness = !options.dryRun && expectedArtifactsPerVariant > 0;
|
|
172
|
+
const deficientVariants = program.variants
|
|
173
|
+
.map((variant, index) => ({ variant, result: variantResults[index] }))
|
|
174
|
+
.filter(({ result }) => !result
|
|
175
|
+
|| !result.success
|
|
176
|
+
|| (enforceArtifactCompleteness && result.artifacts.length < expectedArtifactsPerVariant))
|
|
177
|
+
.map(({ variant, result }) => !result
|
|
178
|
+
? `${variant.id} (not executed)`
|
|
179
|
+
: !result.success
|
|
180
|
+
? `${variant.id} (failed: ${result.error ?? 'unknown error'})`
|
|
181
|
+
: `${variant.id} (${result.artifacts.length}/${expectedArtifactsPerVariant} artifacts)`);
|
|
182
|
+
const incompleteDelivery = deficientVariants.length > 0;
|
|
183
|
+
const success = !aborted && completedVariantResults.length > 0 && !incompleteDelivery;
|
|
161
184
|
const detectedAppVersion = completedVariantResults.reduce((acc, variantResult) => acc ?? (variantResult.detectedAppVersion ?? null), null);
|
|
162
185
|
// AUT-241 — surface (don't mask) cuts: aggregate every recording warning from
|
|
163
186
|
// each variant's clip/video artifacts. Diagnostic only; never affects success.
|
|
@@ -172,7 +195,11 @@ export async function executeProgram(program, createAdapter, options = {}) {
|
|
|
172
195
|
totalDurationMs: Date.now() - startTime,
|
|
173
196
|
detectedAppVersion,
|
|
174
197
|
warnings: aggregatedWarnings.length ? aggregatedWarnings : undefined,
|
|
175
|
-
error: aborted
|
|
198
|
+
error: aborted
|
|
199
|
+
? 'aborted'
|
|
200
|
+
: success
|
|
201
|
+
? undefined
|
|
202
|
+
: `incomplete run: ${deficientVariants.length}/${program.variants.length} variant(s) did not deliver expected artifacts — ${deficientVariants.join('; ')}`,
|
|
176
203
|
failureKind: success ? undefined : completedVariantResults.find(v => v.failureKind)?.failureKind,
|
|
177
204
|
};
|
|
178
205
|
}
|
|
@@ -320,6 +347,13 @@ function softSkipResult(opcode, index, startTime, reason, telemetry) {
|
|
|
320
347
|
error: reason,
|
|
321
348
|
};
|
|
322
349
|
}
|
|
350
|
+
/** Opcodes whose ACTION produces a persisted artifact (a screenshot or a finalized
|
|
351
|
+
* video clip). A passing postcondition does NOT imply the artifact exists, so these
|
|
352
|
+
* get special handling on the recovery path (executeOpcode → failWithRecovery) and
|
|
353
|
+
* in the run-level completeness gate (executeProgram). */
|
|
354
|
+
function isArtifactProducingOpcode(kind) {
|
|
355
|
+
return kind === 'CAPTURE_SCREENSHOT' || kind === 'END_CLIP';
|
|
356
|
+
}
|
|
323
357
|
async function executeOpcode(opcode, index, adapter, verifier, breaker, recoveryChain, telemetry, healerPatches, opcodeTimings, artifacts, options, variantId, executionState, artifactPlan, mockDataGroups, currentVariant, credentials) {
|
|
324
358
|
const startTime = Date.now();
|
|
325
359
|
const effectiveTimeoutMs = resolveOpcodeTimeoutMs(opcode);
|
|
@@ -333,6 +367,47 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
|
|
|
333
367
|
const getProgress = makeProgressGetter(adapter);
|
|
334
368
|
const actionEffectPolicy = getOpcodeActionEffectPolicy(opcode);
|
|
335
369
|
const isSoft = isSoftOpcodeKind(opcode.kind);
|
|
370
|
+
// Snapshot so we can tell whether THIS opcode produced its artifact. A recovered
|
|
371
|
+
// CAPTURE_SCREENSHOT must not pass as a phantom success when no screenshot was
|
|
372
|
+
// taken — the postcondition (e.g. element_visible) can pass without a capture.
|
|
373
|
+
const artifactCountAtStart = artifacts.length;
|
|
374
|
+
// Wraps handleFailure: when recovery succeeds for an artifact-producing opcode
|
|
375
|
+
// that pushed NO artifact for itself, re-run the capture once so the artifact
|
|
376
|
+
// truly exists; hard-fail if it still can't. END_CLIP finalization is stateful,
|
|
377
|
+
// so it hard-fails without re-running. Without this a "recovered"
|
|
378
|
+
// CAPTURE_SCREENSHOT finishes ok with no screenshot — the silent partial loss.
|
|
379
|
+
const failWithRecovery = async (reason) => {
|
|
380
|
+
const failureResult = await handleFailure(opcode, index, adapter, verifier, breaker, recoveryChain, telemetry, healerPatches, options, executionState, variantId, currentVariant, startTime, deadlineMs, globalDeadlineMs, effectiveTimeoutMs, reason);
|
|
381
|
+
if (failureResult.status !== 'recovered'
|
|
382
|
+
|| !isArtifactProducingOpcode(opcode.kind)
|
|
383
|
+
|| artifacts.length > artifactCountAtStart) {
|
|
384
|
+
return failureResult;
|
|
385
|
+
}
|
|
386
|
+
if (opcode.kind === 'CAPTURE_SCREENSHOT') {
|
|
387
|
+
const recaptureBudgetMs = getRemainingTimeMs(globalDeadlineMs);
|
|
388
|
+
if (recaptureBudgetMs > 0) {
|
|
389
|
+
let recapture;
|
|
390
|
+
try {
|
|
391
|
+
recapture = await withTimeout(() => executeOpcodeAction(opcode, index, adapter, artifacts, telemetry, currentVariant, executionState, artifactPlan, mockDataGroups, options, credentials), recaptureBudgetMs);
|
|
392
|
+
}
|
|
393
|
+
catch (err) {
|
|
394
|
+
recapture = { success: false, error: err instanceof Error ? err.message : String(err) };
|
|
395
|
+
}
|
|
396
|
+
if (recapture.success && artifacts.length > artifactCountAtStart) {
|
|
397
|
+
logger.debug(`[opcode ${index}] re-captured screenshot after recovery (${reason})`);
|
|
398
|
+
return failureResult;
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
return {
|
|
403
|
+
opcodeIndex: index,
|
|
404
|
+
kind: opcode.kind,
|
|
405
|
+
status: 'failed',
|
|
406
|
+
durationMs: Date.now() - startTime,
|
|
407
|
+
recoveryAttempts: failureResult.recoveryAttempts ?? 1,
|
|
408
|
+
error: `recovery succeeded but produced no ${opcode.kind === 'END_CLIP' ? 'clip' : 'screenshot'} artifact: ${reason}`,
|
|
409
|
+
};
|
|
410
|
+
};
|
|
336
411
|
// Track page context for circuit breaker
|
|
337
412
|
try {
|
|
338
413
|
const url = await adapter.getCurrentUrl();
|
|
@@ -358,7 +433,7 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
|
|
|
358
433
|
logger.debug(`[opcode ${index}] no budget left after captureBeforeState (deadline=${actionDeadlineMs}, now=${Date.now()})`);
|
|
359
434
|
if (isSoft)
|
|
360
435
|
return softSkipResult(opcode, index, startTime, reason, telemetry);
|
|
361
|
-
return
|
|
436
|
+
return failWithRecovery(reason);
|
|
362
437
|
}
|
|
363
438
|
// For mediaMode='video', capture pre-action timing + bbox metadata inside
|
|
364
439
|
// the active clip window only. Opcodes outside a clip are not part of the
|
|
@@ -407,7 +482,7 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
|
|
|
407
482
|
const reason = result.error ?? 'action failed';
|
|
408
483
|
if (isSoft)
|
|
409
484
|
return softSkipResult(opcode, index, startTime, reason, telemetry);
|
|
410
|
-
return
|
|
485
|
+
return failWithRecovery(reason);
|
|
411
486
|
}
|
|
412
487
|
// Verify postcondition — extend-on-progress up to the global deadline so a
|
|
413
488
|
// slow action no longer starves it (failure mode #3: clamped to ~1ms).
|
|
@@ -417,7 +492,7 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
|
|
|
417
492
|
logger.debug(`[opcode ${index}] no budget left for postcondition check`);
|
|
418
493
|
if (isSoft)
|
|
419
494
|
return softSkipResult(opcode, index, startTime, reason, telemetry);
|
|
420
|
-
return
|
|
495
|
+
return failWithRecovery(reason);
|
|
421
496
|
}
|
|
422
497
|
const runtimePostcondition = resolveRuntimePostcondition(opcode);
|
|
423
498
|
const postStart = Date.now();
|
|
@@ -430,7 +505,7 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
|
|
|
430
505
|
const reason = `postcondition failed: ${postcondition.reason}`;
|
|
431
506
|
if (isSoft)
|
|
432
507
|
return softSkipResult(opcode, index, startTime, reason, telemetry);
|
|
433
|
-
return
|
|
508
|
+
return failWithRecovery(reason);
|
|
434
509
|
}
|
|
435
510
|
// Verify action effects through the shared policy. Weak `any_change`
|
|
436
511
|
// postconditions are only meaningful if this verifier observes a real
|
|
@@ -446,7 +521,7 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
|
|
|
446
521
|
`postcondition passed, treating as redundant-but-successful`);
|
|
447
522
|
}
|
|
448
523
|
else {
|
|
449
|
-
return
|
|
524
|
+
return failWithRecovery(`action had no effect: ${verification.summary}`);
|
|
450
525
|
}
|
|
451
526
|
}
|
|
452
527
|
}
|
|
@@ -480,7 +555,7 @@ async function executeOpcode(opcode, index, adapter, verifier, breaker, recovery
|
|
|
480
555
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
481
556
|
if (isSoft)
|
|
482
557
|
return softSkipResult(opcode, index, startTime, errorMsg, telemetry);
|
|
483
|
-
return
|
|
558
|
+
return failWithRecovery(errorMsg);
|
|
484
559
|
}
|
|
485
560
|
}
|
|
486
561
|
/** Post-action breathing room (ms) injected between visible interactions
|
package/dist/postcondition.js
CHANGED
|
@@ -100,8 +100,8 @@ function compileRoutePattern(pattern) {
|
|
|
100
100
|
// Support glob-like patterns: ** matches anything (incl. slashes / empty),
|
|
101
101
|
// * matches a single path segment, ? matches one non-slash char.
|
|
102
102
|
// Tokenize in one pass so the `*` rewrite doesn't clobber the `*` produced
|
|
103
|
-
// by the `**` rewrite (e.g. `/home**` must compile to
|
|
104
|
-
//
|
|
103
|
+
// by the `**` rewrite (e.g. `/home**` must compile to `/home.*`, not
|
|
104
|
+
// `/home.[^/]*` which would reject `/home` itself).
|
|
105
105
|
let regexStr = '';
|
|
106
106
|
for (let i = 0; i < pattern.length; i++) {
|
|
107
107
|
const ch = pattern[i];
|
|
@@ -122,7 +122,14 @@ function compileRoutePattern(pattern) {
|
|
|
122
122
|
regexStr += ch;
|
|
123
123
|
}
|
|
124
124
|
}
|
|
125
|
-
|
|
125
|
+
// Substring (contains) match — NOT anchored. Generated programs author bare
|
|
126
|
+
// patterns that are either a prefix of the real path (`/projects/` ⊂
|
|
127
|
+
// `/projects/<id>`) or a nested segment (`/tracking` ⊂
|
|
128
|
+
// `/projects/<id>/tracking`). An anchored `^…$` could match neither, which
|
|
129
|
+
// surfaced as a misleading "page stuck, no progress" failure after the
|
|
130
|
+
// navigation had actually succeeded. Callers needing strict matching pass an
|
|
131
|
+
// anchored regex (handled above).
|
|
132
|
+
return new RegExp(regexStr);
|
|
126
133
|
}
|
|
127
134
|
async function checkElementVisible(adapter, selector) {
|
|
128
135
|
// Primary check: use Playwright waitFor (fast, reliable)
|