@qawolf/cli 1.9.2 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js
CHANGED
|
@@ -156412,12 +156412,14 @@ var interactiveRunnerMessages = {
|
|
|
156412
156412
|
launched: (id) => `Launched runner ${id}.`,
|
|
156413
156413
|
launchedForCommand: (id) => `No runner was given, so launched ${id} for this command. Its browser is fresh: nothing has been run on it and nothing is signed in. It bills until it is stopped or idles out, so stop it with qawolf runner stop --runner ${id} when you are done.`,
|
|
156414
156414
|
followEventsTimedOut: (stream, seconds) => `Stopped following ${stream} after ${formatSeconds(seconds * 1000)}: reading keeps the runner alive and billing, so a follow does not run unbounded. Pass --timeout to wait longer, or follow again to continue.`,
|
|
156415
|
+
followEndCutShort: "The run settled, but the last window of its followed streams could not be read, so the output above may be missing its final lines.",
|
|
156415
156416
|
followTimedOut: (runId, runnerId, seconds) => `Stopped following run ${runId} after ${formatSeconds(seconds * 1000)}. The run may still be going: read it with qawolf runner events run-status --run ${runId}, and stop the runner with qawolf runner stop --runner ${runnerId} when you are done. Pass --timeout to wait longer.`,
|
|
156416
156417
|
missingPackageJson: "No package.json in the current directory. A run reads its npm dependencies from one, so it has to travel with the flow.",
|
|
156417
156418
|
noRunnerIdForScreenshot: `${noRunnerId} A screenshot also needs a screen, which a runner gets from its first run: run a flow on it with qawolf runner run.`,
|
|
156418
156419
|
noRunnerId,
|
|
156419
156420
|
notRunning: (id) => `Runner ${id} was not running.`,
|
|
156420
156421
|
runFailed: (errorMessage2) => errorMessage2 === undefined ? "The run failed and reported no reason." : `The run failed: ${errorMessage2}`,
|
|
156422
|
+
runInProgress: "The run is in progress.",
|
|
156421
156423
|
runPassed: "The run passed.",
|
|
156422
156424
|
runSettledUnknown: (status) => `The run settled as "${status}", which this version of the CLI does not recognize. Upgrade to read it.`,
|
|
156423
156425
|
runSubmitAnsweredUnknown: (outcome) => `The runner answered the submission with "${outcome}", which this version of the CLI does not recognize. Upgrade to read it.`,
|
|
@@ -175909,7 +175911,7 @@ function startUpdateCheck(deps) {
|
|
|
175909
175911
|
// package.json
|
|
175910
175912
|
var package_default = {
|
|
175911
175913
|
name: "@qawolf/cli",
|
|
175912
|
-
version: "1.
|
|
175914
|
+
version: "1.11.0",
|
|
175913
175915
|
description: "Run and manage QA Wolf flows from the terminal, CI, or an AI agent",
|
|
175914
175916
|
keywords: [
|
|
175915
175917
|
"automation",
|
|
@@ -185968,6 +185970,14 @@ function flagUsage(field, kind) {
|
|
|
185968
185970
|
return `${name} <KEY=VALUE...>`;
|
|
185969
185971
|
return `${name} <value>`;
|
|
185970
185972
|
}
|
|
185973
|
+
function describeFlag(schema) {
|
|
185974
|
+
const description = schema.description ?? "";
|
|
185975
|
+
const values = schema.enum ?? schema.items?.enum;
|
|
185976
|
+
if (!values?.length)
|
|
185977
|
+
return description;
|
|
185978
|
+
const choices = `One of: ${values.join(", ")}`;
|
|
185979
|
+
return description ? `${description} ${choices}` : choices;
|
|
185980
|
+
}
|
|
185971
185981
|
function buildFlagSpecs(inputSchema) {
|
|
185972
185982
|
const jsonSchema = exports_external.toJSONSchema(inputSchema, {
|
|
185973
185983
|
io: "input"
|
|
@@ -185983,7 +185993,7 @@ function buildFlagSpecs(inputSchema) {
|
|
|
185983
185993
|
flags.push({
|
|
185984
185994
|
field,
|
|
185985
185995
|
flag: flagUsage(field, kind.kind),
|
|
185986
|
-
description: fieldSchema
|
|
185996
|
+
description: describeFlag(fieldSchema),
|
|
185987
185997
|
required: result.shape.required.has(field),
|
|
185988
185998
|
kind: kind.kind
|
|
185989
185999
|
});
|
|
@@ -186090,7 +186100,7 @@ async function handlePublicApiCommand(ctx, spec, options) {
|
|
|
186090
186100
|
}
|
|
186091
186101
|
|
|
186092
186102
|
// src/domains/publicApi/skippedContracts.ts
|
|
186093
|
-
var
|
|
186103
|
+
var skippedContractNames = new Set([
|
|
186094
186104
|
"flow.list",
|
|
186095
186105
|
"runner.evaluateSnippet",
|
|
186096
186106
|
"runner.launch",
|
|
@@ -186100,13 +186110,6 @@ var handWrittenContractNames = new Set([
|
|
|
186100
186110
|
"runner.stop",
|
|
186101
186111
|
"runner.takeScreenshot"
|
|
186102
186112
|
]);
|
|
186103
|
-
var unexpressibleContractNames = new Set([
|
|
186104
|
-
"issue.update"
|
|
186105
|
-
]);
|
|
186106
|
-
var skippedContractNames = new Set([
|
|
186107
|
-
...handWrittenContractNames,
|
|
186108
|
-
...unexpressibleContractNames
|
|
186109
|
-
]);
|
|
186110
186113
|
|
|
186111
186114
|
// src/commands/publicApi/index.ts
|
|
186112
186115
|
var groupDescriptions = {
|
|
@@ -186853,6 +186856,20 @@ function readRunSettlement(payload) {
|
|
|
186853
186856
|
type: "settled"
|
|
186854
186857
|
};
|
|
186855
186858
|
}
|
|
186859
|
+
function findSettlement(entries) {
|
|
186860
|
+
for (const entry of entries) {
|
|
186861
|
+
const settlement = readRunSettlement(entry.payload);
|
|
186862
|
+
if (settlement.type !== "settled")
|
|
186863
|
+
continue;
|
|
186864
|
+
if (settlement.status === "passed")
|
|
186865
|
+
return { type: "passed" };
|
|
186866
|
+
if (settlement.status === "failed") {
|
|
186867
|
+
return { errorMessage: settlement.errorMessage, type: "failed" };
|
|
186868
|
+
}
|
|
186869
|
+
return { status: settlement.status, type: "unrecognized" };
|
|
186870
|
+
}
|
|
186871
|
+
return;
|
|
186872
|
+
}
|
|
186856
186873
|
var runLogPayloadSchema = exports_external.object({ message: exports_external.string() });
|
|
186857
186874
|
function formatRunLogLine(payload) {
|
|
186858
186875
|
const parsed = runLogPayloadSchema.safeParse(payload);
|
|
@@ -186915,6 +186932,18 @@ function createJournalCursor(ctx, runnerId, request) {
|
|
|
186915
186932
|
return { entries: window2.value.entries, type: "entries" };
|
|
186916
186933
|
};
|
|
186917
186934
|
}
|
|
186935
|
+
function createPrintingCursor(ctx, runnerId, request, format) {
|
|
186936
|
+
const read = createJournalCursor(ctx, runnerId, request);
|
|
186937
|
+
return async () => {
|
|
186938
|
+
const window2 = await read();
|
|
186939
|
+
if (window2.type !== "entries")
|
|
186940
|
+
return window2;
|
|
186941
|
+
for (const entry of window2.entries) {
|
|
186942
|
+
ctx.ui.stream(entry, format(entry.payload));
|
|
186943
|
+
}
|
|
186944
|
+
return window2;
|
|
186945
|
+
};
|
|
186946
|
+
}
|
|
186918
186947
|
var unreachableGraceMs = 60000;
|
|
186919
186948
|
function createUnreachableBudget(pollIntervalMs2) {
|
|
186920
186949
|
const limit = Math.ceil(unreachableGraceMs / pollIntervalMs2);
|
|
@@ -186996,22 +187025,49 @@ function readErrorPath(error51) {
|
|
|
186996
187025
|
return typeof path9 === "string" ? path9 : undefined;
|
|
186997
187026
|
}
|
|
186998
187027
|
|
|
186999
|
-
// src/domains/interactiveRunner/
|
|
187000
|
-
var
|
|
187001
|
-
function
|
|
187002
|
-
|
|
187003
|
-
|
|
187004
|
-
|
|
187005
|
-
|
|
187006
|
-
|
|
187007
|
-
|
|
187008
|
-
|
|
187009
|
-
|
|
187028
|
+
// src/domains/interactiveRunner/followPrinters.ts
|
|
187029
|
+
var anchorPollIntervalMs = 1000;
|
|
187030
|
+
async function resolveRecorderAnchor(ctx, resolved, deps) {
|
|
187031
|
+
if (resolved.type === "launched")
|
|
187032
|
+
return { ok: true, sinceSequence: 0 };
|
|
187033
|
+
return anchorRecorderCursor(ctx, resolved.runnerId, deps);
|
|
187034
|
+
}
|
|
187035
|
+
async function anchorRecorderCursor(ctx, runnerId, deps) {
|
|
187036
|
+
const unreachable = createUnreachableBudget(anchorPollIntervalMs);
|
|
187037
|
+
for (;; ) {
|
|
187038
|
+
const anchor = await readJournal(ctx, runnerId, {
|
|
187039
|
+
stream: "recorder",
|
|
187040
|
+
tail: 1
|
|
187041
|
+
});
|
|
187042
|
+
if (anchor.type === "read") {
|
|
187043
|
+
return { ok: true, sinceSequence: anchor.value.nextSequence };
|
|
187010
187044
|
}
|
|
187011
|
-
|
|
187045
|
+
if (anchor.type === "failed") {
|
|
187046
|
+
return { failure: journalReadFailure(anchor), ok: false };
|
|
187047
|
+
}
|
|
187048
|
+
if (unreachable.exhausted()) {
|
|
187049
|
+
return { failure: { ...unreachableFailure }, ok: false };
|
|
187050
|
+
}
|
|
187051
|
+
await deps.sleep(anchorPollIntervalMs);
|
|
187012
187052
|
}
|
|
187013
|
-
return;
|
|
187014
187053
|
}
|
|
187054
|
+
function createFollowPrinters(ctx, options) {
|
|
187055
|
+
const jsonLine = (payload) => JSON.stringify(payload);
|
|
187056
|
+
const printers = [];
|
|
187057
|
+
if (options.logs) {
|
|
187058
|
+
printers.push(createPrintingCursor(ctx, options.runnerId, { runId: options.runId, stream: "run-logs" }, formatRunLogLine));
|
|
187059
|
+
}
|
|
187060
|
+
if (options.runEvents) {
|
|
187061
|
+
printers.push(createPrintingCursor(ctx, options.runnerId, { runId: options.runId, stream: "run-events" }, jsonLine));
|
|
187062
|
+
}
|
|
187063
|
+
if (options.recorderSinceSequence !== undefined) {
|
|
187064
|
+
printers.push(createPrintingCursor(ctx, options.runnerId, { sinceSequence: options.recorderSinceSequence, stream: "recorder" }, jsonLine));
|
|
187065
|
+
}
|
|
187066
|
+
return printers;
|
|
187067
|
+
}
|
|
187068
|
+
|
|
187069
|
+
// src/domains/interactiveRunner/followRun.ts
|
|
187070
|
+
var pollIntervalMs3 = 1000;
|
|
187015
187071
|
function reportSettlement(ctx, settlement) {
|
|
187016
187072
|
if (settlement.type === "passed") {
|
|
187017
187073
|
ctx.ui.success(interactiveRunnerMessages.runPassed);
|
|
@@ -187023,40 +187079,49 @@ function reportSettlement(ctx, settlement) {
|
|
|
187023
187079
|
};
|
|
187024
187080
|
}
|
|
187025
187081
|
async function followRun(ctx, options, deps) {
|
|
187026
|
-
const
|
|
187027
|
-
runId: options.runId,
|
|
187028
|
-
stream: "run-logs"
|
|
187029
|
-
});
|
|
187082
|
+
const printers = createFollowPrinters(ctx, options);
|
|
187030
187083
|
const readStatus = createJournalCursor(ctx, options.runnerId, {
|
|
187031
187084
|
runId: options.runId,
|
|
187032
187085
|
stream: "run-status"
|
|
187033
187086
|
});
|
|
187034
187087
|
const unreachable = createUnreachableBudget(pollIntervalMs3);
|
|
187035
|
-
const
|
|
187036
|
-
const
|
|
187037
|
-
|
|
187038
|
-
|
|
187039
|
-
|
|
187040
|
-
ctx.ui.stream(entry, formatRunLogLine(entry.payload));
|
|
187088
|
+
const printAll = async () => {
|
|
187089
|
+
for (const print of printers) {
|
|
187090
|
+
const window2 = await print();
|
|
187091
|
+
if (window2.type !== "entries")
|
|
187092
|
+
return window2;
|
|
187041
187093
|
}
|
|
187042
|
-
return
|
|
187094
|
+
return;
|
|
187095
|
+
};
|
|
187096
|
+
let progressReported = false;
|
|
187097
|
+
const printProgress = (entries) => {
|
|
187098
|
+
if (printers.length > 0 || progressReported)
|
|
187099
|
+
return;
|
|
187100
|
+
const entry = entries.find((e) => readRunSettlement(e.payload).type === "in-progress");
|
|
187101
|
+
if (entry === undefined)
|
|
187102
|
+
return;
|
|
187103
|
+
progressReported = true;
|
|
187104
|
+
ctx.ui.stream(entry, interactiveRunnerMessages.runInProgress);
|
|
187043
187105
|
};
|
|
187044
187106
|
const maxPolls = Math.max(1, Math.ceil(options.timeoutSeconds * 1000 / pollIntervalMs3));
|
|
187045
187107
|
for (let poll = 1;; poll++) {
|
|
187046
|
-
const
|
|
187047
|
-
if (
|
|
187048
|
-
return journalReadFailure(
|
|
187049
|
-
const status =
|
|
187108
|
+
const interrupted = await printAll();
|
|
187109
|
+
if (interrupted?.type === "failed")
|
|
187110
|
+
return journalReadFailure(interrupted);
|
|
187111
|
+
const status = interrupted?.type === "unreachable" ? interrupted : await readStatus();
|
|
187050
187112
|
if (status.type === "failed")
|
|
187051
187113
|
return journalReadFailure(status);
|
|
187052
|
-
if (
|
|
187114
|
+
if (status.type === "unreachable") {
|
|
187053
187115
|
if (unreachable.exhausted())
|
|
187054
187116
|
return { ...unreachableFailure };
|
|
187055
187117
|
} else {
|
|
187056
187118
|
unreachable.reset();
|
|
187119
|
+
printProgress(status.entries);
|
|
187057
187120
|
const settlement = findSettlement(status.entries);
|
|
187058
187121
|
if (settlement !== undefined) {
|
|
187059
|
-
await
|
|
187122
|
+
if (await printAll() !== undefined) {
|
|
187123
|
+
ctx.ui.warn(interactiveRunnerMessages.followEndCutShort);
|
|
187124
|
+
}
|
|
187060
187125
|
return reportSettlement(ctx, settlement);
|
|
187061
187126
|
}
|
|
187062
187127
|
}
|
|
@@ -187094,6 +187159,13 @@ async function handleRunnerRun(ctx, options, deps) {
|
|
|
187094
187159
|
return { ...failureFields(resolved), exitCode: resolved.exitCode };
|
|
187095
187160
|
}
|
|
187096
187161
|
announceRunner(ctx, resolved);
|
|
187162
|
+
let recorderSinceSequence;
|
|
187163
|
+
if (options.recorderEvents) {
|
|
187164
|
+
const anchor = await resolveRecorderAnchor(ctx, resolved, deps);
|
|
187165
|
+
if (!anchor.ok)
|
|
187166
|
+
return { ...anchor.failure };
|
|
187167
|
+
recorderSinceSequence = anchor.sinceSequence;
|
|
187168
|
+
}
|
|
187097
187169
|
const result = await ctx.platformClient.callPublicApi(publicContractsV1.runner.runFlow, { entryPointPath, files, id: resolved.runnerId }, runnerCallOptions);
|
|
187098
187170
|
if (!result.ok) {
|
|
187099
187171
|
return { ...failureFields(result), exitCode: exitCodes.network };
|
|
@@ -187111,12 +187183,16 @@ async function handleRunnerRun(ctx, options, deps) {
|
|
|
187111
187183
|
};
|
|
187112
187184
|
case "submitted": {
|
|
187113
187185
|
const runId = result.value.runId;
|
|
187114
|
-
|
|
187186
|
+
const follow = options.follow || options.logs || options.runEvents || options.recorderEvents;
|
|
187187
|
+
if (!follow) {
|
|
187115
187188
|
ctx.ui.output({ runId, runnerId: resolved.runnerId }, interactiveRunnerMessages.runSubmitted(runId));
|
|
187116
187189
|
return;
|
|
187117
187190
|
}
|
|
187118
187191
|
ctx.ui.info(interactiveRunnerMessages.runSubmitted(runId));
|
|
187119
187192
|
return followRun(ctx, {
|
|
187193
|
+
logs: options.logs,
|
|
187194
|
+
recorderSinceSequence,
|
|
187195
|
+
runEvents: options.runEvents,
|
|
187120
187196
|
runId,
|
|
187121
187197
|
runnerId: resolved.runnerId,
|
|
187122
187198
|
timeoutSeconds: timeout.seconds
|
|
@@ -187136,16 +187212,20 @@ async function handleRunnerRun(ctx, options, deps) {
|
|
|
187136
187212
|
var runExamples2 = `
|
|
187137
187213
|
Examples:
|
|
187138
187214
|
$ qawolf runner run flows/checkout.flow.ts
|
|
187139
|
-
$ qawolf runner run flows/checkout.flow.ts --follow
|
|
187215
|
+
$ qawolf runner run flows/checkout.flow.ts --follow
|
|
187216
|
+
$ qawolf runner run flows/checkout.flow.ts --follow --logs`;
|
|
187140
187217
|
var eventsExamples = `
|
|
187141
187218
|
Examples:
|
|
187142
187219
|
$ qawolf runner events recorder --tail 5
|
|
187143
187220
|
$ qawolf runner events run-logs --run <runId> --follow
|
|
187144
187221
|
$ qawolf runner events console --since 120 --json`;
|
|
187145
187222
|
function registerRunnerRunCommands(runner, signals) {
|
|
187146
|
-
declareCommandKind(runner.command("run <file>"), "write").description("Run a flow on an interactive runner, shipping the current directory's files with it").option("--follow", "
|
|
187223
|
+
declareCommandKind(runner.command("run <file>"), "write").description("Run a flow on an interactive runner, shipping the current directory's files with it").option("--follow", "Report the run's status until it settles: in progress, then passed or failed", false).option("--logs", "Stream every log line the run produces while following. Implies --follow", false).option("--run-events", "Stream the run's progress events as JSON lines while following. Implies --follow", false).option("--recorder-events", "Stream the browser actions the runner records as JSON lines while following, from an anchor taken just before submission: the recorder is runner-wide, not run-scoped. Implies --follow", false).option("--runner <id>", runnerFlagDescription).option("--timeout <seconds>", "Give up following after this long. Following keeps the runner alive, so a run that never settles would otherwise bill until the terminal closed", String(defaultFollowTimeoutSeconds)).addHelpText("after", runExamples2).action((file2, opts, command) => withAuthContext(signals, (ctx) => handleRunnerRun(ctx, {
|
|
187147
187224
|
entryPoint: file2,
|
|
187148
187225
|
follow: opts.follow,
|
|
187226
|
+
logs: opts.logs,
|
|
187227
|
+
recorderEvents: opts.recorderEvents,
|
|
187228
|
+
runEvents: opts.runEvents,
|
|
187149
187229
|
runner: opts.runner,
|
|
187150
187230
|
timeout: opts.timeout
|
|
187151
187231
|
}, runnerDeps(ctx)))(opts, command));
|
|
@@ -187205,4 +187285,4 @@ createProgram({ signals }).parseAsync().catch(() => {
|
|
|
187205
187285
|
process.exitCode = 1;
|
|
187206
187286
|
}).finally(() => flushAndExit(typeof process.exitCode === "number" ? process.exitCode : 0));
|
|
187207
187287
|
|
|
187208
|
-
//# debugId=
|
|
187288
|
+
//# debugId=229B48C8231EE6BC64756E2164756E21
|
package/package.json
CHANGED
|
@@ -72,6 +72,15 @@ each printed line from the payload alone to the whole envelope (`sequence`,
|
|
|
72
72
|
`recordedAt`, `payload`). Both are JSON. Pass it when you want to page by
|
|
73
73
|
sequence, omit it when you want the payloads themselves.
|
|
74
74
|
|
|
75
|
+
A `--json` response shows you most of its own shape, so read it first.
|
|
76
|
+
|
|
77
|
+
`qawolf run get` is the exception worth reading about before you use it. Its
|
|
78
|
+
artifact URLs expire, its failure fields are absent from a passing run, and its
|
|
79
|
+
`traceUrl` downloads a Playwright trace that you can read as JSON without
|
|
80
|
+
opening the trace viewer. **Read
|
|
81
|
+
[`references/run-results.md`](references/run-results.md) before reporting on a
|
|
82
|
+
run's outcome or opening its trace.**
|
|
83
|
+
|
|
75
84
|
## Safety: reads vs writes
|
|
76
85
|
|
|
77
86
|
Read commands do not change team data, but some have operational effects noted
|
|
@@ -127,6 +136,7 @@ current branch.
|
|
|
127
136
|
| `qawolf issue create` | write | Create a bug or coverage request issue for the caller's team. Maintenance issues cannot be created through the public API. |
|
|
128
137
|
| `qawolf issue find` | read | List the team's bug reports, maintenance reports, or coverage requests, newest first. |
|
|
129
138
|
| `qawolf issue get` | read | Get an issue by id. |
|
|
139
|
+
| `qawolf issue update` | write | Update an issue owned by the caller's team. Omitted fields remain unchanged. |
|
|
130
140
|
| `qawolf run create` | write | Create a run for the selected flows and/or tags in an environment. |
|
|
131
141
|
| `qawolf run find` | read | List an environment's recent runs, newest first. |
|
|
132
142
|
| `qawolf run get` | read | Get a run's status, per-flow results, and links. |
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
# Reading a run's results
|
|
2
|
+
|
|
3
|
+
How to use what `qawolf run get --run-id <id> --json` returns, and how to read
|
|
4
|
+
the Playwright trace it links to.
|
|
5
|
+
|
|
6
|
+
Call `run get` with `--json` and you see most of the response immediately. Read
|
|
7
|
+
this file for the parts a single response cannot show you: fields that appear
|
|
8
|
+
only when something fails, rules about the artifact URLs, and how to read a
|
|
9
|
+
trace without opening the trace viewer.
|
|
10
|
+
|
|
11
|
+
## The shape
|
|
12
|
+
|
|
13
|
+
A run holds flows, a flow holds attempts, and artifacts hang off an attempt:
|
|
14
|
+
|
|
15
|
+
```text
|
|
16
|
+
run
|
|
17
|
+
└── flows[]
|
|
18
|
+
├── failure only when the flow failed
|
|
19
|
+
└── attempts[] oldest first
|
|
20
|
+
├── logsUrl
|
|
21
|
+
├── traceUrl
|
|
22
|
+
└── videoUrl
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
`runId` in the response is canonical and can differ from the id you asked for.
|
|
26
|
+
Use the returned value for follow-up calls.
|
|
27
|
+
|
|
28
|
+
Poll `status` until it reaches `passed`, `failed` or `canceled`. The other
|
|
29
|
+
values mean the run is still going.
|
|
30
|
+
|
|
31
|
+
## Fields a passing run does not show you
|
|
32
|
+
|
|
33
|
+
- `flows[].failure` exists only when a flow failed. Every flow passing means
|
|
34
|
+
there is no failure object at all, so its diagnosis and issue id are invisible
|
|
35
|
+
until something breaks. Do not conclude the field does not exist.
|
|
36
|
+
- `git` is populated only when a deploy notification started the run. A run
|
|
37
|
+
started manually or with `run create` has an empty object here.
|
|
38
|
+
- An attempt's `kind` and `status` select which other fields it has. Only
|
|
39
|
+
automated attempts that reached a verdict carry artifact URLs; canceled
|
|
40
|
+
attempts and manual Wolf Browser attempts carry none.
|
|
41
|
+
- A flow that passed after a retry still lists its failed attempts. Read the
|
|
42
|
+
last attempt for the outcome, and the earlier ones to see what went wrong.
|
|
43
|
+
|
|
44
|
+
## Artifact URLs
|
|
45
|
+
|
|
46
|
+
Each automated attempt links `logsUrl` (execution logs), `videoUrl` (screen
|
|
47
|
+
recording) and `traceUrl` (a Playwright `trace.zip`).
|
|
48
|
+
|
|
49
|
+
1. They are signed URLs with a limited life. The contract guarantees at least a
|
|
50
|
+
day. Call `run get` again for fresh ones instead of storing them; a stored
|
|
51
|
+
URL becomes a dead link.
|
|
52
|
+
2. A URL can return 404 when that attempt did not produce that artifact. Handle
|
|
53
|
+
the 404 rather than treating the URL's presence as a guarantee of content.
|
|
54
|
+
3. Download with a plain HTTP GET. The signature is in the URL, so no
|
|
55
|
+
authentication header is needed and no QA Wolf credentials are involved.
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
qawolf run get --run-id "$RUN_ID" --json \
|
|
59
|
+
| jq -r '.flows[].attempts[-1].traceUrl // empty' \
|
|
60
|
+
| head -1 \
|
|
61
|
+
| xargs -r curl -sS -o trace.zip
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## Reading the Playwright trace
|
|
65
|
+
|
|
66
|
+
The usual advice is `npx playwright show-trace trace.zip`, which opens a
|
|
67
|
+
browser window. That is useless in a shell and unnecessary: the zip holds
|
|
68
|
+
newline-delimited JSON files, and reading them directly is faster than
|
|
69
|
+
downloading a viewer.
|
|
70
|
+
|
|
71
|
+
The zip holds `trace.trace` (the events), `trace.network` (one request and
|
|
72
|
+
response per line) and a `resources/` directory of screencast frames. The
|
|
73
|
+
frames are most of the size, so extract only what you need.
|
|
74
|
+
|
|
75
|
+
### The event types
|
|
76
|
+
|
|
77
|
+
Every line of `trace.trace` is one JSON object with a `type`:
|
|
78
|
+
|
|
79
|
+
- `before` — a call started. Carries `callId`, `startTime`, `class`, `method`
|
|
80
|
+
and `params`. `params.selector` or `params.url` is usually the target.
|
|
81
|
+
- `after` — that call finished. Matched to its `before` by `callId`. Carries
|
|
82
|
+
`endTime` and `result`, and an `error` when the call failed.
|
|
83
|
+
- `console` — a browser console message, with `messageType` and `text`.
|
|
84
|
+
- `log` — Playwright's own progress notes for a call.
|
|
85
|
+
- `screencast-frame`, `frame-snapshot` — the filmstrip and DOM snapshots the
|
|
86
|
+
viewer renders. Usually not worth reading directly.
|
|
87
|
+
|
|
88
|
+
Two details cost time if you miss them:
|
|
89
|
+
|
|
90
|
+
- **Times are monotonic milliseconds, not seconds.** A `goto` whose `startTime`
|
|
91
|
+
and `endTime` differ by `161.6` took 161 milliseconds. Subtract the smallest
|
|
92
|
+
`startTime` to get an offset from the start of the trace.
|
|
93
|
+
- **Return values use a serialized envelope.** `{"value":{"s":"passed"}}` is
|
|
94
|
+
the string `passed`, `{"n":640}` is the number `640`, and `{"o":[...]}` is an
|
|
95
|
+
object as a list of key and value pairs.
|
|
96
|
+
|
|
97
|
+
### A worked example
|
|
98
|
+
|
|
99
|
+
Pairing `before` with `after` gives an action timeline with durations and
|
|
100
|
+
failures:
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
import json, sys, zipfile
|
|
104
|
+
|
|
105
|
+
with zipfile.ZipFile(sys.argv[1]) as z:
|
|
106
|
+
events = [json.loads(line) for line in z.read("trace.trace").decode().splitlines()]
|
|
107
|
+
|
|
108
|
+
starts = {e["callId"]: e for e in events if e["type"] == "before"}
|
|
109
|
+
ends = {e["callId"]: e for e in events if e["type"] == "after"}
|
|
110
|
+
t0 = min(e["startTime"] for e in starts.values())
|
|
111
|
+
|
|
112
|
+
for call_id, before in starts.items():
|
|
113
|
+
after = ends.get(call_id, {})
|
|
114
|
+
params = before.get("params", {})
|
|
115
|
+
target = params.get("selector") or params.get("url") or ""
|
|
116
|
+
error = after.get("error")
|
|
117
|
+
print(
|
|
118
|
+
f'{(before["startTime"] - t0) / 1000:7.2f}s'
|
|
119
|
+
f' {after.get("endTime", before["startTime"]) - before["startTime"]:7.1f}ms'
|
|
120
|
+
f' {before["class"]}.{before["method"]:<18} {target[:40]}'
|
|
121
|
+
f'{" FAILED: " + json.dumps(error)[:60] if error else ""}'
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
for e in events:
|
|
125
|
+
if e["type"] == "console" and e["messageType"] == "error":
|
|
126
|
+
print(f'console error: {e["text"][:70]}')
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
It prints one line per call, in order:
|
|
130
|
+
|
|
131
|
+
```text
|
|
132
|
+
0.00s 161.6ms Frame.goto https://example.com/
|
|
133
|
+
0.17s 28.3ms Frame.waitForSelector #screen
|
|
134
|
+
0.20s 4.2ms Frame.innerText #fps_stats
|
|
135
|
+
console error: Failed to load resource: the server responded with a status of 404 ()
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
To find why an attempt failed, read the last `after` that carries an `error`,
|
|
139
|
+
then the `console` errors near it in time. To see what the page did, read
|
|
140
|
+
`trace.network`.
|
|
141
|
+
|
|
142
|
+
## Response fields
|
|
143
|
+
|
|
144
|
+
Every documented field of the `run.get` response. `[]` marks an array, so
|
|
145
|
+
`flows[].attempts[].traceUrl` is the trace URL of one attempt of one flow.
|
|
146
|
+
|
|
147
|
+
<!-- fields:start — generated by `bun run generate`, do not edit -->
|
|
148
|
+
|
|
149
|
+
- `completedAt` — When the run finished executing. Absent while queued or running, and also absent for a terminal run that never completed execution (e.g. every flow was canceled or skipped).
|
|
150
|
+
- `git` — The branch and commit under test. The fields are present when a deploy notification started the run, and absent for runs started another way, for example manually or with run.create.
|
|
151
|
+
- `git.commitUrl` — Link to the commit on the code host.
|
|
152
|
+
- `runId` — The run this response describes. Treat it as canonical: it can differ from the id you asked for. A deploy notification returns a run id before the run exists, and if a second notification for the same commit is folded into an earlier run, that id resolves to the earlier run instead.
|
|
153
|
+
- `status` — One of: queued, running, passed, failed, canceled
|
|
154
|
+
- `flows` — The run's flows, ordered alphabetically by name.
|
|
155
|
+
- `flows[].attempts` — The flow's finished execution attempts, oldest first, including manual Wolf Browser attempts. Present once at least one attempt has finished, so a flow that passed after retries also lists its failed attempts. Artifact URLs appear only on automated attempts that reached a verdict, stay valid for at least a day (call run.get again for fresh ones), and can return 404 when the attempt did not produce that artifact.
|
|
156
|
+
- `flows[].attempts[].logsUrl` — Signed URL for the attempt's execution logs.
|
|
157
|
+
- `flows[].attempts[].traceUrl` — Signed URL for the attempt's Playwright trace (a trace.zip; open it with `npx playwright show-trace`).
|
|
158
|
+
- `flows[].attempts[].videoUrl` — Signed URL for the attempt's screen recording.
|
|
159
|
+
- `flows[].attempts[].kind` — One of: automated, manual
|
|
160
|
+
- `flows[].attempts[].startedAt` — Absent when the attempt failed before it could start.
|
|
161
|
+
- `flows[].attempts[].status` — One of: passed, failed, canceled
|
|
162
|
+
- `flows[].failure.diagnosis` — QA Wolf's investigation verdict for the failure: `bug` means the application is broken, `maintenance` means the test needed an update and the failure does not indicate an application problem. Absent until the investigation reaches a verdict. Pass issueId to issue.get for details.
|
|
163
|
+
- `flows[].failure.diagnosis.issueId` — The id of the issue.
|
|
164
|
+
- `flows[].failure.diagnosis.type` — One of: bug, maintenance
|
|
165
|
+
- `flows[].flowId` — The id of the flow.
|
|
166
|
+
- `flows[].status` — One of: failed, queued, running, passed, canceled
|
|
167
|
+
- `url` — Absolute URL of the run page.
|
|
168
|
+
|
|
169
|
+
<!-- fields:end -->
|
|
@@ -166,13 +166,23 @@ The call answers with a run id as soon as the run is accepted. **The outcome is
|
|
|
166
166
|
not in that answer**, it is in the `run-status` stream, whose entries carry
|
|
167
167
|
`runId`, `status` and an `errorMessage` when there is one.
|
|
168
168
|
|
|
169
|
-
**Pass `--follow` to `run` and let it wait for you.** It
|
|
170
|
-
|
|
169
|
+
**Pass `--follow` to `run` and let it wait for you.** It reports the run's
|
|
170
|
+
status — in progress, then passed or failed — and ends on the settled status.
|
|
171
|
+
Exit code `1` means the run did not pass. Three flags mirror more streams into
|
|
172
|
+
the follow, and each implies `--follow` on its own: `--logs` streams every log
|
|
173
|
+
line the run produces, `--run-events` streams the run's progress events as JSON
|
|
174
|
+
lines, and `--recorder-events` streams the browser actions the runner records
|
|
175
|
+
as JSON lines — the recorder is runner-wide rather than run-scoped, so that one
|
|
176
|
+
carries whatever is recorded after an anchor taken just before submission. Whatever mirrors are on, the
|
|
177
|
+
follow still ends on the status, never on them, so a run that prints nothing
|
|
171
178
|
still terminates the follow and a run that dies mid-sentence still reports how.
|
|
172
|
-
|
|
179
|
+
Combining mirror flags interleaves their lines with nothing saying which stream
|
|
180
|
+
a line came from — fine for eyeballs; when parsing, follow one stream at a time.
|
|
173
181
|
|
|
174
182
|
```sh
|
|
175
183
|
qawolf runner run flows/checkout.flow.ts --follow
|
|
184
|
+
qawolf runner run flows/checkout.flow.ts --follow --logs
|
|
185
|
+
qawolf runner run flows/checkout.flow.ts --follow --recorder-events
|
|
176
186
|
```
|
|
177
187
|
|
|
178
188
|
If you would rather submit and come back later, note that `--follow` on `events`
|