omniqa 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/dist/.tsbuildinfo +1 -1
- package/dist/auth.d.ts +17 -0
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +29 -1
- package/dist/auth.js.map +1 -1
- package/dist/cli.js +18 -5
- package/dist/cli.js.map +1 -1
- package/dist/commands/chat.js +1 -1
- package/dist/commands/chat.js.map +1 -1
- package/dist/commands/crawl.d.ts.map +1 -1
- package/dist/commands/crawl.js +17 -5
- package/dist/commands/crawl.js.map +1 -1
- package/dist/commands/doctor.d.ts +23 -0
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +152 -6
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/generate.js +1 -1
- package/dist/commands/generate.js.map +1 -1
- package/dist/commands/init.d.ts +3 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +217 -50
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/knowledge.d.ts +61 -1
- package/dist/commands/knowledge.d.ts.map +1 -1
- package/dist/commands/knowledge.js +232 -50
- package/dist/commands/knowledge.js.map +1 -1
- package/dist/commands/persist.d.ts +6 -0
- package/dist/commands/persist.d.ts.map +1 -1
- package/dist/commands/persist.js +9 -2
- package/dist/commands/persist.js.map +1 -1
- package/dist/commands/plan-add.d.ts.map +1 -1
- package/dist/commands/plan-add.js +13 -3
- package/dist/commands/plan-add.js.map +1 -1
- package/dist/commands/plan-review.d.ts +6 -2
- package/dist/commands/plan-review.d.ts.map +1 -1
- package/dist/commands/plan-review.js +16 -5
- package/dist/commands/plan-review.js.map +1 -1
- package/dist/commands/plan.d.ts +15 -0
- package/dist/commands/plan.d.ts.map +1 -1
- package/dist/commands/plan.js +103 -32
- package/dist/commands/plan.js.map +1 -1
- package/dist/commands/reindex.d.ts +1 -1
- package/dist/commands/reindex.d.ts.map +1 -1
- package/dist/commands/reindex.js +15 -1
- package/dist/commands/reindex.js.map +1 -1
- package/dist/commands/security.d.ts +4 -2
- package/dist/commands/security.d.ts.map +1 -1
- package/dist/commands/security.js +18 -7
- package/dist/commands/security.js.map +1 -1
- package/dist/commands/serve.d.ts +2 -0
- package/dist/commands/serve.d.ts.map +1 -1
- package/dist/commands/serve.js +126 -30
- package/dist/commands/serve.js.map +1 -1
- package/dist/commands/sync.d.ts.map +1 -1
- package/dist/commands/sync.js +19 -2
- package/dist/commands/sync.js.map +1 -1
- package/dist/commands/test.d.ts +5 -1
- package/dist/commands/test.d.ts.map +1 -1
- package/dist/commands/test.js +106 -6
- package/dist/commands/test.js.map +1 -1
- package/dist/commands/write.d.ts +8 -2
- package/dist/commands/write.d.ts.map +1 -1
- package/dist/commands/write.js +101 -18
- package/dist/commands/write.js.map +1 -1
- package/dist/devices.d.ts +3 -1
- package/dist/devices.d.ts.map +1 -1
- package/dist/devices.js +5 -3
- package/dist/devices.js.map +1 -1
- package/dist/fixer.d.ts.map +1 -1
- package/dist/fixer.js +19 -9
- package/dist/fixer.js.map +1 -1
- package/dist/jobs.d.ts +6 -0
- package/dist/jobs.d.ts.map +1 -1
- package/dist/jobs.js +10 -1
- package/dist/jobs.js.map +1 -1
- package/dist/mcp/server.d.ts +1 -1
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +1 -1
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/session.d.ts.map +1 -1
- package/dist/mcp/session.js +7 -7
- package/dist/mcp/session.js.map +1 -1
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +9 -6
- package/dist/mcp/tools.js.map +1 -1
- package/dist/page.d.ts +1 -1
- package/dist/page.d.ts.map +1 -1
- package/dist/page.js +11 -0
- package/dist/page.js.map +1 -1
- package/dist/planner.d.ts.map +1 -1
- package/dist/planner.js +8 -4
- package/dist/planner.js.map +1 -1
- package/dist/prompt.d.ts +7 -1
- package/dist/prompt.d.ts.map +1 -1
- package/dist/prompt.js +23 -4
- package/dist/prompt.js.map +1 -1
- package/dist/qa.d.ts +7 -0
- package/dist/qa.d.ts.map +1 -1
- package/dist/qa.js +34 -9
- package/dist/qa.js.map +1 -1
- package/dist/runner.d.ts +7 -2
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +28 -8
- package/dist/runner.js.map +1 -1
- package/dist/security-checks.d.ts +46 -4
- package/dist/security-checks.d.ts.map +1 -1
- package/dist/security-checks.js +114 -39
- package/dist/security-checks.js.map +1 -1
- package/dist/security-gates.d.ts +64 -7
- package/dist/security-gates.d.ts.map +1 -1
- package/dist/security-gates.js +125 -16
- package/dist/security-gates.js.map +1 -1
- package/dist/security-mask.d.ts +35 -0
- package/dist/security-mask.d.ts.map +1 -0
- package/dist/security-mask.js +104 -0
- package/dist/security-mask.js.map +1 -0
- package/dist/security-report.d.ts +93 -2
- package/dist/security-report.d.ts.map +1 -1
- package/dist/security-report.js +153 -14
- package/dist/security-report.js.map +1 -1
- package/dist/security-request.d.ts +27 -4
- package/dist/security-request.d.ts.map +1 -1
- package/dist/security-request.js +56 -14
- package/dist/security-request.js.map +1 -1
- package/dist/security-research.d.ts +132 -0
- package/dist/security-research.d.ts.map +1 -0
- package/dist/security-research.js +599 -0
- package/dist/security-research.js.map +1 -0
- package/dist/security-scan.d.ts +98 -0
- package/dist/security-scan.d.ts.map +1 -0
- package/dist/security-scan.js +463 -0
- package/dist/security-scan.js.map +1 -0
- package/dist/security.d.ts +18 -4
- package/dist/security.d.ts.map +1 -1
- package/dist/security.js +181 -37
- package/dist/security.js.map +1 -1
- package/dist/sync.d.ts +55 -3
- package/dist/sync.d.ts.map +1 -1
- package/dist/sync.js +403 -54
- package/dist/sync.js.map +1 -1
- package/dist/test.d.ts +26 -2
- package/dist/test.d.ts.map +1 -1
- package/dist/test.js +152 -29
- package/dist/test.js.map +1 -1
- package/dist/writer.d.ts +29 -10
- package/dist/writer.d.ts.map +1 -1
- package/dist/writer.js +116 -36
- package/dist/writer.js.map +1 -1
- package/package.json +7 -7
- package/skills/knowledge.md +1 -1
- package/skills/security-research.md +57 -0
- package/skills/security-review.md +27 -0
- package/skills/onboarding.md +0 -5
package/dist/writer.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { execFile } from "node:child_process";
|
|
2
2
|
import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
|
-
import { dirname, join, relative, resolve, sep } from "node:path";
|
|
3
|
+
import { basename, dirname, join, relative, resolve, sep } from "node:path";
|
|
4
4
|
import { requestApi } from "@omniqa/browser";
|
|
5
|
-
import { launchBrowser, pageApi, seedContext } from "@omniqa/web";
|
|
6
|
-
import { AUTH_FILE, AUTH_MARK, loginJson, parseSignUp, SIGNUP_MARK, signUpRefusal, signUpWith, tearDown } from "./auth.js";
|
|
7
|
-
import { actionMutates, BudgetExceeded, caseTags, describeAction, INDEX_FILE, KNOWLEDGE_DIR, matchPath, ModelError, openIndex, openRecording, parseEndpoints, paymentRequired, planPath, PRIORITIES, productSummary, pruneRecording, readCrawl, readInstructions, readPlan, readState, RECORDINGS_DIR, recordingPath, recordFirstRun, recordRun, recordStop, recordTests, redactor, renderHits, REVIEW_SCHEMA, runAgent, secretValues, stepKey, suiteFile, testTitle, tierEnvironment, USER_ADDED, withReview, writePlan } from "@omniqa/core";
|
|
5
|
+
import { contextOptionsFor, launchBrowser, pageApi, seedContext } from "@omniqa/web";
|
|
6
|
+
import { AUTH_FILE, AUTH_MARK, loginJson, parseSignUp, SIGNUP_MARK, signUpRefusal, signUpWith, sitePath, tearDown } from "./auth.js";
|
|
7
|
+
import { actionMutates, BudgetExceeded, caseTags, describeAction, INDEX_FILE, KNOWLEDGE_DIR, matchPath, ModelError, openIndex, openRecording, parseEndpoints, paymentRequired, planPath, PRIORITIES, productSummary, pruneRecording, readCrawl, readInstructions, readPlan, readState, RECORDINGS_DIR, recordingPath, recordFirstRun, recordRun, recordStop, recordTests, redactor, renderHits, REVIEW_SCHEMA, runAgent, secretValues, stepKey, suiteFile, testTitle, tierEnvironment, USER_ADDED, withReview, writePlan, productSecrets } from "@omniqa/core";
|
|
8
8
|
import { failedSpecs } from "./commands/generate.js";
|
|
9
9
|
import { hasKnowledge } from "./commands/reindex.js";
|
|
10
10
|
import { TITLE } from "./commands/status.js";
|
|
@@ -25,12 +25,24 @@ export function selectCases(plan, f) {
|
|
|
25
25
|
const picked = all.filter(({ suite, case: c }) => (wanted === undefined ? c.status === "planned" : wanted.includes(c.id)) && (f.suite === undefined || suite.slug === f.suite) && (f.priority === undefined || c.priority === f.priority) && (f.tier === undefined || c.tiers.includes(f.tier)) && (f.type === undefined || c.type === f.type));
|
|
26
26
|
return picked.map((s, i) => ({ s, i })).toSorted((a, b) => rank(a.s.case) - rank(b.s.case) || a.i - b.i).map(({ s }) => s);
|
|
27
27
|
}
|
|
28
|
+
/** Why `--case` selects less than it names: an id the plan does not have, or one the other filters leave out. Undefined when every named case is selected. */
|
|
29
|
+
export function unselected(plan, f) {
|
|
30
|
+
const wanted = f.case?.split(",").map((s) => s.trim()).filter((s) => s !== "") ?? [];
|
|
31
|
+
const known = new Set(plan.suites.flatMap((suite) => suite.cases.map((c) => c.id)));
|
|
32
|
+
const picked = new Set(selectCases(plan, f).map((s) => s.case.id));
|
|
33
|
+
const unknown = wanted.filter((id) => !known.has(id));
|
|
34
|
+
if (unknown.length > 0)
|
|
35
|
+
return `no case ${unknown.join(", ")} in the plan (omniqa plan review shows the ids)`;
|
|
36
|
+
const out = wanted.filter((id) => !picked.has(id));
|
|
37
|
+
return out.length === 0 ? undefined : `${out.join(", ")} ${out.length === 1 ? "is" : "are"} left out by the other flags (--suite, --priority, --tier, --type): name a case alone`;
|
|
38
|
+
}
|
|
28
39
|
export { suiteFile } from "@omniqa/core";
|
|
29
40
|
const q = (s) => JSON.stringify(s);
|
|
30
41
|
/** A substring as a regex literal: `/\/cart\?x=1/`. */
|
|
31
42
|
const regexOf = (s) => `/${s.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&")}/`;
|
|
32
43
|
const tagList = (c) => `[${caseTags(c).map(q).join(", ")}]`;
|
|
33
|
-
|
|
44
|
+
/** The reason above a test.fixme, a comment line each. It is a model's text and quotes a page or a commit: every character that ends a line of JavaScript ends one here, or what follows it would be code. */
|
|
45
|
+
const fixmeLines = (fixme) => (fixme === undefined ? [] : fixme.split(/[\n\r\u2028\u2029]/).map((l) => l.trim()).filter((l) => l !== "").map((l) => `// needs a human: ${l}`));
|
|
34
46
|
const head = (c, fixme, fixtures) => `test${fixme === undefined ? "" : ".fixme"}(${q(testTitle(c))}, { tag: ${tagList(c)} }, async ({ ${fixtures.join(", ")} }) => {`;
|
|
35
47
|
/** `captures`: a loginError step checks what the wrongPassword step returned, so that step keeps it in `loginError`. */
|
|
36
48
|
function stepLine(s, captures = false) {
|
|
@@ -99,10 +111,12 @@ const HEADERS = {
|
|
|
99
111
|
security: '// Written by omniqa write from tests/plan.md and security.md; edit the accepted lists freely.\nimport { test, expect } from "omniqa/test";\n',
|
|
100
112
|
};
|
|
101
113
|
const escapeRegex = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
114
|
+
/** The line that opens the block titled `<id>: …`. */
|
|
115
|
+
const opening = (id) => new RegExp(`^test(?:\\.fixme)?\\(\\s*["'\`]${escapeRegex(id)}: `);
|
|
102
116
|
/** The lines of the block titled `<id>: …`: from its `// needs a human` comments to the first `});` at column 0. */
|
|
103
117
|
export function findTest(source, id) {
|
|
104
118
|
const lines = source.split("\n");
|
|
105
|
-
const open =
|
|
119
|
+
const open = opening(id);
|
|
106
120
|
let start = lines.findIndex((l) => open.test(l));
|
|
107
121
|
if (start < 0)
|
|
108
122
|
return undefined;
|
|
@@ -114,6 +128,41 @@ export function findTest(source, id) {
|
|
|
114
128
|
end = lines.length - 1;
|
|
115
129
|
return { start, end };
|
|
116
130
|
}
|
|
131
|
+
/** The file without the block titled `<id>: …` and the blank line before it; undefined when no test is left in it. */
|
|
132
|
+
export function removeTest(source, id) {
|
|
133
|
+
const found = findTest(source, id);
|
|
134
|
+
const lines = source.split("\n");
|
|
135
|
+
if (found !== undefined) {
|
|
136
|
+
const from = found.start > 0 && lines[found.start - 1] === "" ? found.start - 1 : found.start;
|
|
137
|
+
lines.splice(from, found.end - from + 1);
|
|
138
|
+
}
|
|
139
|
+
return lines.some((line) => /^test(?:\.\w+)?\(/.test(line)) ? lines.join("\n") : undefined;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Why the block under a case's id is not the case's to replace or rewrite: it carries another title, so it is another case's
|
|
143
|
+
* test. Ids were handed out again by position until a re-plan kept them, so a plan revised by an older version can name one
|
|
144
|
+
* case where the file holds the reviewed test of another; a row renamed by hand reads the same. Titles are compared as a
|
|
145
|
+
* re-plan compares them, whatever their letter case. Undefined when the file holds no block under the id, or the case's own.
|
|
146
|
+
*/
|
|
147
|
+
export function otherCase(source, c, rel) {
|
|
148
|
+
const line = source?.split("\n").find((l) => opening(c.id).test(l));
|
|
149
|
+
const m = line === undefined ? null : new RegExp(TITLE.source).exec(line);
|
|
150
|
+
if (m === null)
|
|
151
|
+
return undefined;
|
|
152
|
+
// The title as the file spells it, read as JavaScript reads it: the writer's own (JSON), or one a hand or a linter quoted otherwise.
|
|
153
|
+
let title = m[2].replace(/\\(.)/g, "$1");
|
|
154
|
+
if (m[1] === '"') {
|
|
155
|
+
try {
|
|
156
|
+
title = JSON.parse(`"${m[2]}"`);
|
|
157
|
+
}
|
|
158
|
+
catch {
|
|
159
|
+
// a title only a hand could write: compared as it is spelled, its escapes dropped
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
if (title.trim().toLowerCase() === testTitle(c).trim().toLowerCase())
|
|
163
|
+
return undefined;
|
|
164
|
+
return `${rel} holds a test titled ${q(title)} under this case's id, and this case is ${q(testTitle(c))}: another case's test is not replaced. Give that test the id and title of its own row in tests/plan.md, or remove it when it has none, then write ${c.id} again`;
|
|
165
|
+
}
|
|
117
166
|
/** `block` replaces the block of the same id, or is appended; a new file starts with the header and the import; an api test that signs in gets the _auth import. */
|
|
118
167
|
export function upsertTest(source, id, block, kind) {
|
|
119
168
|
let text = source ?? HEADERS[kind];
|
|
@@ -213,27 +262,37 @@ export function loginPrelude(authMd, method, path) {
|
|
|
213
262
|
}
|
|
214
263
|
return [];
|
|
215
264
|
}
|
|
216
|
-
/**
|
|
265
|
+
/** A model's words inside a block comment: the two characters that end one are kept apart. */
|
|
266
|
+
const inComment = (s) => s.replaceAll("*/", "* /");
|
|
267
|
+
/** The same inside a template literal: a backslash, a backtick and `${` are written as the characters they are. */
|
|
268
|
+
const inTemplate = (s) => s.replace(/\\|`|\$\{/g, "\\$&");
|
|
269
|
+
/** JSON on a `//` line: JSON.stringify leaves U+2028 and U+2029 as they are, and each ends a line of JavaScript, so they are written as the escapes that read back the same. */
|
|
270
|
+
const jsonLine = (value) => JSON.stringify(value).replace(/[\u2028\u2029]/g, (c) => `\\u${c.charCodeAt(0).toString(16)}`);
|
|
271
|
+
/**
|
|
272
|
+
* `tests/api/_auth.ts`: the login helper (`apiLogIn`) and the sign-up helper (`apiSignUp`), each behind its spec line.
|
|
273
|
+
* A path, a method and a cookie name are a model's words (or a hand edit read back): each is written as data, in a string, or
|
|
274
|
+
* escaped for the comment, the template or the marker line it stands in, so nothing in one can become code of the helper.
|
|
275
|
+
*/
|
|
217
276
|
export function renderAuth(login, signUp) {
|
|
218
277
|
const imports = [...(login === undefined ? [] : ["accountFields"]), ...(signUp === undefined ? [] : ["signUpWith"])];
|
|
219
278
|
const loginLines = login === undefined ? [] : [
|
|
220
|
-
`/** Sign the request context in as a product.yaml account: ${login.method} ${login.path} with the account's fields${describeBefore(login.before)}${login.sessionCookie === "" ? "" : `; the session is the ${login.sessionCookie} cookie`}. */`,
|
|
279
|
+
`/** Sign the request context in as a product.yaml account: ${inComment(`${login.method} ${login.path} with the account's fields${describeBefore(login.before)}${login.sessionCookie === "" ? "" : `; the session is the ${login.sessionCookie} cookie`}`)}. */`,
|
|
221
280
|
"export async function apiLogIn(api: APIRequestContext, account: string): Promise<void> {",
|
|
222
281
|
" const fields = accountFields(account);",
|
|
223
282
|
...(login.before === undefined || login.before.length === 0 ? [] : [` for (const r of ${JSON.stringify(login.before)}) await api.fetch(r.path, { method: r.method });`]),
|
|
224
283
|
` const response = await api.fetch(${q(login.path)}, { method: ${q(login.method)}, ${login.body === "json" ? "data" : "form"}: { ${Object.entries(login.fields).map(([k, v]) => `${q(k)}: fields[${q(v)}]`).join(", ")} } });`,
|
|
225
|
-
` if (!response.ok()) throw new Error(\`apiLogIn(\${account}): ${login.method} ${login.path} answered HTTP \${response.status()}\`);`,
|
|
284
|
+
` if (!response.ok()) throw new Error(\`apiLogIn(\${account}): ${inTemplate(`${login.method} ${login.path}`)} answered HTTP \${response.status()}\`);`,
|
|
226
285
|
"}",
|
|
227
286
|
"",
|
|
228
287
|
];
|
|
229
288
|
const signUpLines = signUp === undefined ? [] : [
|
|
230
|
-
`/** Sign a throwaway account up through ${signUp.method} ${signUp.path} with a generated email and password (overrides win: an existing account's email for the taken case); the response comes back unjudged with the credentials. Refused where mutations are off, on either tier. */`,
|
|
289
|
+
`/** Sign a throwaway account up through ${inComment(`${signUp.method} ${signUp.path}`)} with a generated email and password (overrides win: an existing account's email for the taken case); the response comes back unjudged with the credentials. Refused where mutations are off, on either tier. */`,
|
|
231
290
|
`export const apiSignUp = (api: APIRequestContext, overrides: Record<string, string> = {}) => signUpWith(api, ${JSON.stringify(signUp)}, overrides);`,
|
|
232
291
|
"",
|
|
233
292
|
];
|
|
234
293
|
return [
|
|
235
|
-
...(login === undefined ? [] : [`${AUTH_MARK}${
|
|
236
|
-
...(signUp === undefined ? [] : [`${SIGNUP_MARK}${
|
|
294
|
+
...(login === undefined ? [] : [`${AUTH_MARK}${jsonLine(login)}`]),
|
|
295
|
+
...(signUp === undefined ? [] : [`${SIGNUP_MARK}${jsonLine(signUp)}`]),
|
|
237
296
|
"// Written by omniqa write from api/auth.md and crawl/forms.md: how an API test signs in or signs up. Edit freely; the first lines are what omniqa write reads back.",
|
|
238
297
|
'import type { APIRequestContext } from "omniqa/test";',
|
|
239
298
|
`import { ${imports.join(", ")} } from "omniqa/test";`,
|
|
@@ -276,12 +335,12 @@ function excerptOf(contentType, text, redact) {
|
|
|
276
335
|
/** A request context the way the `api` fixture builds it in each mode. */
|
|
277
336
|
async function openApi(baseUrl, o) {
|
|
278
337
|
if (o.mode !== "page") {
|
|
279
|
-
const api = requestApi(baseUrl);
|
|
338
|
+
const api = requestApi(baseUrl, o.seed?.httpAuth);
|
|
280
339
|
return { api, close: () => api.dispose() };
|
|
281
340
|
}
|
|
282
341
|
const browser = await launchBrowser();
|
|
283
342
|
try {
|
|
284
|
-
const context = await browser.newContext({ baseURL: baseUrl });
|
|
343
|
+
const context = await browser.newContext({ baseURL: baseUrl, ...contextOptionsFor(o.seed, baseUrl) });
|
|
285
344
|
if (o.seed !== undefined)
|
|
286
345
|
await seedContext(context, o.seed, baseUrl);
|
|
287
346
|
const page = await context.newPage();
|
|
@@ -395,11 +454,16 @@ export async function recordedStepLines(testDir, file, c, steps, project) {
|
|
|
395
454
|
return lines;
|
|
396
455
|
}
|
|
397
456
|
/** One omniqa test run of the given case ids (their titles start with `<id>: `) on a project and environment, with a JSON report; explore and heal are opt-in. */
|
|
457
|
+
/**
|
|
458
|
+
* The pattern that runs these cases and no other: each id whole, at the start of a test's title. The runner matches it against
|
|
459
|
+
* the project, the file, the title path and the tags with a space between, and an id found anywhere in that would run the case
|
|
460
|
+
* of a suite whose name ends with this one's too (`auth-1: ` is in `sec-auth-1: `, the wrong-password probes).
|
|
461
|
+
*/
|
|
462
|
+
export const caseGrep = (ids) => `(?:^| )(?:${ids.map((id) => escapeRegex(`${id}: `)).join("|")})`;
|
|
398
463
|
export async function runCase(cwd, env, o) {
|
|
399
464
|
const report = join(cwd, "test-results", `${o.reportName}.json`);
|
|
400
465
|
await rm(report, { force: true });
|
|
401
|
-
const
|
|
402
|
-
const code = await runSuite(["--retries=0", "--reporter=json", `--grep=${grep}`, `--project=${o.project}`, ...(o.files ?? [])], cwd, { ...env, OMNIQA_ENV: o.envName, ...(o.explore === true ? { OMNIQA_ALLOW_EXPLORE: o.envName } : {}), ...(o.heal === true ? { OMNIQA_HEAL: "1" } : {}), OMNIQA_JSON_OUTPUT_FILE: report }, (line) => o.log(`run: ${line}`));
|
|
466
|
+
const code = await runSuite(["--retries=0", "--reporter=json", "--pass-with-no-tests", `--grep=${caseGrep(o.ids)}`, `--project=${o.project}`, ...(o.files ?? [])], cwd, { ...env, OMNIQA_ENV: o.envName, ...(o.explore === true ? { OMNIQA_ALLOW_EXPLORE: o.envName } : {}), ...(o.heal === true ? { OMNIQA_HEAL: "1" } : {}), OMNIQA_JSON_OUTPUT_FILE: report }, (line) => o.log(`run: ${line}`));
|
|
403
467
|
try {
|
|
404
468
|
const json = JSON.parse(await readFile(report, "utf8"));
|
|
405
469
|
return { code, report: json, summary: summarizeRun(json, { at: new Date().toISOString(), environment: o.envName, exitCode: code }) };
|
|
@@ -470,6 +534,13 @@ const carries = (text, secrets) => secrets.some((v) => v.length >= 4 && text.inc
|
|
|
470
534
|
* reaches them), non-empty values, paths, a known account, no account value or secret (`secrets`: product.yaml's account fields
|
|
471
535
|
* and the env values the config names) in any value, at least one assertion.
|
|
472
536
|
*/
|
|
537
|
+
/** A path a model named for a request of the writer's own is a path of the environment: one that names another host would take the trial request there, with an account's fields. */
|
|
538
|
+
function onProduct(path, what = "path") {
|
|
539
|
+
if (!path.startsWith("/"))
|
|
540
|
+
throw new Error(`refused: ${what} must start with /, not ${JSON.stringify(path)}`);
|
|
541
|
+
if (!sitePath(path))
|
|
542
|
+
throw new Error(`refused: ${what} must stay on the environment's own host, and ${JSON.stringify(path)} names another`);
|
|
543
|
+
}
|
|
473
544
|
export function checkSteps(raw, accounts, secrets = []) {
|
|
474
545
|
const steps = raw;
|
|
475
546
|
if (!Array.isArray(steps) || steps.length === 0)
|
|
@@ -481,8 +552,8 @@ export function checkSteps(raw, accounts, secrets = []) {
|
|
|
481
552
|
// ponytail: a substring like the transcript's redactor; a word-boundary match if an account value that is a common word gets in the way.
|
|
482
553
|
if (s.kind !== "login" && s.kind !== "wrongPassword" && carries(s.value, secrets))
|
|
483
554
|
throw new Error(`refused: ${at}: it carries an account value or a secret (an account field of product.yaml or an env value the config names): describe it instead ("the header shows the signed-in member's email"), never quote it`);
|
|
484
|
-
if (s.kind === "goto" && !s.value
|
|
485
|
-
throw new Error(`refused: ${at}: goto takes a path starting with /, not ${JSON.stringify(s.value)}`);
|
|
555
|
+
if (s.kind === "goto" && !sitePath(s.value))
|
|
556
|
+
throw new Error(`refused: ${at}: goto takes a path of the product starting with /, not ${JSON.stringify(s.value)}`);
|
|
486
557
|
if (s.kind === "goto" && i > 0)
|
|
487
558
|
throw new Error(`refused: ${at}: goto opens the starting page only (the first step); reach ${JSON.stringify(s.value.trim())} the way a user does: a do step that clicks the link or button that leads there`);
|
|
488
559
|
if (s.kind === "expectUrl" && !URL_PATH.test(s.value.trim()))
|
|
@@ -507,7 +578,7 @@ async function restore(file, content) {
|
|
|
507
578
|
else
|
|
508
579
|
await writeFile(file, content);
|
|
509
580
|
}
|
|
510
|
-
/** The plan row of a case moves to `status`, with the test file it now has. */
|
|
581
|
+
/** The plan row of a case moves to `status`, with the test file it now has; without one, the row names no file. */
|
|
511
582
|
export async function setCaseStatus(planFile, id, status, file) {
|
|
512
583
|
const plan = await readPlan(planFile);
|
|
513
584
|
if (plan === undefined)
|
|
@@ -516,13 +587,19 @@ export async function setCaseStatus(planFile, id, status, file) {
|
|
|
516
587
|
if (c.id !== id)
|
|
517
588
|
continue;
|
|
518
589
|
c.status = status;
|
|
519
|
-
|
|
590
|
+
if (file === undefined)
|
|
591
|
+
delete c.file;
|
|
592
|
+
else
|
|
593
|
+
c.file = file;
|
|
520
594
|
}
|
|
521
595
|
await writePlan(planFile, plan);
|
|
522
596
|
}
|
|
523
|
-
/** Upsert the block into the file and check it parses; a block that does not parse is rolled back and refused. */
|
|
597
|
+
/** Upsert the block into the file and check it parses; a block that does not parse is rolled back and refused, and so is one that would replace another case's test. */
|
|
524
598
|
export async function writeBlock(c, file, block, kind) {
|
|
525
599
|
const previous = await readFile(file, "utf8").catch(() => undefined);
|
|
600
|
+
const taken = otherCase(previous, c, basename(file));
|
|
601
|
+
if (taken !== undefined)
|
|
602
|
+
throw new Error(`refused: ${taken}`);
|
|
526
603
|
await mkdir(dirname(file), { recursive: true });
|
|
527
604
|
await writeFile(file, upsertTest(previous, c.id, block, kind));
|
|
528
605
|
const problem = await checkSyntax(file);
|
|
@@ -724,7 +801,7 @@ export async function createWriter(o) {
|
|
|
724
801
|
return undefined;
|
|
725
802
|
}
|
|
726
803
|
/** What a test, its recording and its fixme comments never hold: the accounts' values and the env values the config names. */
|
|
727
|
-
const secrets = [...secretValues(config, o.env), ...
|
|
804
|
+
const secrets = [...secretValues(config, o.env), ...productSecrets(product)];
|
|
728
805
|
/** A request's path and body carry no account value or secret (they travel through account and fields, read at run time). */
|
|
729
806
|
const refuseSecrets = (what, p, b) => {
|
|
730
807
|
for (const [part, text] of [["path", p], ["body", b ?? ""]])
|
|
@@ -800,13 +877,11 @@ export async function createWriter(o) {
|
|
|
800
877
|
const accountOf = (c) => /account (\w+)/.exec(c.preconditions)?.[1] ?? Object.keys(product.accounts)[0];
|
|
801
878
|
async function writeAuth(args) {
|
|
802
879
|
const spec = { method: String(args.method).toUpperCase(), path: String(args.path), body: args.body, fields: args.fields, sessionCookie: typeof args.sessionCookie === "string" ? args.sessionCookie : "" };
|
|
803
|
-
|
|
804
|
-
throw new Error(`refused: path must start with /, not ${JSON.stringify(spec.path)}`);
|
|
880
|
+
onProduct(spec.path);
|
|
805
881
|
// The prelude: what the writer passed, else what api/auth.md's login flow shows between the page load and the login.
|
|
806
882
|
const before = Array.isArray(args.before) ? args.before.map((b) => ({ method: String(b.method).toUpperCase(), path: String(b.path) })) : loginPrelude(await readFile(join(knowledgeDir, "api", "auth.md"), "utf8").catch(() => ""), spec.method, spec.path);
|
|
807
883
|
for (const b of before)
|
|
808
|
-
|
|
809
|
-
throw new Error(`refused: before[].path must start with /, not ${JSON.stringify(b.path)}`);
|
|
884
|
+
onProduct(b.path, "before[].path");
|
|
810
885
|
if (before.length > 0)
|
|
811
886
|
spec.before = before;
|
|
812
887
|
const account = accountOf(current.case);
|
|
@@ -843,13 +918,12 @@ export async function createWriter(o) {
|
|
|
843
918
|
throw new Error(`refused: the helper does not parse: ${problem}`);
|
|
844
919
|
}
|
|
845
920
|
const where = () => ({ name: o.envName, environment: o.environment });
|
|
846
|
-
const accountValues =
|
|
921
|
+
const accountValues = productSecrets(product);
|
|
847
922
|
/** How the writer's own requests go out: the environment's api mode and seed, account values redacted from any excerpt. */
|
|
848
923
|
const capture = () => ({ mode: o.environment.api.mode, seed: o.environment, redact: redactor(accountValues) });
|
|
849
924
|
async function writeSignUp(args) {
|
|
850
925
|
const spec = { method: String(args.method).toUpperCase(), path: String(args.path), body: args.body, fields: args.fields };
|
|
851
|
-
|
|
852
|
-
throw new Error(`refused: path must start with /, not ${JSON.stringify(spec.path)}`);
|
|
926
|
+
onProduct(spec.path);
|
|
853
927
|
if (typeof spec.fields !== "object" || spec.fields === null || Object.values(spec.fields).some((v) => typeof v !== "string"))
|
|
854
928
|
throw new Error("refused: fields must map request fields to email, password, username or a literal");
|
|
855
929
|
const values = Object.values(spec.fields);
|
|
@@ -859,8 +933,7 @@ export async function createWriter(o) {
|
|
|
859
933
|
throw new Error("refused: fields must map a request field to password, generated per sign-up");
|
|
860
934
|
const before = Array.isArray(args.before) ? args.before.map((b) => ({ method: String(b.method).toUpperCase(), path: String(b.path) })) : loginPrelude(await readFile(join(knowledgeDir, "api", "auth.md"), "utf8").catch(() => ""), spec.method, spec.path);
|
|
861
935
|
for (const b of before)
|
|
862
|
-
|
|
863
|
-
throw new Error(`refused: before[].path must start with /, not ${JSON.stringify(b.path)}`);
|
|
936
|
+
onProduct(b.path, "before[].path");
|
|
864
937
|
if (before.length > 0)
|
|
865
938
|
spec.before = before;
|
|
866
939
|
if (typeof args.emailDomain === "string" && args.emailDomain.trim() !== "")
|
|
@@ -870,6 +943,8 @@ export async function createWriter(o) {
|
|
|
870
943
|
const teardown = t === undefined ? selfDeletion(`${await readFile(join(knowledgeDir, "api", "endpoints.md"), "utf8").catch(() => "")}\n${await readFile(join(knowledgeDir, "xref.md"), "utf8").catch(() => "")}`, spec.path) : { method: String(t.method ?? "DELETE").toUpperCase(), path: String(t.path), idKey: typeof t.idKey === "string" && t.idKey !== "" ? t.idKey : "id", fields: (t.fields ?? { password: "password" }) };
|
|
871
944
|
if (teardown !== undefined && (!teardown.path.startsWith("/") || !teardown.path.includes("{id}")))
|
|
872
945
|
throw new Error(`refused: teardown.path must start with / and carry {id}, not ${JSON.stringify(teardown.path)}`);
|
|
946
|
+
if (teardown !== undefined)
|
|
947
|
+
onProduct(teardown.path, "teardown.path");
|
|
873
948
|
if (teardown?.fields !== undefined && (typeof teardown.fields !== "object" || Object.values(teardown.fields).some((v) => typeof v !== "string")))
|
|
874
949
|
throw new Error("refused: teardown.fields must map request fields to email, password, username or a literal");
|
|
875
950
|
if (teardown !== undefined)
|
|
@@ -911,8 +986,7 @@ export async function createWriter(o) {
|
|
|
911
986
|
if (c.tiers.includes("production") && !productionWritable)
|
|
912
987
|
throw new Error(`refused: ${m} ${p} mutates and the case runs on the read-only production tier`);
|
|
913
988
|
};
|
|
914
|
-
|
|
915
|
-
throw new Error(`refused: path must start with /, not ${JSON.stringify(path)}`);
|
|
989
|
+
onProduct(path);
|
|
916
990
|
refuseSecrets("", path, body);
|
|
917
991
|
if (actor !== "guest" && product.accounts[actor] === undefined)
|
|
918
992
|
throw new Error(`refused: actor must be guest or an account of product.yaml (${Object.keys(product.accounts).join(", ") || "none"}), not ${JSON.stringify(actor)}`);
|
|
@@ -950,8 +1024,7 @@ export async function createWriter(o) {
|
|
|
950
1024
|
const r = { method: String(b.method).toUpperCase(), path: String(b.path) };
|
|
951
1025
|
if (typeof b.body === "string" && b.body.trim() !== "")
|
|
952
1026
|
r.body = b.body.trim();
|
|
953
|
-
|
|
954
|
-
throw new Error(`refused: ${at}.path must start with /, not ${JSON.stringify(r.path)}`);
|
|
1027
|
+
onProduct(r.path, `${at}.path`);
|
|
955
1028
|
refuseSecrets(`${at} `, r.path, r.body);
|
|
956
1029
|
refuseBody(`${at}.body`, r.body);
|
|
957
1030
|
if (b.fields !== undefined) {
|
|
@@ -1148,7 +1221,7 @@ export async function createWriter(o) {
|
|
|
1148
1221
|
const report = join(cwd, "test-results", `omniqa-write-${c.id}.json`);
|
|
1149
1222
|
await rm(report, { force: true });
|
|
1150
1223
|
const t0 = Date.now();
|
|
1151
|
-
const code = await runSuite(["--retries=0", "--reporter=json", `--grep=${
|
|
1224
|
+
const code = await runSuite(["--retries=0", "--reporter=json", "--pass-with-no-tests", `--grep=${caseGrep([c.id])}`, `--project=${project}`, d.rel], cwd, { ...o.env, OMNIQA_ENV: o.envName, OMNIQA_ALLOW_EXPLORE: o.envName, OMNIQA_JSON_OUTPUT_FILE: report }, (line) => progress(`first run: ${line}`));
|
|
1152
1225
|
let json;
|
|
1153
1226
|
try {
|
|
1154
1227
|
json = JSON.parse(await readFile(report, "utf8"));
|
|
@@ -1214,6 +1287,13 @@ export async function createWriter(o) {
|
|
|
1214
1287
|
const untouched = { test: await readFile(own, "utf8").catch(() => undefined), recording: await readFile(sidecar, "utf8").catch(() => undefined) };
|
|
1215
1288
|
const existing = findTest(untouched.test ?? "", c.id) === undefined ? undefined : relative(cwd, own);
|
|
1216
1289
|
const result = (status, extra = {}) => ({ id: c.id, status, rounds: round, calls: by.writer + by.session + by.run + by.reviewer, by, ms: Date.now() - t0, ...extra });
|
|
1290
|
+
// Another case's test under this id: nothing is asked of a model for a test that could not be written over it.
|
|
1291
|
+
const taken = otherCase(untouched.test, c, relative(cwd, own));
|
|
1292
|
+
if (taken !== undefined) {
|
|
1293
|
+
await recordTests(cwd, { case: { id: c.id, status: "failed", reason: taken } });
|
|
1294
|
+
out(`${c.id}: failed: ${taken}`);
|
|
1295
|
+
return result("failed", { feedback: taken });
|
|
1296
|
+
}
|
|
1217
1297
|
/** The case ends for a human: the draft becomes a test.fixme under the reason (it may quote what a run showed and is committed with the test, so it is redacted); the plan row and the state say needs-human. */
|
|
1218
1298
|
const forHuman = async (d, reason) => {
|
|
1219
1299
|
const feedback = hide(reason);
|