omniqa 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli.js +1 -1
- package/dist/commands/init.d.ts +8 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +144 -14
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/knowledge.d.ts.map +1 -1
- package/dist/commands/knowledge.js +3 -2
- package/dist/commands/knowledge.js.map +1 -1
- package/dist/commands/plan.d.ts.map +1 -1
- package/dist/commands/plan.js +3 -2
- package/dist/commands/plan.js.map +1 -1
- package/dist/commands/status.js +2 -2
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/sync.d.ts.map +1 -1
- package/dist/commands/sync.js +26 -14
- package/dist/commands/sync.js.map +1 -1
- package/dist/fixer.d.ts +1 -1
- package/dist/fixer.d.ts.map +1 -1
- package/dist/qa.js +3 -3
- package/dist/qa.js.map +1 -1
- package/dist/sync.d.ts +36 -9
- package/dist/sync.d.ts.map +1 -1
- package/dist/sync.js +185 -75
- package/dist/sync.js.map +1 -1
- package/package.json +6 -6
package/dist/sync.d.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { type Budgeted, type ChangeEvent, type ChangeSet, type ChangeSource, type Environment, type Impact, type OmniqaConfig, type PlanCase, type Product, type SyncSummary, type Transcript } from "@omniqa/core";
|
|
1
|
+
import { type Budgeted, type ChangeEvent, type ChangeSet, type ChangeSource, type Environment, type Impact, type OmniqaConfig, type PlanCase, type Product, type State, type SyncSummary, type Transcript } from "@omniqa/core";
|
|
2
2
|
import { type JsonReport } from "./commands/test.ts";
|
|
3
|
+
import { type Selected } from "./writer.ts";
|
|
3
4
|
export declare const DEFAULT_SYNC_BUDGET = 150;
|
|
4
5
|
/** Pages re-crawled per sync at most, the exact ones first. */
|
|
5
6
|
export declare const MAX_TARGET_PAGES = 20;
|
|
6
7
|
export interface SyncOptions {
|
|
7
|
-
/** The environment
|
|
8
|
+
/** The environment the deploy landed on: the suite runs there (`suite`), the impacted pages are re-crawled there, the production-only cases run there when it is a production tier (default: each tier's first exploring environment, else its first). */
|
|
8
9
|
env?: string;
|
|
9
10
|
budget?: number;
|
|
10
11
|
/** Build the change set and the impact, then stop. */
|
|
@@ -12,6 +13,8 @@ export interface SyncOptions {
|
|
|
12
13
|
/** Commit or open a pull request for what the sync changed (default: on in CI). */
|
|
13
14
|
persist?: boolean;
|
|
14
15
|
scheduled?: boolean;
|
|
16
|
+
/** Run every written case of --env's tier on it before the maintenance, instead of the impacted cases: what a deploy job asks. Needs env. */
|
|
17
|
+
suite?: boolean;
|
|
15
18
|
/** Lines for the user (stdout). */
|
|
16
19
|
out: (line: string) => void;
|
|
17
20
|
/** Progress lines (stderr). */
|
|
@@ -21,7 +24,7 @@ export interface SyncOptions {
|
|
|
21
24
|
export type SyncRequest = {
|
|
22
25
|
kind: "manual";
|
|
23
26
|
repo?: string;
|
|
24
|
-
from
|
|
27
|
+
from?: string;
|
|
25
28
|
to?: string;
|
|
26
29
|
} | {
|
|
27
30
|
kind: "poll";
|
|
@@ -56,10 +59,12 @@ export declare function devEnvironment(product: Product, name?: string): {
|
|
|
56
59
|
environment: Environment;
|
|
57
60
|
} | undefined;
|
|
58
61
|
export declare function syncContext(cwd: string, env: NodeJS.ProcessEnv, o: Pick<SyncOptions, "out" | "log" | "now">): Promise<SyncContext>;
|
|
59
|
-
/** The git host that answers for the product repository: the configured changelog host, else the one gitHosts names. */
|
|
62
|
+
/** The git host that answers for the product repository: the configured changelog host, else the one gitHosts names; it reads with changelog.tokenEnv when that is set (a token for a private product repository), else the host's own token. */
|
|
60
63
|
export declare function changeSource(ctx: SyncContext, host?: ChangeEvent["host"]): ChangeSource & {
|
|
61
64
|
name: string;
|
|
62
65
|
};
|
|
66
|
+
/** Where a sync without --from starts: the last synced commit of the repository, else the commit its knowledge was built from. */
|
|
67
|
+
export declare const lastSynced: (state: State, name: string) => string | undefined;
|
|
63
68
|
/** The change sets a request asks for; `--poll` and the first poll are explained on `out`. */
|
|
64
69
|
export declare function resolveChangeSets(ctx: SyncContext, request: SyncRequest, dryRun?: boolean): Promise<Resolved[]>;
|
|
65
70
|
/** The change set as the output shows it: `sync: ` before the first line. */
|
|
@@ -86,6 +91,8 @@ interface SyncRun {
|
|
|
86
91
|
};
|
|
87
92
|
/** Set once a case ran on production: the sync then persists through a pull request (D5). */
|
|
88
93
|
ranOnProduction?: boolean;
|
|
94
|
+
/** The runs' model calls: counted in the total, never charged to the budget (M5). */
|
|
95
|
+
runCalls: number;
|
|
89
96
|
}
|
|
90
97
|
export interface KnowledgeUpdate {
|
|
91
98
|
/** Repo maps rewritten. */
|
|
@@ -102,8 +109,8 @@ export interface KnowledgeUpdate {
|
|
|
102
109
|
commit: string;
|
|
103
110
|
stamp: string;
|
|
104
111
|
}
|
|
105
|
-
/** The impacted knowledge rewritten: the changed repository's map (its overview kept), a crawl of the impacted pages as every actor with their endpoint rows merged, the cross-reference, the scheme's impacted statements revised by the analyst behind the reviewer, the index and the knowledge commit. */
|
|
106
|
-
export declare function updateKnowledge(ctx: SyncContext, r: Resolved, impact: Impact, run: SyncRun): Promise<KnowledgeUpdate>;
|
|
112
|
+
/** The impacted knowledge rewritten: the changed repository's map (its overview kept), a crawl of the impacted pages as every actor with their endpoint rows merged, on `preferred` (the --env environment, M6) when it is given and explores, the cross-reference, the scheme's impacted statements revised by the analyst behind the reviewer, the index and the knowledge commit. */
|
|
113
|
+
export declare function updateKnowledge(ctx: SyncContext, r: Resolved, impact: Impact, run: SyncRun, preferred?: Where): Promise<KnowledgeUpdate>;
|
|
107
114
|
export type Outcome = "pass" | "healed" | "behaviour-changed" | "unsure" | "failed" | "fixed" | "regression" | "needs-human" | "skipped" | "not-run";
|
|
108
115
|
export interface CaseResult {
|
|
109
116
|
id: string;
|
|
@@ -119,8 +126,24 @@ export declare function classifyRun(report: JsonReport): Map<string, {
|
|
|
119
126
|
detail?: string;
|
|
120
127
|
calls: number;
|
|
121
128
|
}>;
|
|
122
|
-
|
|
123
|
-
|
|
129
|
+
type Where = {
|
|
130
|
+
name: string;
|
|
131
|
+
environment: Environment;
|
|
132
|
+
};
|
|
133
|
+
/** A case the run left failed, behaviour changed or unsure, where it ran and the run's error line: the fixer's input. */
|
|
134
|
+
interface Failed {
|
|
135
|
+
selected: Selected;
|
|
136
|
+
detail: string;
|
|
137
|
+
where: Where;
|
|
138
|
+
}
|
|
139
|
+
/** Pass 1: the selected cases run, one omniqa test run per environment; each ends pass, healed, behaviour changed, unsure, failed or not run. Resolves to the failures for the fixer. The runs' model calls are counted, not charged (M5). */
|
|
140
|
+
export declare function runCases(ctx: SyncContext, impact: Impact, run: SyncRun, suite: Where | undefined, into: {
|
|
141
|
+
cases: CaseResult[];
|
|
142
|
+
}): Promise<Failed[]>;
|
|
143
|
+
/** Pass 5: every failed case goes to the fixer on the environment it failed on, with the change set and the rewritten knowledge (M4); a production environment that does not explore gets no fixer (D5), a case on a route the change removed waits for the plan pass. */
|
|
144
|
+
export declare function fixFailed(ctx: SyncContext, r: Resolved, impact: Impact, k: KnowledgeUpdate, run: SyncRun, failed: Failed[], cases: CaseResult[]): Promise<void>;
|
|
145
|
+
/** Pass 6: cases on routes the change removed are proposed as skipped; added routes and handlers become planned cases through one planner call. */
|
|
146
|
+
export declare function updatePlan(ctx: SyncContext, k: KnowledgeUpdate, run: SyncRun, into: {
|
|
124
147
|
cases: CaseResult[];
|
|
125
148
|
planned: PlanCase[];
|
|
126
149
|
}): Promise<void>;
|
|
@@ -142,7 +165,11 @@ export interface SyncOutcome {
|
|
|
142
165
|
export declare function renderSyncReport(o: SyncOutcome, lines: string[], persisted?: string): string;
|
|
143
166
|
/** `tests: ran 3: passed 1, healed 1, fixed 1, regressions 0, needs a human 0, failed 0; skipped 1, planned 1; 7 model calls` */
|
|
144
167
|
export declare function testsLine(o: SyncOutcome): string;
|
|
145
|
-
/** The
|
|
168
|
+
/** The first notification: which tests failed where, after which deploy, and whether the healer runs. */
|
|
169
|
+
export declare function failuresText(changeSet: ChangeSet, failed: Failed[], ran: number, healer: string): string;
|
|
170
|
+
/** The second notification: what the healer made of the failures, what the knowledge update did, where it all went. */
|
|
171
|
+
export declare function outcomeText(o: SyncOutcome, failed: Failed[]): string;
|
|
172
|
+
/** The pipeline for one request: resolve the change sets; for each: the run, the impact, the knowledge update, the fixer, the plan and the persistence, in that order. */
|
|
146
173
|
export declare function runSync(cwd: string, env: NodeJS.ProcessEnv, request: SyncRequest, o: SyncOptions): Promise<SyncOutcome[]>;
|
|
147
174
|
export {};
|
|
148
175
|
//# sourceMappingURL=sync.d.ts.map
|
package/dist/sync.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../src/sync.ts"],"names":[],"mappings":"AAIA,OAAO,
|
|
1
|
+
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../src/sync.ts"],"names":[],"mappings":"AAIA,OAAO,EAA8qB,KAAK,QAAQ,EAAE,KAAK,WAAW,EAAE,KAAK,SAAS,EAAE,KAAK,YAAY,EAAE,KAAK,WAAW,EAAE,KAAK,MAAM,EAAwB,KAAK,YAAY,EAAa,KAAK,QAAQ,EAAE,KAAK,OAAO,EAAqD,KAAK,KAAK,EAAE,KAAK,WAAW,EAAE,KAAK,UAAU,EAAE,MAAM,cAAc,CAAC;AASh+B,OAAO,EAAkB,KAAK,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAIrE,OAAO,EAAW,KAAK,QAAQ,EAAE,MAAM,aAAa,CAAC;AAErD,eAAO,MAAM,mBAAmB,MAAM,CAAC;AACvC,+DAA+D;AAC/D,eAAO,MAAM,gBAAgB,KAAK,CAAC;AAEnC,MAAM,WAAW,WAAW;IAC1B,0PAA0P;IAC1P,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,sDAAsD;IACtD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,mFAAmF;IACnF,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,6IAA6I;IAC7I,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,mCAAmC;IACnC,GAAG,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5B,+BAA+B;IAC/B,GAAG,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5B,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB;AAED,MAAM,MAAM,WAAW,GACnB;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,EAAE,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7D;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,WAAW,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5D,6HAA6H;AAC7H,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,SAAS,CAAC;IACrB,iFAAiF;IACjF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC;IACvB,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,OAAO,CAAC;IACjB,qCAAqC;IACrC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,GAAG,EAAE,MAAM,IAAI,CAAC;IAChB,GAAG,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5B,GAAG,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7B;AASD,wKAAwK;AACxK,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,WAAW,CAAA;CAAE,GAAG,SAAS,CAEtH;AAED,wBAAsB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAIxI;AAED,iPAAiP;AACjP,wBAAgB,YAAY,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,YAAY,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAM1G;AAOD,kIAAkI;AAClI,eAAO,MAAM,UAAU,GAAI,OAAO,KAAK,EAAE,MAAM,MAAM,KAAG,MAAM,GAAG,SAA8E,CAAC;AA0BhJ,8FAA8F;AAC9F,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,UAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAkEnH;AAED,6EAA6E;AAC7E,wBAAgB,aAAa,CAAC,CAAC,EAAE,SAAS,GAAG,MAAM,EAAE,CAGpD;AAED,uGAAuG;AACvG,wBAAsB,aAAa,CAAC,GAAG,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAWrH;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAM9E;AAED,6HAA6H;AAC7H,UAAU,OAAO;IACf,MAAM,EAAE,QAAQ,CAAC;IACjB,UAAU,EAAE,UAAU,CAAC;IACvB,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;IACjC,GAAG,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,WAAW,CAAA;KAAE,CAAC;IACjD,2CAA2C;IAC3C,UAAU,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,WAAW,CAAA;KAAE,CAAC;IACxD,6FAA6F;IAC7F,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,qFAAqF;IACrF,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,2BAA2B;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,sCAAsC;IACtC,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC3C,iGAAiG;IACjG,SAAS,EAAE;QAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QAAC,SAAS,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IACrD,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,wXAAwX;AACxX,wBAAsB,eAAe,CAAC,GAAG,EAAE,WAAW,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC,eAAe,CAAC,CAkK9I;AAED,MAAM,MAAM,OAAO,GAAG,MAAM,GAAG,QAAQ,GAAG,mBAAmB,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,YAAY,GAAG,aAAa,GAAG,SAAS,GAAG,SAAS,CAAC;AAErJ,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,mFAAmF;IACnF,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAGD,oLAAoL;AACpL,wBAAgB,WAAW,CAAC,MAAM,EAAE,UAAU,GAAG,GAAG,CAAC,MAAM,EAAE;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAqBjH;AAED,KAAK,KAAK,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,WAAW,CAAA;CAAE,CAAC;AAGxD,yHAAyH;AACzH,UAAU,MAAM;IACd,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,KAAK,CAAC;CACd;AAuCD,8OAA8O;AAC9O,wBAAsB,QAAQ,CAAC,GAAG,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE;IAAE,KAAK,EAAE,UAAU,EAAE,CAAA;CAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAuCzJ;AAED,0QAA0Q;AAC1Q,wBAAsB,SAAS,CAAC,GAAG,EAAE,WAAW,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,eAAe,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA+BrK;AAED,mJAAmJ;AACnJ,wBAAsB,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,CAAC,EAAE,eAAe,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE;IAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IAAC,OAAO,EAAE,QAAQ,EAAE,CAAA;CAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAiCtJ;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,SAAS,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,OAAO,EAAE,QAAQ,EAAE,CAAC;IACpB,sDAAsD;IACtD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,iDAAiD;IACjD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAMD,4NAA4N;AAC5N,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAqB5F;AAED,iIAAiI;AACjI,wBAAgB,SAAS,CAAC,CAAC,EAAE,WAAW,GAAG,MAAM,CAIhD;AAID,yGAAyG;AACzG,wBAAgB,YAAY,CAAC,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAMxG;AAED,uHAAuH;AACvH,wBAAgB,WAAW,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CASpE;AA4BD,0KAA0K;AAC1K,wBAAsB,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAyF/H"}
|
package/dist/sync.js
CHANGED
|
@@ -2,7 +2,7 @@ import { existsSync } from "node:fs";
|
|
|
2
2
|
import { copyFile, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { join, relative, resolve } from "node:path";
|
|
5
|
-
import { budgeted, BudgetExceeded, changeSetLines, createModels, describeChangeSet, git, ingest, KNOWLEDGE_DIR, knowledgeCommit, localChangeSet, loadConfig, loadProduct, mapImpact, mergeEndpoints, openTranscript, parseEndpoints, parseStaticSections, planPath, readCrawl, readPlan, readState, README_FILE, readmeHasBlock, recordKnowledge, recordLastSeen, recordRun, recordSync, redactor, updateSync, removePendingChange, renderImpact, renderReview, repoDirsOf, repoNameOf, resolveEnvironmentUrl, RUNS_DIR, schemeCounts, secretValues, stampKnowledge, stampMaps, STATE_FILE, tierEnvironment, transcriptName, withReview, writeCoverageFile, writePlan } from "@omniqa/core";
|
|
5
|
+
import { budgeted, BudgetExceeded, changelogGitEnv, changeSetLines, createModels, describeChangeSet, git, ingest, KNOWLEDGE_DIR, knowledgeCommit, localChangeSet, loadConfig, loadProduct, mapImpact, mergeEndpoints, openTranscript, parseEndpoints, parseStaticSections, planPath, postText, readCrawl, readPlan, readState, README_FILE, readmeHasBlock, recordKnowledge, recordLastSeen, recordRun, recordSync, redactor, updateSync, removePendingChange, renderImpact, renderReview, repoDirsOf, repoNameOf, resolveEnvironmentUrl, RUNS_DIR, selectEnvironment, schemeCounts, secretValues, stampKnowledge, stampMaps, STATE_FILE, tierEnvironment, transcriptName, withReview, writeCoverageFile, writePlan } from "@omniqa/core";
|
|
6
6
|
import { crawlSite, createNetMap, launchBrowser, writeNetMap } from "@omniqa/web";
|
|
7
7
|
import { createGitHub } from "@omniqa/github";
|
|
8
8
|
import { createGitLab } from "@omniqa/gitlab";
|
|
@@ -33,12 +33,19 @@ export async function syncContext(cwd, env, o) {
|
|
|
33
33
|
const product = await loadProduct(cwd, env);
|
|
34
34
|
return { cwd, env, config, product, repoDirs: repoDirsOf(cwd, product), now: o.now ?? (() => new Date()), out: o.out, log: o.log };
|
|
35
35
|
}
|
|
36
|
-
/** The git host that answers for the product repository: the configured changelog host, else the one gitHosts names. */
|
|
36
|
+
/** The git host that answers for the product repository: the configured changelog host, else the one gitHosts names; it reads with changelog.tokenEnv when that is set (a token for a private product repository), else the host's own token. */
|
|
37
37
|
export function changeSource(ctx, host) {
|
|
38
38
|
const which = host ?? ctx.config.changelog?.host ?? (ctx.config.gitHosts.gitlab !== undefined && ctx.config.gitHosts.github === undefined ? "gitlab" : "github");
|
|
39
|
-
|
|
39
|
+
// the product token belongs to the changelog host only: a webhook from the other host keeps that host's own token
|
|
40
|
+
const tokenEnv = which === ctx.config.changelog?.host ? ctx.config.changelog.tokenEnv : undefined;
|
|
41
|
+
const withToken = (raw) => (tokenEnv === undefined ? raw : { ...raw, tokenEnv });
|
|
42
|
+
return which === "gitlab" ? createGitLab(withToken(ctx.config.gitHosts.gitlab ?? {}), ctx.env) : createGitHub(withToken(ctx.config.gitHosts.github ?? {}), ctx.env);
|
|
40
43
|
}
|
|
41
44
|
const knows = async (repoDir, sha) => git(repoDir, ["cat-file", "-e", `${sha}^{commit}`]).then(() => true, () => false);
|
|
45
|
+
/** The commit of `name` the knowledge base describes: `state.knowledge.commit` is `name sha[, name sha]` (knowledgeCommit). */
|
|
46
|
+
const knowledgeCommitOf = (state, name) => state.knowledge?.commit?.split(", ").map((part) => part.split(" ")).find(([n]) => n === name)?.[1];
|
|
47
|
+
/** Where a sync without --from starts: the last synced commit of the repository, else the commit its knowledge was built from. */
|
|
48
|
+
export const lastSynced = (state, name) => state.changelog?.lastSeen[name] ?? knowledgeCommitOf(state, name);
|
|
42
49
|
/** The directory of the repository named by a host path (`o/r`) or a name, when product.yaml has it. */
|
|
43
50
|
const localDir = (ctx, repo) => ctx.repoDirs[repoNameOf(repo)];
|
|
44
51
|
/** A change set for a webhook event: from the repository on disk when it has both commits, else from the host's compare. */
|
|
@@ -70,10 +77,15 @@ export async function resolveChangeSets(ctx, request, dryRun = false) {
|
|
|
70
77
|
let repo = request.repo ?? (names.length === 1 ? names[0] : configured);
|
|
71
78
|
if (repo === undefined)
|
|
72
79
|
throw new Error(names.length === 0 ? "no repository to compare: add repos: to product.yaml or changelog.repo to omniqa.config.ts" : `name the repository with --repo (${names.join(", ")})`);
|
|
80
|
+
const name = repoNameOf(repo);
|
|
81
|
+
// ponytail: a lastSeen ahead of --to (a poll recorded the head after a deploy) compares backwards; check ancestry with merge-base when it bites.
|
|
82
|
+
const from = request.from ?? lastSynced(await readState(ctx.cwd), name);
|
|
83
|
+
if (from === undefined)
|
|
84
|
+
throw new Error(`no commit to compare from for ${name}: nothing synced yet and no knowledge built from it; pass --from <sha>`);
|
|
73
85
|
const dir = localDir(ctx, repo);
|
|
74
86
|
if (dir !== undefined) {
|
|
75
87
|
const to = request.to ?? (await git(dir, ["rev-parse", "HEAD"]));
|
|
76
|
-
return [{ changeSet: await localChangeSet(dir,
|
|
88
|
+
return [{ changeSet: await localChangeSet(dir, name, from, to), repoDir: dir }];
|
|
77
89
|
}
|
|
78
90
|
if (configured !== undefined && (repo === configured || repoNameOf(configured) === repo))
|
|
79
91
|
repo = configured;
|
|
@@ -81,8 +93,8 @@ export async function resolveChangeSets(ctx, request, dryRun = false) {
|
|
|
81
93
|
throw new Error(`unknown repository ${JSON.stringify(repo)}: not a repos: path of product.yaml (${names.join(", ") || "none"}) nor changelog.repo (${configured ?? "none"})`);
|
|
82
94
|
const source = changeSource(ctx);
|
|
83
95
|
const to = request.to ?? (await source.branchHead(repo, ctx.config.changelog?.branch ?? "main")).sha;
|
|
84
|
-
const c = await source.compare(repo,
|
|
85
|
-
return [{ changeSet: { host: ctx.config.changelog?.host ?? source.name, repo, kind: "manual", before:
|
|
96
|
+
const c = await source.compare(repo, from, to);
|
|
97
|
+
return [{ changeSet: { host: ctx.config.changelog?.host ?? source.name, repo, kind: "manual", before: from, after: to, files: c.files, commits: c.commits, ...(c.url === undefined ? {} : { url: c.url }), ...(c.commits.at(-1) === undefined ? {} : { title: c.commits.at(-1).title }) } }];
|
|
86
98
|
}
|
|
87
99
|
if (request.kind === "poll") {
|
|
88
100
|
const changelog = ctx.config.changelog;
|
|
@@ -159,8 +171,8 @@ export function impactLine(impact, file, cwd) {
|
|
|
159
171
|
...(impact.cases.length === 0 ? [] : [` cases: ${impact.cases.map((c) => `${c.id} (${c.confidence})`).join(", ")}; exact ${count("exact")}, pattern ${count("pattern")}, guess ${count("guess")}`]),
|
|
160
172
|
];
|
|
161
173
|
}
|
|
162
|
-
/** The impacted knowledge rewritten: the changed repository's map (its overview kept), a crawl of the impacted pages as every actor with their endpoint rows merged, the cross-reference, the scheme's impacted statements revised by the analyst behind the reviewer, the index and the knowledge commit. */
|
|
163
|
-
export async function updateKnowledge(ctx, r, impact, run) {
|
|
174
|
+
/** The impacted knowledge rewritten: the changed repository's map (its overview kept), a crawl of the impacted pages as every actor with their endpoint rows merged, on `preferred` (the --env environment, M6) when it is given and explores, the cross-reference, the scheme's impacted statements revised by the analyst behind the reviewer, the index and the knowledge commit. */
|
|
175
|
+
export async function updateKnowledge(ctx, r, impact, run, preferred) {
|
|
164
176
|
const knowledgeDir = join(ctx.cwd, KNOWLEDGE_DIR);
|
|
165
177
|
const name = repoNameOf(r.changeSet.repo);
|
|
166
178
|
const mapFile = join(knowledgeDir, "repos", name, "map.md");
|
|
@@ -175,7 +187,8 @@ export async function updateKnowledge(ctx, r, impact, run) {
|
|
|
175
187
|
ctx.out(`static: ${r.changeSet.repo} is not a repos: source of product.yaml; its map was not rewritten`);
|
|
176
188
|
}
|
|
177
189
|
else {
|
|
178
|
-
const
|
|
190
|
+
const gitEnv = changelogGitEnv(ctx.config, ctx.env);
|
|
191
|
+
const result = await ingest({ source: isRemote(source) ? source : resolve(ctx.cwd, source), knowledgeDir, keepOverview: true, ...(gitEnv === undefined ? {} : { gitEnv }), log: (l) => ctx.log(`ingest: ${l}`) });
|
|
179
192
|
const parsed = result.static === undefined ? parseStaticSections(await readFile(result.file, "utf8")) : undefined;
|
|
180
193
|
ctx.out(staticLine(result.name, result.files, result.status, result.static, parsed, 0));
|
|
181
194
|
if (result.commit !== undefined && !r.changeSet.after.startsWith(result.commit) && !result.commit.startsWith(r.changeSet.after))
|
|
@@ -201,8 +214,11 @@ export async function updateKnowledge(ctx, r, impact, run) {
|
|
|
201
214
|
// a route with a placeholder is opened at the path the crawl once used for it
|
|
202
215
|
const targets = impact.pages.toSorted((a, b) => RANK[b.confidence] - RANK[a.confidence]).map((p) => known.find((k) => k.route === p.name.route)?.path ?? p.name.route).filter((p) => !/[{}]/.test(p)).slice(0, MAX_TARGET_PAGES);
|
|
203
216
|
const pages = [];
|
|
204
|
-
// The knowledge build's rule
|
|
205
|
-
const
|
|
217
|
+
// The crawl follows --env (M6) unless that is a production environment with explore off: the knowledge build's rule, a replay-only production is never crawled; then the dev tier, else a production environment that explores.
|
|
218
|
+
const fallback = run.dev ?? (run.production?.environment.explore === true ? run.production : undefined);
|
|
219
|
+
const crawlOn = preferred !== undefined && explores(preferred, ctx.env) ? preferred : fallback;
|
|
220
|
+
if (preferred !== undefined && !explores(preferred, ctx.env))
|
|
221
|
+
ctx.out(`crawl: ${preferred.name} is a production environment with explore off; ${fallback === undefined ? "nothing is crawled" : `crawling on ${fallback.name} instead`}`);
|
|
206
222
|
if (crawlOn === undefined)
|
|
207
223
|
ctx.out("crawl: skipped (no dev-tier environment, no production one with explore on)");
|
|
208
224
|
else if (targets.length === 0)
|
|
@@ -352,110 +368,155 @@ export function classifyRun(report) {
|
|
|
352
368
|
visit(report);
|
|
353
369
|
return out;
|
|
354
370
|
}
|
|
355
|
-
/**
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
371
|
+
/** A dev-tier environment, a production one with explore on, or one OMNIQA_ALLOW_EXPLORE names: the crawl and the fixer may act there. */
|
|
372
|
+
const explores = (w, env) => w.environment.tier !== "production" || w.environment.explore || env.OMNIQA_ALLOW_EXPLORE === w.name;
|
|
373
|
+
const FAILED = new Set(["failed", "behaviour-changed", "unsure"]);
|
|
374
|
+
/** The written cases a sync runs, grouped by environment: with `suite`, every written case of that environment's tier, on it; else the impacted written cases, the dev-tier ones on the dev environment, the production-only ones on the production one. Cases that cannot run land in `notRun`. */
|
|
375
|
+
function selectCases(plan, impact, run, suite, notRun) {
|
|
376
|
+
const groups = new Map();
|
|
377
|
+
const add = (where, s) => {
|
|
378
|
+
const g = groups.get(where.name) ?? { where, heal: where.environment.tier !== "production", selected: [] };
|
|
379
|
+
g.selected.push(s);
|
|
380
|
+
groups.set(where.name, g);
|
|
381
|
+
};
|
|
382
|
+
const all = plan.suites.flatMap((s) => s.cases.map((c) => ({ suite: s, case: c })));
|
|
383
|
+
if (suite !== undefined) {
|
|
384
|
+
for (const s of all)
|
|
385
|
+
if (s.case.status === "written" && s.case.file !== undefined && s.case.tiers.includes(suite.environment.tier))
|
|
386
|
+
add(suite, s);
|
|
387
|
+
return [...groups.values()];
|
|
363
388
|
}
|
|
364
|
-
const byId = new Map(
|
|
365
|
-
|
|
366
|
-
// filled as the stage goes, so a budget stop keeps what ran
|
|
367
|
-
const cases = into.cases;
|
|
368
|
-
const toRun = [];
|
|
369
|
-
const onProduction = [];
|
|
370
|
-
const vanished = (c) => c.routes.find((route) => k.removedRoutes.includes(route));
|
|
389
|
+
const byId = new Map(all.map((x) => [x.case.id, x]));
|
|
390
|
+
// ponytail: a case on a route the change removed runs too (removed routes are known only after the map is rewritten) and is marked skipped by the plan pass; the plan pass runs last, so a budget stop leaves it failed.
|
|
371
391
|
for (const i of impact.cases) {
|
|
372
392
|
const found = byId.get(i.id);
|
|
373
393
|
if (found === undefined)
|
|
374
394
|
continue;
|
|
375
395
|
const c = found.case;
|
|
376
396
|
const base = { id: c.id, title: c.title, ...(c.file === undefined ? {} : { file: c.file }) };
|
|
377
|
-
const gone = vanished(c);
|
|
378
397
|
if (c.status !== "written" || c.file === undefined)
|
|
379
|
-
|
|
380
|
-
else if (gone !== undefined)
|
|
381
|
-
cases.push({ ...base, outcome: "not-run", detail: `route ${gone} is no longer declared` });
|
|
398
|
+
notRun.push({ ...base, outcome: "not-run", detail: c.status });
|
|
382
399
|
else if (!c.tiers.includes("dev")) {
|
|
383
400
|
if (run.production === undefined)
|
|
384
|
-
|
|
401
|
+
notRun.push({ ...base, outcome: "not-run", detail: "production-only and no production-tier environment" });
|
|
385
402
|
else
|
|
386
|
-
|
|
403
|
+
add(run.production, found);
|
|
387
404
|
}
|
|
388
405
|
else if (run.dev === undefined)
|
|
389
|
-
|
|
406
|
+
notRun.push({ ...base, outcome: "not-run", detail: "no dev-tier environment" });
|
|
390
407
|
else
|
|
391
|
-
|
|
408
|
+
add(run.dev, found);
|
|
409
|
+
}
|
|
410
|
+
return [...groups.values()];
|
|
411
|
+
}
|
|
412
|
+
/** Pass 1: the selected cases run, one omniqa test run per environment; each ends pass, healed, behaviour changed, unsure, failed or not run. Resolves to the failures for the fixer. The runs' model calls are counted, not charged (M5). */
|
|
413
|
+
export async function runCases(ctx, impact, run, suite, into) {
|
|
414
|
+
const plan = await readPlan(planPath(join(ctx.cwd, ctx.config.testDir)));
|
|
415
|
+
if (plan === undefined) {
|
|
416
|
+
ctx.out("tests: skipped (no plan at tests/plan.md; run omniqa plan)");
|
|
417
|
+
return [];
|
|
392
418
|
}
|
|
419
|
+
const project = ctx.config.projects[0]?.name;
|
|
420
|
+
const cases = into.cases;
|
|
421
|
+
const groups = selectCases(plan, impact, run, suite, cases);
|
|
393
422
|
const notRun = cases.filter((c) => c.outcome === "not-run");
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
423
|
+
if (suite !== undefined) {
|
|
424
|
+
const g = groups[0];
|
|
425
|
+
ctx.out(`tests: suite on ${suite.name} (${project}): ${plural(g?.selected.length ?? 0, "written case")} of the ${suite.environment.tier} tier${g === undefined ? "" : `: ${ids(g.selected)}`}`);
|
|
426
|
+
}
|
|
427
|
+
else {
|
|
428
|
+
const dev = groups.find((g) => g.where.environment.tier !== "production");
|
|
429
|
+
const prod = groups.find((g) => g.where.environment.tier === "production");
|
|
430
|
+
ctx.out(`tests: ${plural(impact.cases.length, "impacted case")}; running ${dev?.selected.length ?? 0} written${run.dev === undefined ? "" : ` on ${run.dev.name} (${project})`}: ${dev === undefined ? "none" : ids(dev.selected)}${prod === undefined ? "" : `; ${prod.selected.length} production-only on ${prod.where.name}: ${ids(prod.selected)}`}${notRun.length === 0 ? "" : `; not run: ${notRun.map((c) => `${c.id} (${c.detail})`).join(", ")}`}`);
|
|
431
|
+
}
|
|
432
|
+
const failed = [];
|
|
433
|
+
for (const g of groups) {
|
|
434
|
+
const onProduction = g.where.environment.tier === "production";
|
|
435
|
+
if (onProduction)
|
|
436
|
+
run.ranOnProduction = true;
|
|
437
|
+
const reportName = onProduction ? "omniqa-sync-production" : "omniqa-sync";
|
|
438
|
+
const result = await runCase(ctx.cwd, ctx.env, { ids: g.selected.map((s) => s.case.id), envName: g.where.name, project, heal: g.heal, reportName, log: ctx.log });
|
|
439
|
+
// ponytail: a run that leaves no report (no browser, a runner crash) fails the sync loudly instead of passing as not run; per-group not-run outcomes are the upgrade when a second environment's results matter.
|
|
398
440
|
if (result.report === undefined)
|
|
399
441
|
throw new Error(result.error ?? "the run left no report");
|
|
400
|
-
run.
|
|
442
|
+
run.runCalls += result.summary?.modelCalls ?? 0;
|
|
401
443
|
if (result.summary !== undefined)
|
|
402
444
|
await recordRun(ctx.cwd, { ...result.summary, report: join("test-results", `${reportName}.json`) });
|
|
403
445
|
const classified = classifyRun(result.report);
|
|
404
|
-
const
|
|
405
|
-
for (const s of selected) {
|
|
446
|
+
for (const s of g.selected) {
|
|
406
447
|
const title = `${s.case.id}: ${s.case.title}`;
|
|
407
448
|
const got = classified.get(title) ?? { outcome: "failed", detail: `no test titled ${JSON.stringify(title)} ran`, calls: 0 };
|
|
408
449
|
cases.push({ id: s.case.id, title: s.case.title, file: s.case.file, outcome: got.outcome, ...(got.detail === undefined ? {} : { detail: got.detail }) });
|
|
409
450
|
ctx.out(` ${s.case.id}: ${OUTCOME_WORDS[got.outcome]}${got.detail === undefined ? "" : ` — ${got.detail}`}`);
|
|
410
|
-
if (got.outcome
|
|
411
|
-
|
|
451
|
+
if (FAILED.has(got.outcome))
|
|
452
|
+
failed.push({ selected: s, detail: got.detail ?? OUTCOME_WORDS[got.outcome], where: g.where });
|
|
412
453
|
}
|
|
413
454
|
for (const line of result.summary?.productionSteps ?? [])
|
|
414
455
|
ctx.out(` ${line}`);
|
|
415
|
-
return changed;
|
|
416
|
-
};
|
|
417
|
-
const changed = toRun.length > 0 && run.dev !== undefined ? await runOn(run.dev, toRun, true, "omniqa-sync") : [];
|
|
418
|
-
if (onProduction.length > 0 && run.production !== undefined) {
|
|
419
|
-
run.ranOnProduction = true;
|
|
420
|
-
// No --heal on production: a miss heals there only where its explore is on, and explore: false keeps it a replay.
|
|
421
|
-
// ponytail: a behaviour change found on production is reported, not handed to the fixer, which rewrites on the dev tier; a fixer on production is the upgrade.
|
|
422
|
-
await runOn(run.production, onProduction, false, "omniqa-sync-production");
|
|
423
456
|
}
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
457
|
+
return failed;
|
|
458
|
+
}
|
|
459
|
+
/** Pass 5: every failed case goes to the fixer on the environment it failed on, with the change set and the rewritten knowledge (M4); a production environment that does not explore gets no fixer (D5), a case on a route the change removed waits for the plan pass. */
|
|
460
|
+
export async function fixFailed(ctx, r, impact, k, run, failed, cases) {
|
|
461
|
+
const fixable = failed.filter((f) => !f.selected.case.routes.some((route) => k.removedRoutes.includes(route)));
|
|
462
|
+
if (fixable.length === 0)
|
|
463
|
+
return;
|
|
464
|
+
ctx.out(`fix: ${plural(fixable.length, "failed case")}: ${fixable.map((f) => f.selected.case.id).join(", ")}`);
|
|
465
|
+
const project = ctx.config.projects[0]?.name;
|
|
466
|
+
const knowledge = [`sections rewritten: ${impact.sections.map((x) => x.name).join(", ") || "none"}`, `pages re-crawled: ${k.pages.join(", ") || "none"}`, `scheme: ${k.scheme}`].join("\n");
|
|
467
|
+
const byEnv = new Map();
|
|
468
|
+
for (const f of fixable)
|
|
469
|
+
byEnv.set(f.where.name, [...(byEnv.get(f.where.name) ?? []), f]);
|
|
470
|
+
for (const group of byEnv.values()) {
|
|
471
|
+
const where = group[0].where;
|
|
472
|
+
const entry = (f) => cases.find((c) => c.id === f.selected.case.id);
|
|
473
|
+
if (!explores(where, ctx.env)) {
|
|
474
|
+
for (const f of group) {
|
|
475
|
+
entry(f).outcome = "failed";
|
|
476
|
+
entry(f).detail = `${f.detail}; no fixer: ${where.name} does not explore`;
|
|
477
|
+
ctx.out(` ${f.selected.case.id}: not fixed — ${where.name} is a production environment with explore off`);
|
|
478
|
+
}
|
|
479
|
+
continue;
|
|
480
|
+
}
|
|
481
|
+
const fixer = await createFixer({ cwd: ctx.cwd, env: ctx.env, config: ctx.config, product: ctx.product, envName: where.name, environment: where.environment, project, budget: run.budget, transcript: run.transcript, changeSet: r.changeSet, mappedFiles: impact.files.filter((f) => f.confidence !== undefined).map((f) => f.path), knowledge, out: ctx.out, log: ctx.log });
|
|
428
482
|
try {
|
|
429
|
-
for (const
|
|
430
|
-
const fixed = await fixer.fix(selected, detail);
|
|
431
|
-
const
|
|
432
|
-
|
|
483
|
+
for (const f of group) {
|
|
484
|
+
const fixed = await fixer.fix(f.selected, f.detail);
|
|
485
|
+
const e = entry(f);
|
|
486
|
+
e.outcome = fixed.status;
|
|
433
487
|
if (fixed.feedback !== undefined)
|
|
434
|
-
|
|
488
|
+
e.detail = fixed.feedback;
|
|
435
489
|
}
|
|
436
490
|
}
|
|
437
491
|
finally {
|
|
438
492
|
fixer.close();
|
|
439
493
|
}
|
|
440
494
|
}
|
|
441
|
-
|
|
495
|
+
}
|
|
496
|
+
/** Pass 6: cases on routes the change removed are proposed as skipped; added routes and handlers become planned cases through one planner call. */
|
|
497
|
+
export async function updatePlan(ctx, k, run, into) {
|
|
498
|
+
const planFile = planPath(join(ctx.cwd, ctx.config.testDir));
|
|
499
|
+
const current = await readPlan(planFile);
|
|
500
|
+
if (current === undefined)
|
|
501
|
+
return;
|
|
502
|
+
const cases = into.cases;
|
|
442
503
|
const planLines = [];
|
|
443
|
-
const current = (await readPlan(planFile)) ?? plan;
|
|
444
504
|
const skipped = [];
|
|
445
505
|
for (const c of current.suites.flatMap((s) => s.cases)) {
|
|
446
|
-
const gone =
|
|
506
|
+
const gone = c.routes.find((route) => k.removedRoutes.includes(route));
|
|
447
507
|
if (gone === undefined || c.status === "skipped")
|
|
448
508
|
continue;
|
|
449
509
|
c.status = "skipped";
|
|
450
510
|
skipped.push(c.id);
|
|
451
|
-
|
|
511
|
+
const detail = `route ${gone} is no longer declared`;
|
|
512
|
+
planLines.push(`${c.id} skipped (${detail})`);
|
|
452
513
|
const entry = cases.find((x) => x.id === c.id);
|
|
453
514
|
if (entry !== undefined) {
|
|
454
515
|
entry.outcome = "skipped";
|
|
455
|
-
entry.detail =
|
|
516
|
+
entry.detail = detail;
|
|
456
517
|
}
|
|
457
518
|
else
|
|
458
|
-
cases.push({ id: c.id, title: c.title, ...(c.file === undefined ? {} : { file: c.file }), outcome: "skipped", detail
|
|
519
|
+
cases.push({ id: c.id, title: c.title, ...(c.file === undefined ? {} : { file: c.file }), outcome: "skipped", detail });
|
|
459
520
|
}
|
|
460
521
|
if (skipped.length > 0)
|
|
461
522
|
await writePlan(planFile, current);
|
|
@@ -473,8 +534,6 @@ export async function runImpacted(ctx, r, impact, k, run, into) {
|
|
|
473
534
|
}
|
|
474
535
|
}
|
|
475
536
|
ctx.out(`plan: ${planLines.join("; ") || "unchanged"}`);
|
|
476
|
-
const order = new Map(impact.cases.map((c, i) => [c.id, i]));
|
|
477
|
-
cases.sort((a, b) => (order.get(a.id) ?? Infinity) - (order.get(b.id) ?? Infinity));
|
|
478
537
|
}
|
|
479
538
|
const count = (cases, outcome) => cases.filter((c) => c.outcome === outcome).length;
|
|
480
539
|
const cell = (text) => text.replace(/\|/g, "\\|").replace(/\s*\n\s*/g, " ").trim() || "none";
|
|
@@ -508,13 +567,34 @@ export function testsLine(o) {
|
|
|
508
567
|
return "tests: none";
|
|
509
568
|
return `tests: ran ${t.ran}: passed ${t.passed}, healed ${t.healed}, fixed ${t.fixed}, regressions ${t.regressions}, needs a human ${t.needsHuman}, failed ${t.failed}; skipped ${t.skipped}, planned ${o.summary?.planned ?? 0}; ${plural(o.modelCalls, "model call")}`;
|
|
510
569
|
}
|
|
511
|
-
|
|
570
|
+
const MAX_NOTIFY_LINES = 10;
|
|
571
|
+
/** The first notification: which tests failed where, after which deploy, and whether the healer runs. */
|
|
572
|
+
export function failuresText(changeSet, failed, ran, healer) {
|
|
573
|
+
const where = [...new Set(failed.map((f) => f.where.name))].join(", ");
|
|
574
|
+
const head = `omniqa: ${failed.length} of ${ran} tests failed on ${where} after ${changeSet.repo} ${short(changeSet.after)}${changeSet.title === undefined ? "" : ` (${changeSet.title})`}; ${healer}`;
|
|
575
|
+
const lines = failed.slice(0, MAX_NOTIFY_LINES).map((f) => ` ${f.selected.case.id}: ${f.selected.case.title} — ${f.detail}`);
|
|
576
|
+
const more = failed.length - lines.length;
|
|
577
|
+
return [head, ...lines, ...(more > 0 ? [` … and ${more} more (see the sync report)`] : [])].join("\n");
|
|
578
|
+
}
|
|
579
|
+
/** The second notification: what the healer made of the failures, what the knowledge update did, where it all went. */
|
|
580
|
+
export function outcomeText(o, failed) {
|
|
581
|
+
const s = o.summary;
|
|
582
|
+
const where = [...new Set(failed.map((f) => f.where.name))].join(", ");
|
|
583
|
+
const list = (outcomes) => {
|
|
584
|
+
const ids = o.cases.filter((c) => outcomes.includes(c.outcome)).map((c) => c.id);
|
|
585
|
+
return ids.length === 0 ? "" : ` (${ids.slice(0, MAX_NOTIFY_LINES).join(", ")}${ids.length > MAX_NOTIFY_LINES ? ", …" : ""})`;
|
|
586
|
+
};
|
|
587
|
+
const t = s.tests;
|
|
588
|
+
return `omniqa: healer on ${where} after ${s.repo} ${short(s.after)}: fixed ${t.fixed}${list(["fixed"])}, regression ${t.regressions}${list(["regression"])}, needs a human ${t.needsHuman}${list(["needs-human", "unsure"])}, still failing ${t.failed}${list(["failed", "behaviour-changed"])}; knowledge: ${plural(s.knowledge.maps, "map")}, ${plural(s.knowledge.pages, "page")}, scheme ${s.knowledge.scheme}; ${persistedText(s.persisted)}${s.stopped === undefined ? "" : `; stopped: ${s.stopped}`}`;
|
|
589
|
+
}
|
|
590
|
+
function summarize(o, at, budget, runCalls, suite) {
|
|
512
591
|
const ran = o.cases.filter((c) => !["not-run", "skipped"].includes(c.outcome));
|
|
513
592
|
return {
|
|
514
593
|
at,
|
|
515
594
|
host: o.changeSet.host,
|
|
516
595
|
repo: o.changeSet.repo,
|
|
517
596
|
kind: o.changeSet.kind,
|
|
597
|
+
...(suite === undefined ? {} : { suite: true, environment: suite.name }),
|
|
518
598
|
before: o.changeSet.before,
|
|
519
599
|
after: o.changeSet.after,
|
|
520
600
|
...(o.changeSet.title === undefined ? {} : { title: o.changeSet.title }),
|
|
@@ -526,12 +606,13 @@ function summarize(o, at, budget) {
|
|
|
526
606
|
tests: { ran: ran.length, passed: count(o.cases, "pass"), healed: count(o.cases, "healed"), fixed: count(o.cases, "fixed"), regressions: count(o.cases, "regression"), needsHuman: count(o.cases, "needs-human") + count(o.cases, "unsure"), skipped: count(o.cases, "skipped"), failed: count(o.cases, "failed") + count(o.cases, "behaviour-changed") },
|
|
527
607
|
planned: o.planned.length,
|
|
528
608
|
modelCalls: o.modelCalls,
|
|
609
|
+
...(runCalls === 0 ? {} : { runCalls }),
|
|
529
610
|
budget,
|
|
530
611
|
...(o.stopped === undefined ? {} : { stopped: o.stopped }),
|
|
531
612
|
persisted: { kind: "off" },
|
|
532
613
|
};
|
|
533
614
|
}
|
|
534
|
-
/** The pipeline for one request: resolve the change sets
|
|
615
|
+
/** The pipeline for one request: resolve the change sets; for each: the run, the impact, the knowledge update, the fixer, the plan and the persistence, in that order. */
|
|
535
616
|
export async function runSync(cwd, env, request, o) {
|
|
536
617
|
const ctx = await syncContext(cwd, env, o);
|
|
537
618
|
const dev = devEnvironment(ctx.product, o.env);
|
|
@@ -542,10 +623,27 @@ export async function runSync(cwd, env, request, o) {
|
|
|
542
623
|
const outcomes = [];
|
|
543
624
|
const lines = [];
|
|
544
625
|
const out = ctx.out;
|
|
626
|
+
/** One { text } post to notify.webhookEnv, redacted; a missing variable or a channel that is down is one line, never a failure. */
|
|
627
|
+
const notify = async (text) => {
|
|
628
|
+
const n = ctx.config.notify;
|
|
629
|
+
if (n === undefined)
|
|
630
|
+
return;
|
|
631
|
+
const url = env[n.webhookEnv];
|
|
632
|
+
if (!url) {
|
|
633
|
+
ctx.log(`notify: ${n.webhookEnv} is not set; nothing sent`);
|
|
634
|
+
return;
|
|
635
|
+
}
|
|
636
|
+
const error = await postText(url, redact(text));
|
|
637
|
+
ctx.out(error === undefined ? "notify: sent" : redact(`notify: not sent: ${error}`));
|
|
638
|
+
};
|
|
545
639
|
ctx.out = (line) => {
|
|
546
640
|
lines.push(line);
|
|
547
641
|
out(line);
|
|
548
642
|
};
|
|
643
|
+
const named = o.env === undefined ? undefined : selectEnvironment(ctx.product, o.env);
|
|
644
|
+
if (o.suite === true && named === undefined)
|
|
645
|
+
throw new Error("--suite needs --env <name>, the environment the deploy landed on");
|
|
646
|
+
const suite = o.suite === true ? named : undefined;
|
|
549
647
|
for (const r of await resolveChangeSets(ctx, request, o.dryRun === true)) {
|
|
550
648
|
lines.length = 0;
|
|
551
649
|
for (const line of describeLines(r.changeSet))
|
|
@@ -562,10 +660,17 @@ export async function runSync(cwd, env, request, o) {
|
|
|
562
660
|
const transcript = await openTranscript(join(cwd, RUNS_DIR), transcriptName("sync", ctx.now()), { redact });
|
|
563
661
|
await transcript.append({ kind: "sync", changeSet: { ...r.changeSet, files: r.changeSet.files.map((f) => ({ path: f.path, status: f.status })) }, impact: relative(cwd, file) });
|
|
564
662
|
const outcome = { changeSet: r.changeSet, impact, impactFile: file, cases: [], planned: [], modelCalls: 0 };
|
|
565
|
-
const run = { budget, transcript, redact, ...(dev === undefined ? {} : { dev }), ...(production === undefined ? {} : { production }) };
|
|
663
|
+
const run = { budget, transcript, redact, runCalls: 0, ...(dev === undefined ? {} : { dev }), ...(production === undefined ? {} : { production }) };
|
|
664
|
+
let failed = [];
|
|
566
665
|
try {
|
|
567
|
-
|
|
568
|
-
|
|
666
|
+
failed = await runCases(ctx, impact, run, suite, outcome);
|
|
667
|
+
if (failed.length > 0) {
|
|
668
|
+
const healer = failed.some((f) => explores(f.where, env)) ? "healer launched" : `no healer: ${[...new Set(failed.filter((f) => !explores(f.where, env)).map((f) => f.where.name))].join(", ")} does not explore`;
|
|
669
|
+
await notify(failuresText(r.changeSet, failed, outcome.cases.filter((c) => !["not-run", "skipped"].includes(c.outcome)).length, healer));
|
|
670
|
+
}
|
|
671
|
+
outcome.knowledge = await updateKnowledge(ctx, r, impact, run, named);
|
|
672
|
+
await fixFailed(ctx, r, impact, outcome.knowledge, run, failed, outcome.cases);
|
|
673
|
+
await updatePlan(ctx, outcome.knowledge, run, outcome);
|
|
569
674
|
if (r.eventId !== undefined)
|
|
570
675
|
await removePendingChange(cwd, r.eventId);
|
|
571
676
|
}
|
|
@@ -574,8 +679,10 @@ export async function runSync(cwd, env, request, o) {
|
|
|
574
679
|
throw err;
|
|
575
680
|
outcome.stopped = err.message;
|
|
576
681
|
}
|
|
577
|
-
|
|
578
|
-
outcome.
|
|
682
|
+
const order = new Map(impact.cases.map((c, i) => [c.id, i]));
|
|
683
|
+
outcome.cases.sort((a, b) => (order.get(a.id) ?? Infinity) - (order.get(b.id) ?? Infinity));
|
|
684
|
+
outcome.modelCalls = budget.used + run.runCalls;
|
|
685
|
+
outcome.summary = summarize(outcome, ctx.now().toISOString(), limit, run.runCalls, suite);
|
|
579
686
|
ctx.out(testsLine(outcome));
|
|
580
687
|
try {
|
|
581
688
|
ctx.out(`coverage: ${(await writeCoverageFile(cwd)).line}`);
|
|
@@ -600,6 +707,9 @@ export async function runSync(cwd, env, request, o) {
|
|
|
600
707
|
if (persisted.error !== undefined)
|
|
601
708
|
throw new Error(`persist failed: ${persisted.error}`);
|
|
602
709
|
}
|
|
710
|
+
// a persist error threw above: the CI job fails and says why, so the outcome message is skipped
|
|
711
|
+
if (failed.length > 0)
|
|
712
|
+
await notify(outcomeText(outcome, failed));
|
|
603
713
|
outcomes.push(outcome);
|
|
604
714
|
}
|
|
605
715
|
return outcomes;
|