ckb-cycle-tools 0.0.0-stage → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +180 -0
- package/LICENSE +21 -0
- package/README.md +652 -3
- package/dist/analyze.d.ts +50 -0
- package/dist/analyze.js +95 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +405 -0
- package/dist/debugger.d.ts +28 -0
- package/dist/debugger.js +56 -0
- package/dist/diff.d.ts +218 -0
- package/dist/diff.js +441 -0
- package/dist/elf.d.ts +18 -0
- package/dist/elf.js +209 -0
- package/dist/flame.d.ts +65 -0
- package/dist/flame.js +199 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.js +11 -0
- package/dist/mock.d.ts +109 -0
- package/dist/mock.js +197 -0
- package/dist/profile.d.ts +72 -0
- package/dist/profile.js +144 -0
- package/dist/replay.d.ts +167 -0
- package/dist/replay.js +296 -0
- package/dist/report.d.ts +90 -0
- package/dist/report.js +445 -0
- package/dist/scan.d.ts +46 -0
- package/dist/scan.js +122 -0
- package/dist/substitute.d.ts +48 -0
- package/dist/substitute.js +143 -0
- package/dist/wasm.d.ts +22 -0
- package/dist/wasm.js +58 -0
- package/package.json +75 -6
package/dist/diff.d.ts
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Compares two cycle reports — the base of a pull request and its head — group
|
|
3
|
+
* by group, and formats the result for a terminal or a PR comment.
|
|
4
|
+
*
|
|
5
|
+
* Pure: no network, no file system.
|
|
6
|
+
*/
|
|
7
|
+
/** What `ckb-cycles report` writes. */
|
|
8
|
+
export interface CycleReport {
|
|
9
|
+
tool: "ckb-cycles";
|
|
10
|
+
version: 1;
|
|
11
|
+
/** ckb-debugger's `--version`, so two reports can be checked for comparability. */
|
|
12
|
+
debugger?: string;
|
|
13
|
+
scenarios: ScenarioResult[];
|
|
14
|
+
/**
|
|
15
|
+
* What made `report` exit 2, beyond the scenarios' own errors: a binary that
|
|
16
|
+
* matched no cell dep in any scenario, so nothing measured it. A gated diff
|
|
17
|
+
* fails on these, from either side.
|
|
18
|
+
*/
|
|
19
|
+
errors?: string[];
|
|
20
|
+
}
|
|
21
|
+
export interface ScenarioResult {
|
|
22
|
+
name: string;
|
|
23
|
+
/** Set when the scenario could not be run at all (a missing file, a node that is down). */
|
|
24
|
+
error?: string;
|
|
25
|
+
groups: GroupResult[];
|
|
26
|
+
/** Warnings from binary substitution, kept so a reader can tell why a result changed. */
|
|
27
|
+
warnings?: string[];
|
|
28
|
+
/** The binaries swapped into this scenario. */
|
|
29
|
+
substituted?: string[];
|
|
30
|
+
/** Binaries offered to this scenario (it lists none) that match none of its cell deps. */
|
|
31
|
+
unmatched?: string[];
|
|
32
|
+
/**
|
|
33
|
+
* A binary the scenario was offered does not exist where `--binaries-from`
|
|
34
|
+
* points (`missingBinaries` names them). The scenario ran with the others;
|
|
35
|
+
* the groups whose code is a missing one carry `missingBinary` and are not
|
|
36
|
+
* compared. (A report before 1.1.0 set this with `error` instead, and no
|
|
37
|
+
* groups: then every head group of the scenario counts as new.)
|
|
38
|
+
*/
|
|
39
|
+
binaryMissing?: boolean;
|
|
40
|
+
missingBinaries?: string[];
|
|
41
|
+
}
|
|
42
|
+
export interface GroupResult {
|
|
43
|
+
/** ckb-debugger's selector, e.g. `input.0.lock`: the key a group is matched by across reports. */
|
|
44
|
+
selector: string;
|
|
45
|
+
role: "lock" | "type";
|
|
46
|
+
/** A name for the script, or its short code hash. */
|
|
47
|
+
script: string;
|
|
48
|
+
codeHash: string;
|
|
49
|
+
hashType: string;
|
|
50
|
+
/** The script's exit code: 0 accepts. -1 with `error` when the VM stopped it. */
|
|
51
|
+
result: number;
|
|
52
|
+
error?: string;
|
|
53
|
+
/** Cycles; 0 when the VM stopped the script (`error`), whose count is not a cost. */
|
|
54
|
+
cycles: number;
|
|
55
|
+
/**
|
|
56
|
+
* The binary this group's code stands for was not in the checkout (the base):
|
|
57
|
+
* its numbers are the mock's code, not that build's, so it is not compared.
|
|
58
|
+
*/
|
|
59
|
+
missingBinary?: string;
|
|
60
|
+
/**
|
|
61
|
+
* The costliest functions of the run, heaviest self first (at most
|
|
62
|
+
* FUNCTION_LIMIT). Only for a group whose binary has a `symbols` build.
|
|
63
|
+
*/
|
|
64
|
+
functions?: FunctionResult[];
|
|
65
|
+
/** Why `functions` were not recorded although asked for (the symbols build costs other cycles…). */
|
|
66
|
+
functionsWarning?: string;
|
|
67
|
+
}
|
|
68
|
+
export interface FunctionResult {
|
|
69
|
+
name: string;
|
|
70
|
+
/** Cycles spent in the function itself. */
|
|
71
|
+
self: number;
|
|
72
|
+
/** Cycles spent in it and everything it called. */
|
|
73
|
+
total: number;
|
|
74
|
+
}
|
|
75
|
+
/** How many functions a report keeps per group. */
|
|
76
|
+
export declare const FUNCTION_LIMIT = 15;
|
|
77
|
+
/**
|
|
78
|
+
* One function's self cycles on each side. A side is undefined when it is not
|
|
79
|
+
* known: no functions were recorded there, or the function is not in a list
|
|
80
|
+
* that was cut at FUNCTION_LIMIT. In a shorter list, absent means 0.
|
|
81
|
+
*/
|
|
82
|
+
export interface FunctionDelta {
|
|
83
|
+
name: string;
|
|
84
|
+
base?: number;
|
|
85
|
+
head?: number;
|
|
86
|
+
/** Set when both sides are known. */
|
|
87
|
+
delta?: number;
|
|
88
|
+
}
|
|
89
|
+
/** `error`: the head could not run the scenario. `base-error`: the base could not, so nothing was compared. */
|
|
90
|
+
export type RowKind = "same" | "changed" | "result" | "new" | "removed" | "error" | "base-error";
|
|
91
|
+
export interface DiffRow {
|
|
92
|
+
scenario: string;
|
|
93
|
+
/** The group's selector, or "" for a row about the whole scenario. */
|
|
94
|
+
group: string;
|
|
95
|
+
script: string;
|
|
96
|
+
base?: {
|
|
97
|
+
result: string;
|
|
98
|
+
cycles?: number;
|
|
99
|
+
};
|
|
100
|
+
head?: {
|
|
101
|
+
result: string;
|
|
102
|
+
cycles?: number;
|
|
103
|
+
};
|
|
104
|
+
delta?: number;
|
|
105
|
+
/** Percentage change; undefined when the base cost nothing. */
|
|
106
|
+
percent?: number;
|
|
107
|
+
kind: RowKind;
|
|
108
|
+
/** The increase is over the threshold. */
|
|
109
|
+
overThreshold: boolean;
|
|
110
|
+
/** Free text for an error row. */
|
|
111
|
+
note?: string;
|
|
112
|
+
/** Functions whose self cycles changed, largest change first (when the head recorded functions). */
|
|
113
|
+
functions?: FunctionDelta[];
|
|
114
|
+
/** The head's reason for not recording functions. */
|
|
115
|
+
functionsWarning?: string;
|
|
116
|
+
/** Why the base side of this group was not compared (its binary was not in the base checkout). */
|
|
117
|
+
notCompared?: string;
|
|
118
|
+
}
|
|
119
|
+
export interface CycleDiff {
|
|
120
|
+
rows: DiffRow[];
|
|
121
|
+
/**
|
|
122
|
+
* Groups whose accept/reject outcome changed (ok → exit 12, exit 12 → ok,
|
|
123
|
+
* exit 12 → exit 13), and new groups that do not accept: a new group is
|
|
124
|
+
* compared with "did not run".
|
|
125
|
+
*/
|
|
126
|
+
resultChanges: number;
|
|
127
|
+
/** Groups whose cycles grew by more than the threshold. */
|
|
128
|
+
overThreshold: number;
|
|
129
|
+
/** Scenarios the base could not run: their head groups had nothing to be compared with. */
|
|
130
|
+
baseErrors: number;
|
|
131
|
+
/**
|
|
132
|
+
* Base groups with nothing in the head to be compared with: removed, renamed
|
|
133
|
+
* or moved (another scenario name or selector), or measured without their
|
|
134
|
+
* binary (`notCompared`). A gated diff fails on them unless `allowRemoved`.
|
|
135
|
+
*/
|
|
136
|
+
removed: number;
|
|
137
|
+
/** Head groups with no base group. Reported, never a failure on their own. */
|
|
138
|
+
newGroups: number;
|
|
139
|
+
/** Either report's own errors (`base: ...`, `head: ...`): a gated diff fails on them. */
|
|
140
|
+
reportErrors: string[];
|
|
141
|
+
/** The head's scenario warnings, for the comment. */
|
|
142
|
+
warnings: {
|
|
143
|
+
scenario: string;
|
|
144
|
+
text: string;
|
|
145
|
+
}[];
|
|
146
|
+
/** Set when the two reports name different ckb-debugger versions. */
|
|
147
|
+
debuggers?: {
|
|
148
|
+
base: string;
|
|
149
|
+
head: string;
|
|
150
|
+
};
|
|
151
|
+
threshold?: number;
|
|
152
|
+
}
|
|
153
|
+
export interface DiffOptions {
|
|
154
|
+
/** Percent. An increase above it is flagged, and makes `failed` true. */
|
|
155
|
+
failAbove?: number;
|
|
156
|
+
/** A changed outcome makes `failed` true. */
|
|
157
|
+
failOnResultChange?: boolean;
|
|
158
|
+
/** Removed, renamed or uncompared base groups do not fail a gated diff. */
|
|
159
|
+
allowRemoved?: boolean;
|
|
160
|
+
}
|
|
161
|
+
/** The message for a binary no scenario's cell deps match, in a report's `errors` and the diff. */
|
|
162
|
+
export declare const unmatchedError: (name: string) => string;
|
|
163
|
+
/**
|
|
164
|
+
* `ok`, `exit 12`, or `error: <kind>` when the VM stopped the script: the
|
|
165
|
+
* debugger's message without its `key=value` details (a pc, an instruction),
|
|
166
|
+
* so "invalid instruction" and "max cycles exceeded" are different results.
|
|
167
|
+
*/
|
|
168
|
+
export declare function resultLabel(g: Pick<GroupResult, "result" | "error">): string;
|
|
169
|
+
export declare function diffReports(base: CycleReport, head: CycleReport, options?: DiffOptions): CycleDiff;
|
|
170
|
+
/**
|
|
171
|
+
* The functions whose self cycles differ between two lists, largest change
|
|
172
|
+
* first. One whose other side is not known (see FunctionDelta) is kept, ranked
|
|
173
|
+
* by the side that is.
|
|
174
|
+
*/
|
|
175
|
+
export declare function functionDeltas(base: FunctionResult[] | undefined, head: FunctionResult[]): FunctionDelta[];
|
|
176
|
+
/**
|
|
177
|
+
* Whether the diff should fail a check, per the options it was made with. A
|
|
178
|
+
* scenario the head could not run always fails. With a gate set (either
|
|
179
|
+
* option), so does anything that was not checked: a scenario the base could not
|
|
180
|
+
* run, an error in either report, and a base group with nothing to compare
|
|
181
|
+
* against (removed, renamed, moved, or measured without its binary) unless
|
|
182
|
+
* `allowRemoved`. A head group the VM stopped is a result change, and over any
|
|
183
|
+
* `failAbove` threshold.
|
|
184
|
+
*/
|
|
185
|
+
export declare function diffFailed(diff: CycleDiff, options?: DiffOptions): boolean;
|
|
186
|
+
/**
|
|
187
|
+
* One line saying what matters, for the top of a comment or terminal output.
|
|
188
|
+
* Text from the reports in it (debugger versions) goes through `clean`:
|
|
189
|
+
* termLine by default, the markdown escape in a comment.
|
|
190
|
+
*/
|
|
191
|
+
export declare function summarize(diff: CycleDiff, clean?: (s: string) => string): string;
|
|
192
|
+
/**
|
|
193
|
+
* Text from a config, a build or a node (scenario, script and function names,
|
|
194
|
+
* warnings) for a terminal or a CI log: oneLine, and never `::` at the start of
|
|
195
|
+
* a line, which GitHub Actions would run as a workflow command (`::error::`,
|
|
196
|
+
* `::add-mask::`, `::stop-commands::`); a zero-width space breaks it.
|
|
197
|
+
*/
|
|
198
|
+
export declare const termLine: (s: string) => string;
|
|
199
|
+
/** termLine for text that keeps its line breaks (a debugger's output in an error): each line cleaned the same way. */
|
|
200
|
+
export declare const termLines: (s: string) => string;
|
|
201
|
+
/** How many changed functions a comment lists per group. */
|
|
202
|
+
export declare const FUNCTIONS_SHOWN = 10;
|
|
203
|
+
/** The hidden marker that identifies the sticky comment, so it is updated instead of duplicated. */
|
|
204
|
+
export declare const COMMENT_MARKER = "<!-- ckb-cycles-report -->";
|
|
205
|
+
export interface MarkdownOptions {
|
|
206
|
+
title?: string;
|
|
207
|
+
/** e.g. a short commit hash, shown in the header line. */
|
|
208
|
+
baseLabel?: string;
|
|
209
|
+
headLabel?: string;
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* A markdown comment: what changed in a table, unchanged groups folded away.
|
|
213
|
+
* Result changes come first, since an accept turning into a reject (or the
|
|
214
|
+
* reverse) matters more than any cycle count.
|
|
215
|
+
*/
|
|
216
|
+
export declare function formatMarkdown(diff: CycleDiff, options?: MarkdownOptions): string;
|
|
217
|
+
/** The same table for a terminal. */
|
|
218
|
+
export declare function formatText(diff: CycleDiff): string;
|
package/dist/diff.js
ADDED
|
@@ -0,0 +1,441 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Compares two cycle reports — the base of a pull request and its head — group
|
|
3
|
+
* by group, and formats the result for a terminal or a PR comment.
|
|
4
|
+
*
|
|
5
|
+
* Pure: no network, no file system.
|
|
6
|
+
*/
|
|
7
|
+
/** How many functions a report keeps per group. */
|
|
8
|
+
export const FUNCTION_LIMIT = 15;
|
|
9
|
+
/** The message for a binary no scenario's cell deps match, in a report's `errors` and the diff. */
|
|
10
|
+
export const unmatchedError = (name) => `binary "${name}" matched no cell dep in any scenario: check its outPoint / codeHash`;
|
|
11
|
+
/** The base side of a new group that does not accept: compared with "did not run". */
|
|
12
|
+
const NOT_RUN = "not run";
|
|
13
|
+
/**
|
|
14
|
+
* `ok`, `exit 12`, or `error: <kind>` when the VM stopped the script: the
|
|
15
|
+
* debugger's message without its `key=value` details (a pc, an instruction),
|
|
16
|
+
* so "invalid instruction" and "max cycles exceeded" are different results.
|
|
17
|
+
*/
|
|
18
|
+
export function resultLabel(g) {
|
|
19
|
+
return g.error ? `error: ${errorKind(g.error)}` : g.result === 0 ? "ok" : `exit ${g.result}`;
|
|
20
|
+
}
|
|
21
|
+
function errorKind(error) {
|
|
22
|
+
const kind = error
|
|
23
|
+
.replace(/\s*\b\w+=\S*/g, "")
|
|
24
|
+
.replace(/\b0x[0-9a-fA-F]+\b/g, "")
|
|
25
|
+
.replace(/\s+/g, " ")
|
|
26
|
+
.trim();
|
|
27
|
+
return (kind || "vm").slice(0, 80);
|
|
28
|
+
}
|
|
29
|
+
export function diffReports(base, head, options = {}) {
|
|
30
|
+
const threshold = options.failAbove;
|
|
31
|
+
const rows = [];
|
|
32
|
+
const baseByName = new Map(base.scenarios.map((s) => [s.name, s]));
|
|
33
|
+
const headNames = new Set(head.scenarios.map((s) => s.name));
|
|
34
|
+
const scenarioRows = (h, b) => {
|
|
35
|
+
const name = (h ?? b).name;
|
|
36
|
+
if (h?.error) {
|
|
37
|
+
rows.push({ scenario: name, group: "", script: "", kind: "error", overThreshold: false, note: `head: ${h.error}` });
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
// A base that could not run the scenario has nothing to compare against, so
|
|
41
|
+
// every group is new. That is the base being broken, and the gate must not
|
|
42
|
+
// pass over it in silence. (Only a report before 1.1.0 records a missing
|
|
43
|
+
// binary as a scenario error with binaryMissing; its groups count as new.)
|
|
44
|
+
if (h && b?.error && !b.binaryMissing) {
|
|
45
|
+
rows.push({ scenario: name, group: "", script: "", kind: "base-error", overThreshold: false, note: `base: ${b.error}` });
|
|
46
|
+
}
|
|
47
|
+
const baseGroups = new Map((b && !b.error ? b.groups : []).map((g) => [g.selector, g]));
|
|
48
|
+
for (const g of h?.groups ?? []) {
|
|
49
|
+
rows.push(groupRow(name, baseGroups.get(g.selector), g, threshold));
|
|
50
|
+
baseGroups.delete(g.selector);
|
|
51
|
+
}
|
|
52
|
+
for (const g of baseGroups.values())
|
|
53
|
+
rows.push(groupRow(name, g, undefined, threshold));
|
|
54
|
+
};
|
|
55
|
+
for (const h of head.scenarios)
|
|
56
|
+
scenarioRows(h, baseByName.get(h.name));
|
|
57
|
+
for (const b of base.scenarios)
|
|
58
|
+
if (!headNames.has(b.name))
|
|
59
|
+
scenarioRows(undefined, b);
|
|
60
|
+
const groups = rows.filter((r) => r.kind !== "error" && r.kind !== "base-error");
|
|
61
|
+
return {
|
|
62
|
+
rows,
|
|
63
|
+
resultChanges: rows.filter((r) => r.kind === "result").length,
|
|
64
|
+
overThreshold: rows.filter((r) => r.overThreshold).length,
|
|
65
|
+
baseErrors: rows.filter((r) => r.kind === "base-error").length,
|
|
66
|
+
removed: groups.filter((r) => r.kind === "removed" || r.notCompared).length,
|
|
67
|
+
newGroups: groups.filter((r) => r.kind !== "removed" && (!r.base || r.base.result === NOT_RUN)).length,
|
|
68
|
+
reportErrors: [...reportErrors("base", base), ...reportErrors("head", head)],
|
|
69
|
+
warnings: head.scenarios.flatMap((s) => (s.warnings ?? []).map((text) => ({ scenario: s.name, text }))),
|
|
70
|
+
...(base.debugger && head.debugger && base.debugger !== head.debugger && { debuggers: { base: base.debugger, head: head.debugger } }),
|
|
71
|
+
...(threshold !== undefined && { threshold }),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* A report's errors other than its scenarios' own (those are rows already),
|
|
76
|
+
* and, from its scenarios, any binary offered but swapped in nowhere: a report
|
|
77
|
+
* written without its `errors` still says that nothing measured that binary.
|
|
78
|
+
*/
|
|
79
|
+
function reportErrors(side, r) {
|
|
80
|
+
const ofScenarios = new Set(r.scenarios.filter((s) => s.error).map((s) => `scenario "${s.name}": ${s.error}`));
|
|
81
|
+
const out = new Set((Array.isArray(r.errors) ? r.errors : []).map(String).filter((e) => !ofScenarios.has(e)));
|
|
82
|
+
// As the report does: only when every scenario ran (one that failed may be the one that uses it).
|
|
83
|
+
if (!ofScenarios.size) {
|
|
84
|
+
const swapped = new Set(r.scenarios.flatMap((s) => s.substituted ?? []));
|
|
85
|
+
for (const name of new Set(r.scenarios.flatMap((s) => s.unmatched ?? [])))
|
|
86
|
+
if (!swapped.has(name))
|
|
87
|
+
out.add(unmatchedError(name));
|
|
88
|
+
}
|
|
89
|
+
return [...out].map((e) => `${side}: ${e}`);
|
|
90
|
+
}
|
|
91
|
+
function groupRow(scenario, measured, h, threshold) {
|
|
92
|
+
const side = (g) => ({ result: resultLabel(g), cycles: g.cycles });
|
|
93
|
+
// Measured without its binary (not in the base checkout): the mock's code, not the base's build.
|
|
94
|
+
const b = h && measured?.missingBinary ? undefined : measured;
|
|
95
|
+
const row = {
|
|
96
|
+
scenario,
|
|
97
|
+
group: (h ?? b).selector,
|
|
98
|
+
script: (h ?? b).script,
|
|
99
|
+
...(b && { base: side(b) }),
|
|
100
|
+
...(h && { head: side(h) }),
|
|
101
|
+
kind: !b ? "new" : !h ? "removed" : "same",
|
|
102
|
+
overThreshold: false,
|
|
103
|
+
};
|
|
104
|
+
if (measured?.missingBinary)
|
|
105
|
+
row.notCompared = `binary "${measured.missingBinary}" is not in the base checkout`;
|
|
106
|
+
if (b && h) {
|
|
107
|
+
row.delta = h.cycles - b.cycles;
|
|
108
|
+
const changed = resultLabel(b) !== resultLabel(h);
|
|
109
|
+
row.kind = changed ? "result" : row.delta !== 0 ? "changed" : "same";
|
|
110
|
+
// A run the VM stopped has no cost to compare: no percentage, no growth.
|
|
111
|
+
if (!b.error && !h.error) {
|
|
112
|
+
if (b.cycles > 0)
|
|
113
|
+
row.percent = (row.delta / b.cycles) * 100;
|
|
114
|
+
row.overThreshold =
|
|
115
|
+
threshold !== undefined && row.delta > 0 && (row.percent === undefined || row.percent > threshold);
|
|
116
|
+
}
|
|
117
|
+
// The VM stopped the head (past the cycle limit, a bad instruction): its 0
|
|
118
|
+
// cycles are not a saving. Under --fail-above that is over any threshold.
|
|
119
|
+
if (h.error && changed && threshold !== undefined)
|
|
120
|
+
row.overThreshold = true;
|
|
121
|
+
}
|
|
122
|
+
else if (h && resultLabel(h) !== "ok") {
|
|
123
|
+
// A new group that does not accept: against "did not run", its result changed.
|
|
124
|
+
row.kind = "result";
|
|
125
|
+
row.base = { result: NOT_RUN };
|
|
126
|
+
if (h.error && threshold !== undefined)
|
|
127
|
+
row.overThreshold = true;
|
|
128
|
+
}
|
|
129
|
+
if (h?.functions)
|
|
130
|
+
row.functions = functionDeltas(b?.functions, h.functions);
|
|
131
|
+
if (h?.functionsWarning)
|
|
132
|
+
row.functionsWarning = h.functionsWarning;
|
|
133
|
+
return row;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* The functions whose self cycles differ between two lists, largest change
|
|
137
|
+
* first. One whose other side is not known (see FunctionDelta) is kept, ranked
|
|
138
|
+
* by the side that is.
|
|
139
|
+
*/
|
|
140
|
+
export function functionDeltas(base, head) {
|
|
141
|
+
const side = (list, name) => {
|
|
142
|
+
if (!list)
|
|
143
|
+
return undefined;
|
|
144
|
+
const f = list.find((f) => f.name === name);
|
|
145
|
+
return f ? f.self : list.length < FUNCTION_LIMIT ? 0 : undefined;
|
|
146
|
+
};
|
|
147
|
+
const names = [...new Set([...head.map((f) => f.name), ...(base ?? []).map((f) => f.name)])];
|
|
148
|
+
const size = (d) => Math.abs(d.delta ?? d.head ?? d.base ?? 0);
|
|
149
|
+
return names
|
|
150
|
+
.map((name) => {
|
|
151
|
+
const b = side(base, name);
|
|
152
|
+
const h = side(head, name);
|
|
153
|
+
return { name, ...(b !== undefined && { base: b }), ...(h !== undefined && { head: h }), ...(b !== undefined && h !== undefined && { delta: h - b }) };
|
|
154
|
+
})
|
|
155
|
+
.filter((d) => d.delta !== 0)
|
|
156
|
+
.sort((x, y) => size(y) - size(x) || (x.name < y.name ? -1 : x.name > y.name ? 1 : 0));
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Whether the diff should fail a check, per the options it was made with. A
|
|
160
|
+
* scenario the head could not run always fails. With a gate set (either
|
|
161
|
+
* option), so does anything that was not checked: a scenario the base could not
|
|
162
|
+
* run, an error in either report, and a base group with nothing to compare
|
|
163
|
+
* against (removed, renamed, moved, or measured without its binary) unless
|
|
164
|
+
* `allowRemoved`. A head group the VM stopped is a result change, and over any
|
|
165
|
+
* `failAbove` threshold.
|
|
166
|
+
*/
|
|
167
|
+
export function diffFailed(diff, options = {}) {
|
|
168
|
+
const gated = options.failAbove !== undefined || options.failOnResultChange === true;
|
|
169
|
+
return ((options.failAbove !== undefined && diff.overThreshold > 0) ||
|
|
170
|
+
(options.failOnResultChange === true && diff.resultChanges > 0) ||
|
|
171
|
+
(gated && diff.baseErrors > 0) ||
|
|
172
|
+
(gated && (diff.reportErrors?.length ?? 0) > 0) ||
|
|
173
|
+
(gated && !options.allowRemoved && (diff.removed ?? 0) > 0) ||
|
|
174
|
+
diff.rows.some((r) => r.kind === "error"));
|
|
175
|
+
}
|
|
176
|
+
const fmt = (n) => n.toLocaleString("en-US");
|
|
177
|
+
const signed = (n) => (n > 0 ? `+${fmt(n)}` : n < 0 ? `-${fmt(-n)}` : "0");
|
|
178
|
+
const pct = (p) => p === undefined ? "" : p === 0 ? "0%" : `${p > 0 ? "+" : "-"}${Math.abs(p) < 0.01 ? "<0.01" : Math.abs(p).toFixed(2)}%`;
|
|
179
|
+
const cell = (s) => !s || s.result === NOT_RUN ? "—" : s.result === "ok" ? fmt(s.cycles ?? 0) : `${fmt(s.cycles ?? 0)} (${s.result})`;
|
|
180
|
+
function note(r, threshold) {
|
|
181
|
+
const unpaired = r.notCompared ? ` (NOT COMPARED: ${r.notCompared})` : "";
|
|
182
|
+
switch (r.kind) {
|
|
183
|
+
case "result":
|
|
184
|
+
return r.base.result === NOT_RUN
|
|
185
|
+
? `NEW, DOES NOT ACCEPT: ${r.head.result}${unpaired}`
|
|
186
|
+
: `RESULT CHANGED: ${r.base.result} → ${r.head.result}`;
|
|
187
|
+
case "new":
|
|
188
|
+
return r.notCompared ? `NOT COMPARED: ${r.notCompared}` : "new";
|
|
189
|
+
case "removed":
|
|
190
|
+
return "REMOVED: no head group to compare with";
|
|
191
|
+
case "error":
|
|
192
|
+
return r.note ?? "error";
|
|
193
|
+
case "base-error":
|
|
194
|
+
return `NOT COMPARED: ${r.note ?? "base: error"}`;
|
|
195
|
+
default:
|
|
196
|
+
return r.overThreshold ? `over ${threshold}%` : "";
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* One line saying what matters, for the top of a comment or terminal output.
|
|
201
|
+
* Text from the reports in it (debugger versions) goes through `clean`:
|
|
202
|
+
* termLine by default, the markdown escape in a comment.
|
|
203
|
+
*/
|
|
204
|
+
export function summarize(diff, clean = termLine) {
|
|
205
|
+
const groups = diff.rows.filter((r) => r.kind !== "error" && r.kind !== "base-error");
|
|
206
|
+
const changed = groups.filter((r) => r.kind !== "same").length;
|
|
207
|
+
const errors = diff.rows.filter((r) => r.kind === "error").length;
|
|
208
|
+
// A new group has no base, so no percentage: the threshold cannot apply to it.
|
|
209
|
+
const unchecked = diff.threshold === undefined ? 0 : groups.filter((r) => !r.base || r.base.result === NOT_RUN).length;
|
|
210
|
+
const s = (n) => (n > 1 ? "s" : "");
|
|
211
|
+
const parts = [
|
|
212
|
+
...(diff.resultChanges ? [`${diff.resultChanges} result change${s(diff.resultChanges)}`] : []),
|
|
213
|
+
...(diff.overThreshold ? [`${diff.overThreshold} over the ${diff.threshold}% threshold`] : []),
|
|
214
|
+
...(errors ? [`${errors} scenario${s(errors)} failed to run`] : []),
|
|
215
|
+
...(diff.baseErrors ? [`${diff.baseErrors} scenario${s(diff.baseErrors)} could not run on the base`] : []),
|
|
216
|
+
`${changed} of ${groups.length} script group${groups.length === 1 ? "" : "s"} changed`,
|
|
217
|
+
...(unchecked
|
|
218
|
+
? [`${unchecked} new group${s(unchecked)} not checked against the ${diff.threshold}% threshold (no base to compare with)`]
|
|
219
|
+
: diff.newGroups
|
|
220
|
+
? [`${diff.newGroups} new group${s(diff.newGroups)}`]
|
|
221
|
+
: []),
|
|
222
|
+
...(diff.removed
|
|
223
|
+
? [`${diff.removed} base group${s(diff.removed)} with nothing in the head to compare with (removed, renamed, moved or not built on the base)`]
|
|
224
|
+
: []),
|
|
225
|
+
...(diff.reportErrors?.length ? [`${diff.reportErrors.length} report error${s(diff.reportErrors.length)}`] : []),
|
|
226
|
+
...(diff.debuggers
|
|
227
|
+
? [`the reports come from different debuggers (base ${clean(diff.debuggers.base)}, head ${clean(diff.debuggers.head)}): cycles can differ for that alone`]
|
|
228
|
+
: []),
|
|
229
|
+
];
|
|
230
|
+
return parts.join(", ");
|
|
231
|
+
}
|
|
232
|
+
const MD_ESCAPE = /[|\\`*_<>[\]~&!]/g;
|
|
233
|
+
/**
|
|
234
|
+
* Characters that never belong in a name: C0/C1 controls, and the bidi overrides
|
|
235
|
+
* and isolates that make a name read differently from what it is (review, 27 Oct).
|
|
236
|
+
*/
|
|
237
|
+
const INVISIBLE = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f-\u009f\u202a-\u202e\u2066-\u2069]/g;
|
|
238
|
+
/**
|
|
239
|
+
* Applied after escaping. GitHub links @mentions and #refs after rendering, where a
|
|
240
|
+
* backslash is gone, and turns a bare `https://\u2026` or `www.\u2026` into a link: a
|
|
241
|
+
* zero-width space stops each of them.
|
|
242
|
+
*/
|
|
243
|
+
const unlink = (s) => s
|
|
244
|
+
.replace(/([@#])(?=\w)/g, "$1\u200b")
|
|
245
|
+
.replace(/:\/\//g, ":\u200b//")
|
|
246
|
+
.replace(/\b(www)\./gi, "$1\u200b.");
|
|
247
|
+
/**
|
|
248
|
+
* The longest scenario name, script name or note a comment shows: one long
|
|
249
|
+
* name repeated on every row would pass GitHub's size limit on its own.
|
|
250
|
+
*/
|
|
251
|
+
const TEXT_MAX = 300;
|
|
252
|
+
const oneLine = (s) => {
|
|
253
|
+
const t = String(s).replace(/[\r\n]+/g, " ").replace(INVISIBLE, "");
|
|
254
|
+
return t.length > TEXT_MAX ? `${t.slice(0, TEXT_MAX - 1)}\u2026` : t;
|
|
255
|
+
};
|
|
256
|
+
/**
|
|
257
|
+
* Text from a config, a build or a node (scenario, script and function names,
|
|
258
|
+
* warnings) for a terminal or a CI log: oneLine, and never `::` at the start of
|
|
259
|
+
* a line, which GitHub Actions would run as a workflow command (`::error::`,
|
|
260
|
+
* `::add-mask::`, `::stop-commands::`); a zero-width space breaks it.
|
|
261
|
+
*/
|
|
262
|
+
export const termLine = (s) => oneLine(s).replace(/^(\s*):(?=:)/, "$1:\u200b");
|
|
263
|
+
/** termLine for text that keeps its line breaks (a debugger's output in an error): each line cleaned the same way. */
|
|
264
|
+
export const termLines = (s) => String(s).split(/\r?\n|\r/).map(termLine).join("\n");
|
|
265
|
+
/**
|
|
266
|
+
* Text from the PR's own files (scenario and script names, the title, error
|
|
267
|
+
* messages) as inert markdown: one clipped line, no table breaks, no
|
|
268
|
+
* formatting, no mentions, no links.
|
|
269
|
+
*/
|
|
270
|
+
const md = (s) => unlink(oneLine(s).replace(MD_ESCAPE, (c) => `\\${c}`));
|
|
271
|
+
/** Text for inside an HTML tag (a `<summary>`, which GitHub does not read as markdown). */
|
|
272
|
+
const html = (s) => unlink(oneLine(s)
|
|
273
|
+
.replace(/&/g, "&")
|
|
274
|
+
.replace(/</g, "<")
|
|
275
|
+
.replace(/>/g, ">"));
|
|
276
|
+
/**
|
|
277
|
+
* A code span. A selector or a commit label is plain (`input.0.lock`); anything
|
|
278
|
+
* else (a hand-made report) goes in `<code>`, escaped like any text, where a
|
|
279
|
+
* backtick cannot end it early.
|
|
280
|
+
*/
|
|
281
|
+
const code = (s) => (/^[\w.:-]+$/.test(s) ? `\`${s}\`` : `<code>${md(s)}</code>`);
|
|
282
|
+
/** How many changed functions a comment lists per group. */
|
|
283
|
+
export const FUNCTIONS_SHOWN = 10;
|
|
284
|
+
/** The longest function name a comment shows; Rust generics can run to kilobytes. */
|
|
285
|
+
const NAME_MAX = 120;
|
|
286
|
+
const clipName = (name) => (name.length > NAME_MAX ? `${name.slice(0, NAME_MAX - 1)}…` : name);
|
|
287
|
+
/**
|
|
288
|
+
* GitHub refuses a comment over 65,536 characters, and the Action would then
|
|
289
|
+
* leave the previous push's comment in place (review, 27 Oct). Function blocks
|
|
290
|
+
* are dropped first when the comment would not fit.
|
|
291
|
+
*/
|
|
292
|
+
const COMMENT_MAX = 60_000;
|
|
293
|
+
/** A folded block per group that recorded functions: which of them changed, or why none were recorded. */
|
|
294
|
+
function functionBlocks(rows) {
|
|
295
|
+
const out = [];
|
|
296
|
+
for (const r of rows) {
|
|
297
|
+
const where = `${html(r.scenario)} · <code>${html(r.group)}</code> (${html(r.script)})`;
|
|
298
|
+
if (r.functionsWarning) {
|
|
299
|
+
out.push(`<details><summary>Functions in ${where}: not recorded</summary>`, "", md(r.functionsWarning), "", "</details>", "");
|
|
300
|
+
continue;
|
|
301
|
+
}
|
|
302
|
+
if (!r.functions?.length)
|
|
303
|
+
continue;
|
|
304
|
+
const shown = r.functions.slice(0, FUNCTIONS_SHOWN);
|
|
305
|
+
const more = r.functions.length - shown.length;
|
|
306
|
+
const n = r.functions.length;
|
|
307
|
+
out.push(`<details><summary>Functions in ${where}: ${n} function${n === 1 ? "" : "s"} changed</summary>`, "", "| Function | Base self | Head self | Δ |", "|---|---:|---:|---:|", ...shown.map((f) => `| ${md(clipName(f.name))} | ${fnCell(f.base)} | ${fnCell(f.head)} | ${f.delta === undefined ? "" : signed(f.delta)} |`), "", ...(more > 0 ? [`and ${more} more.`, ""] : []), `<sub>Self cycles, from ckb-debugger's profiler with the symbols build; — means not among the ${FUNCTION_LIMIT} costliest (or not recorded).</sub>`, "", "</details>", "");
|
|
308
|
+
}
|
|
309
|
+
return out;
|
|
310
|
+
}
|
|
311
|
+
const fnCell = (n) => (n === undefined ? "—" : fmt(n));
|
|
312
|
+
/** The hidden marker that identifies the sticky comment, so it is updated instead of duplicated. */
|
|
313
|
+
export const COMMENT_MARKER = "<!-- ckb-cycles-report -->";
|
|
314
|
+
/**
|
|
315
|
+
* A markdown comment: what changed in a table, unchanged groups folded away.
|
|
316
|
+
* Result changes come first, since an accept turning into a reject (or the
|
|
317
|
+
* reverse) matters more than any cycle count.
|
|
318
|
+
*/
|
|
319
|
+
export function formatMarkdown(diff, options = {}) {
|
|
320
|
+
const order = { error: 0, "base-error": 0.5, result: 1, changed: 2, new: 3, removed: 4, same: 5 };
|
|
321
|
+
const rank = (r) => order[r.kind] - (r.overThreshold ? 0.5 : 0);
|
|
322
|
+
const rows = [...diff.rows].sort((a, b) => rank(a) - rank(b));
|
|
323
|
+
const shown = rows.filter((r) => r.kind !== "same");
|
|
324
|
+
const same = rows.filter((r) => r.kind === "same");
|
|
325
|
+
const head = "| Scenario | Group | Script | Base | Head | Δ | Δ% | |\n|---|---|---|---:|---:|---:|---:|---|";
|
|
326
|
+
const line = (r) => {
|
|
327
|
+
const flag = note(r, diff.threshold);
|
|
328
|
+
const strong = r.kind === "result" || r.kind === "error" || r.kind === "base-error" || r.kind === "removed" || r.overThreshold || !!r.notCompared;
|
|
329
|
+
return (`| ${md(r.scenario)} | ${r.group ? code(r.group) : ""} | ${md(r.script)} | ${md(cell(r.base))} | ${md(cell(r.head))}` +
|
|
330
|
+
` | ${r.delta === undefined ? "" : signed(r.delta)} | ${pct(r.percent)} | ${flag ? (strong ? `**${md(flag)}**` : md(flag)) : ""} |`);
|
|
331
|
+
};
|
|
332
|
+
const vs = options.baseLabel && options.headLabel ? ` (${code(options.baseLabel)} → ${code(options.headLabel)})` : "";
|
|
333
|
+
const top = [COMMENT_MARKER, `### ${md(options.title ?? "Cycle report")}`, "", `${summarize(diff, md)}${vs}.`, ""];
|
|
334
|
+
// A report's own errors fail a gated check: they go right under the summary.
|
|
335
|
+
const errors = diff.reportErrors?.length
|
|
336
|
+
? [`**Report errors** (nothing measured what they name):`, "", ...diff.reportErrors.slice(0, 20).map((e) => `- ${md(e)}`), ...(diff.reportErrors.length > 20 ? [`- and ${diff.reportErrors.length - 20} more`] : []), ""]
|
|
337
|
+
: [];
|
|
338
|
+
top.push(...errors);
|
|
339
|
+
const footer = ["<sub>Generated by ckb-cycles (ckb-cycle-tools): each scenario replayed in ckb-debugger, one run per script group.</sub>"];
|
|
340
|
+
let table = shown.length ? [head, ...shown.map(line), ""] : ["No script group changed its cycles or its result.", ""];
|
|
341
|
+
let blocks = functionBlocks(shown);
|
|
342
|
+
let warnings = diff.warnings?.length ? warningBlock(diff.warnings) : [];
|
|
343
|
+
let unchanged = same.length
|
|
344
|
+
? [
|
|
345
|
+
`<details><summary>${same.length} unchanged group${same.length === 1 ? "" : "s"}</summary>`,
|
|
346
|
+
"",
|
|
347
|
+
"| Scenario | Group | Script | Cycles | Result |",
|
|
348
|
+
"|---|---|---|---:|---|",
|
|
349
|
+
...same.map((r) => `| ${md(r.scenario)} | ${code(r.group)} | ${md(r.script)} | ${fmt(r.head.cycles ?? 0)} | ${md(r.head.result)} |`),
|
|
350
|
+
"",
|
|
351
|
+
"</details>",
|
|
352
|
+
"",
|
|
353
|
+
]
|
|
354
|
+
: [];
|
|
355
|
+
// Over GitHub's limit the comment is refused, and the previous push's comment
|
|
356
|
+
// would stay up with old numbers. Least useful first: the function details,
|
|
357
|
+
// the unchanged groups, then the last (least important) changed rows.
|
|
358
|
+
const size = () => [...top, ...table, ...blocks, ...warnings, ...unchanged, ...footer].join("\n").length;
|
|
359
|
+
const s = (n) => (n === 1 ? "" : "s");
|
|
360
|
+
if (size() >= COMMENT_MAX && warnings.length) {
|
|
361
|
+
warnings = [`<sub>${diff.warnings.length} scenario warning${s(diff.warnings.length)} left out: the comment would pass GitHub's size limit.</sub>`, ""];
|
|
362
|
+
}
|
|
363
|
+
if (size() >= COMMENT_MAX && blocks.length) {
|
|
364
|
+
blocks = ["<sub>Function details left out: the comment would pass GitHub's size limit.</sub>", ""];
|
|
365
|
+
}
|
|
366
|
+
if (size() >= COMMENT_MAX && same.length) {
|
|
367
|
+
unchanged = [`<sub>${same.length} unchanged group${s(same.length)} left out: the comment would pass GitHub's size limit.</sub>`, ""];
|
|
368
|
+
}
|
|
369
|
+
if (size() >= COMMENT_MAX && shown.length) {
|
|
370
|
+
const rows = table.slice(2, -1);
|
|
371
|
+
const room = COMMENT_MAX - size() + rows.join("\n").length - 200;
|
|
372
|
+
let kept = 0;
|
|
373
|
+
for (let used = 0; kept < rows.length && used + rows[kept].length + 1 < room; kept++)
|
|
374
|
+
used += rows[kept].length + 1;
|
|
375
|
+
const cut = rows.length - kept;
|
|
376
|
+
table = [...table.slice(0, 2), ...rows.slice(0, kept), "", `<sub>${cut} more changed row${s(cut)} left out: the comment would pass GitHub's size limit.</sub>`, ""];
|
|
377
|
+
}
|
|
378
|
+
return [...top, ...table, ...blocks, ...warnings, ...unchanged, ...footer].join("\n");
|
|
379
|
+
}
|
|
380
|
+
/** How many scenario warnings a comment lists. */
|
|
381
|
+
const WARNINGS_SHOWN = 50;
|
|
382
|
+
/** The head's scenario warnings, folded: why a number may not be the build's (a binary no cell dep matched). */
|
|
383
|
+
function warningBlock(warnings) {
|
|
384
|
+
const n = warnings.length;
|
|
385
|
+
const shown = warnings.slice(0, WARNINGS_SHOWN);
|
|
386
|
+
return [
|
|
387
|
+
`<details><summary>${n} scenario warning${n === 1 ? "" : "s"}</summary>`,
|
|
388
|
+
"",
|
|
389
|
+
...shown.map((w) => `- ${md(w.scenario)}: ${md(w.text)}`),
|
|
390
|
+
...(n > shown.length ? [`- and ${n - shown.length} more`] : []),
|
|
391
|
+
"",
|
|
392
|
+
"</details>",
|
|
393
|
+
"",
|
|
394
|
+
];
|
|
395
|
+
}
|
|
396
|
+
/** The same table for a terminal. */
|
|
397
|
+
export function formatText(diff) {
|
|
398
|
+
const header = ["SCENARIO", "GROUP", "SCRIPT", "BASE", "HEAD", "Δ", "Δ%", ""];
|
|
399
|
+
const rows = diff.rows.map((r) => [
|
|
400
|
+
r.scenario,
|
|
401
|
+
r.group,
|
|
402
|
+
r.script,
|
|
403
|
+
cell(r.base),
|
|
404
|
+
cell(r.head),
|
|
405
|
+
r.delta === undefined ? "" : signed(r.delta),
|
|
406
|
+
pct(r.percent),
|
|
407
|
+
note(r, diff.threshold),
|
|
408
|
+
].map(termLine));
|
|
409
|
+
const widths = header.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)));
|
|
410
|
+
const right = new Set([3, 4, 5, 6]);
|
|
411
|
+
const line = (cells) => cells
|
|
412
|
+
.map((c, i) => (right.has(i) ? c.padStart(widths[i]) : c.padEnd(widths[i])))
|
|
413
|
+
.join(" ")
|
|
414
|
+
.trimEnd();
|
|
415
|
+
// Under the table, one line per group that recorded functions: the largest changes.
|
|
416
|
+
const functions = [];
|
|
417
|
+
for (const r of diff.rows.filter((r) => r.kind !== "same")) {
|
|
418
|
+
const where = `${termLine(r.scenario)} ${termLine(r.group)}`;
|
|
419
|
+
if (r.functionsWarning)
|
|
420
|
+
functions.push(`functions in ${where} not recorded: ${termLine(r.functionsWarning)}`);
|
|
421
|
+
else if (r.functions?.length) {
|
|
422
|
+
const top = r.functions
|
|
423
|
+
.slice(0, 5)
|
|
424
|
+
.map((f) => `${termLine(f.name)} ${f.delta !== undefined ? signed(f.delta) : `${fnText(f.base)} → ${fnText(f.head)}`}`);
|
|
425
|
+
const more = r.functions.length - top.length;
|
|
426
|
+
functions.push(`functions in ${where}: ${top.join(", ")}${more > 0 ? `, and ${more} more` : ""}`);
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
const errors = (diff.reportErrors ?? []).map((e) => `error: ${termLine(e)}`);
|
|
430
|
+
const warnings = (diff.warnings ?? []).map((w) => `${termLine(w.scenario)}: warning: ${termLine(w.text)}`);
|
|
431
|
+
return [
|
|
432
|
+
summarize(diff),
|
|
433
|
+
...(errors.length ? ["", ...errors] : []),
|
|
434
|
+
"",
|
|
435
|
+
line(header),
|
|
436
|
+
...rows.map(line),
|
|
437
|
+
...(functions.length ? ["", ...functions] : []),
|
|
438
|
+
...(warnings.length ? ["", ...warnings] : []),
|
|
439
|
+
].join("\n");
|
|
440
|
+
}
|
|
441
|
+
const fnText = (n) => (n === undefined ? "?" : fmt(n));
|