@blundergoat/gruff-ts 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/CONTRIBUTING.md +87 -0
- package/LICENSE +21 -0
- package/README.md +303 -0
- package/SECURITY.md +45 -0
- package/bin/gruff-ts +25 -0
- package/docs/CONFIGURATION.md +220 -0
- package/docs/RELEASING.md +103 -0
- package/docs/REPORTS_AND_CI.md +156 -0
- package/fixtures/sample.ts +21 -0
- package/package.json +56 -0
- package/scripts/bump-version.sh +145 -0
- package/scripts/check.sh +4 -0
- package/scripts/npm-publish.sh +258 -0
- package/scripts/preflight-checks.sh +357 -0
- package/scripts/start-dev.sh +8 -0
- package/scripts/test-performance.sh +695 -0
- package/src/analyser.ts +461 -0
- package/src/baseline.ts +90 -0
- package/src/blocks.ts +687 -0
- package/src/class-rules.ts +326 -0
- package/src/cli-program.ts +326 -0
- package/src/cli.ts +19 -0
- package/src/comment-rules.ts +605 -0
- package/src/comment-scanner.ts +357 -0
- package/src/config.ts +622 -0
- package/src/constants.ts +4 -0
- package/src/context-doc-rules.ts +241 -0
- package/src/dashboard.ts +114 -0
- package/src/dead-code-rules.ts +183 -0
- package/src/discovery.ts +508 -0
- package/src/doc-rules.ts +368 -0
- package/src/findings-helpers.ts +108 -0
- package/src/findings.ts +45 -0
- package/src/fixture-purpose-rules.ts +334 -0
- package/src/fixtures/rule-catalogue-security-doctrine.ts +132 -0
- package/src/github-actions-rules.ts +413 -0
- package/src/line-rules.ts +538 -0
- package/src/naming-pushers.ts +191 -0
- package/src/project-config-rules.ts +555 -0
- package/src/project-rules.ts +545 -0
- package/src/report-renderers.ts +691 -0
- package/src/rule-list.ts +179 -0
- package/src/rules.ts +135 -0
- package/src/safety-rules.ts +355 -0
- package/src/scoring.ts +74 -0
- package/src/security-flow-rules.ts +112 -0
- package/src/sensitive-data-rules.ts +288 -0
- package/src/source-text.ts +722 -0
- package/src/test-block-rules.ts +347 -0
- package/src/test-fixtures.ts +621 -0
- package/src/text-scans.ts +193 -0
- package/src/types.ts +113 -0
- package/tsconfig.json +15 -0
|
@@ -0,0 +1,695 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
# gruff-perf.v1 is this script's baseline schema; bump only with a migration path.
|
|
5
|
+
SCHEMA_VERSION="gruff-perf.v1"
|
|
6
|
+
|
|
7
|
+
RUNS=5
|
|
8
|
+
TARGET="src"
|
|
9
|
+
OUT_PATH="/tmp/gruff-perf-$$.json"
|
|
10
|
+
QUIET=0
|
|
11
|
+
MATRIX=0
|
|
12
|
+
WRITE_BASELINE=0
|
|
13
|
+
WRITE_BASELINE_PATH=""
|
|
14
|
+
BASELINE_PATH=""
|
|
15
|
+
FAIL_ON_REGRESSION=0
|
|
16
|
+
REGRESSION_TOLERANCE="25"
|
|
17
|
+
REPORT_PATH=""
|
|
18
|
+
FORCE=0
|
|
19
|
+
CLEANUP=1
|
|
20
|
+
TIME_CMD=""
|
|
21
|
+
TMP_DIR=""
|
|
22
|
+
CURRENT_FIXTURE_DIR=""
|
|
23
|
+
|
|
24
|
+
DEFAULT_BASELINE_PATH="$(printf '%s/%s/%s\n' ".goat-flow" "scratchpad/perf" "baseline.json")"
|
|
25
|
+
|
|
26
|
+
usage() {
|
|
27
|
+
cat <<'USAGE'
|
|
28
|
+
Use scripts/test-performance.sh --help to see options.
|
|
29
|
+
|
|
30
|
+
Usage:
|
|
31
|
+
scripts/test-performance.sh [options]
|
|
32
|
+
|
|
33
|
+
Run modes:
|
|
34
|
+
--runs N Repeat count per measured cell (default: 5)
|
|
35
|
+
--target PATH Single-workload target (default: src)
|
|
36
|
+
--matrix Run tiny/self/synthetic workloads across config and format matrix
|
|
37
|
+
|
|
38
|
+
Baseline:
|
|
39
|
+
--write-baseline [PATH] Write the current matrix JSON as a baseline
|
|
40
|
+
--baseline PATH Compare/report against an existing perf baseline
|
|
41
|
+
--fail-on-regression [PCT] Exit 1 when wall time or RSS regresses above PCT (default: 25)
|
|
42
|
+
--force Allow --write-baseline to overwrite an existing file
|
|
43
|
+
|
|
44
|
+
Output:
|
|
45
|
+
--out PATH Write machine JSON to PATH (default: /tmp/gruff-perf-<pid>.json)
|
|
46
|
+
--report PATH Write the Markdown matrix report to PATH
|
|
47
|
+
--quiet Print only the JSON path
|
|
48
|
+
--cleanup Remove previous and current synthetic fixtures (default)
|
|
49
|
+
--no-cleanup Preserve synthetic fixtures for inspection
|
|
50
|
+
--help Show this help
|
|
51
|
+
USAGE
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
die() {
|
|
55
|
+
local code="$1"
|
|
56
|
+
shift
|
|
57
|
+
printf '%s\n' "$*" >&2
|
|
58
|
+
exit "$code"
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
is_positive_integer() {
|
|
62
|
+
[[ "$1" =~ ^[1-9][0-9]*$ ]]
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
is_nonnegative_number() {
|
|
66
|
+
awk -v value="$1" 'BEGIN { exit(value ~ /^[0-9]+([.][0-9]+)?$/ ? 0 : 1) }'
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
repo_root() {
|
|
70
|
+
local script_dir
|
|
71
|
+
script_dir="$(CDPATH='' cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
72
|
+
CDPATH='' cd -- "$script_dir/.." && pwd
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
safe_remove_dir() {
|
|
76
|
+
local path="$1"
|
|
77
|
+
case "$path" in
|
|
78
|
+
/tmp/gruff-perf-work-*|/tmp/gruff-perf-fixture-*)
|
|
79
|
+
rm -rf -- "$path"
|
|
80
|
+
;;
|
|
81
|
+
*)
|
|
82
|
+
die 2 "refusing to remove unexpected path: $path"
|
|
83
|
+
;;
|
|
84
|
+
esac
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
cleanup() {
|
|
88
|
+
if [[ -n "$TMP_DIR" && -d "$TMP_DIR" ]]; then
|
|
89
|
+
safe_remove_dir "$TMP_DIR"
|
|
90
|
+
fi
|
|
91
|
+
if [[ "$CLEANUP" -eq 1 && -n "$CURRENT_FIXTURE_DIR" && -d "$CURRENT_FIXTURE_DIR" ]]; then
|
|
92
|
+
safe_remove_dir "$CURRENT_FIXTURE_DIR"
|
|
93
|
+
fi
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
cleanup_old_fixtures() {
|
|
97
|
+
local fixture
|
|
98
|
+
shopt -s nullglob
|
|
99
|
+
for fixture in /tmp/gruff-perf-fixture-*; do
|
|
100
|
+
if [[ -d "$fixture" ]]; then
|
|
101
|
+
safe_remove_dir "$fixture"
|
|
102
|
+
fi
|
|
103
|
+
done
|
|
104
|
+
shopt -u nullglob
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
parse_args() {
|
|
108
|
+
while [[ "$#" -gt 0 ]]; do
|
|
109
|
+
case "$1" in
|
|
110
|
+
--runs)
|
|
111
|
+
[[ "$#" -ge 2 ]] || die 2 "--runs requires a value"
|
|
112
|
+
RUNS="$2"
|
|
113
|
+
shift
|
|
114
|
+
;;
|
|
115
|
+
--runs=*)
|
|
116
|
+
RUNS="${1#*=}"
|
|
117
|
+
;;
|
|
118
|
+
--target)
|
|
119
|
+
[[ "$#" -ge 2 ]] || die 2 "--target requires a value"
|
|
120
|
+
TARGET="$2"
|
|
121
|
+
shift
|
|
122
|
+
;;
|
|
123
|
+
--target=*)
|
|
124
|
+
TARGET="${1#*=}"
|
|
125
|
+
;;
|
|
126
|
+
--out)
|
|
127
|
+
[[ "$#" -ge 2 ]] || die 2 "--out requires a value"
|
|
128
|
+
OUT_PATH="$2"
|
|
129
|
+
shift
|
|
130
|
+
;;
|
|
131
|
+
--out=*)
|
|
132
|
+
OUT_PATH="${1#*=}"
|
|
133
|
+
;;
|
|
134
|
+
--matrix)
|
|
135
|
+
MATRIX=1
|
|
136
|
+
;;
|
|
137
|
+
--write-baseline)
|
|
138
|
+
WRITE_BASELINE=1
|
|
139
|
+
if [[ "$#" -ge 2 && "$2" != --* ]]; then
|
|
140
|
+
WRITE_BASELINE_PATH="$2"
|
|
141
|
+
shift
|
|
142
|
+
else
|
|
143
|
+
WRITE_BASELINE_PATH="$DEFAULT_BASELINE_PATH"
|
|
144
|
+
fi
|
|
145
|
+
;;
|
|
146
|
+
--write-baseline=*)
|
|
147
|
+
WRITE_BASELINE=1
|
|
148
|
+
WRITE_BASELINE_PATH="${1#*=}"
|
|
149
|
+
;;
|
|
150
|
+
--baseline)
|
|
151
|
+
[[ "$#" -ge 2 ]] || die 2 "--baseline requires a value"
|
|
152
|
+
BASELINE_PATH="$2"
|
|
153
|
+
shift
|
|
154
|
+
;;
|
|
155
|
+
--baseline=*)
|
|
156
|
+
BASELINE_PATH="${1#*=}"
|
|
157
|
+
;;
|
|
158
|
+
--fail-on-regression)
|
|
159
|
+
FAIL_ON_REGRESSION=1
|
|
160
|
+
if [[ "$#" -ge 2 && "$2" != --* ]]; then
|
|
161
|
+
REGRESSION_TOLERANCE="$2"
|
|
162
|
+
shift
|
|
163
|
+
else
|
|
164
|
+
REGRESSION_TOLERANCE="25"
|
|
165
|
+
fi
|
|
166
|
+
;;
|
|
167
|
+
--fail-on-regression=*)
|
|
168
|
+
FAIL_ON_REGRESSION=1
|
|
169
|
+
REGRESSION_TOLERANCE="${1#*=}"
|
|
170
|
+
;;
|
|
171
|
+
--report)
|
|
172
|
+
[[ "$#" -ge 2 ]] || die 2 "--report requires a value"
|
|
173
|
+
REPORT_PATH="$2"
|
|
174
|
+
shift
|
|
175
|
+
;;
|
|
176
|
+
--report=*)
|
|
177
|
+
REPORT_PATH="${1#*=}"
|
|
178
|
+
;;
|
|
179
|
+
--quiet)
|
|
180
|
+
QUIET=1
|
|
181
|
+
;;
|
|
182
|
+
--force)
|
|
183
|
+
FORCE=1
|
|
184
|
+
;;
|
|
185
|
+
--cleanup)
|
|
186
|
+
CLEANUP=1
|
|
187
|
+
;;
|
|
188
|
+
--no-cleanup)
|
|
189
|
+
CLEANUP=0
|
|
190
|
+
;;
|
|
191
|
+
--help|-h)
|
|
192
|
+
usage
|
|
193
|
+
exit 0
|
|
194
|
+
;;
|
|
195
|
+
*)
|
|
196
|
+
die 2 "unknown option: $1"
|
|
197
|
+
;;
|
|
198
|
+
esac
|
|
199
|
+
shift
|
|
200
|
+
done
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
validate_args() {
|
|
204
|
+
is_positive_integer "$RUNS" || die 2 "--runs must be a positive integer"
|
|
205
|
+
if [[ "$FAIL_ON_REGRESSION" -eq 1 ]]; then
|
|
206
|
+
is_nonnegative_number "$REGRESSION_TOLERANCE" || die 2 "--fail-on-regression must be a non-negative number"
|
|
207
|
+
fi
|
|
208
|
+
if [[ "$WRITE_BASELINE" -eq 1 && "$MATRIX" -ne 1 ]]; then
|
|
209
|
+
die 2 "--write-baseline requires --matrix"
|
|
210
|
+
fi
|
|
211
|
+
if [[ -n "$BASELINE_PATH" && "$MATRIX" -ne 1 ]]; then
|
|
212
|
+
die 2 "--baseline requires --matrix"
|
|
213
|
+
fi
|
|
214
|
+
if [[ "$FAIL_ON_REGRESSION" -eq 1 && -z "$BASELINE_PATH" ]]; then
|
|
215
|
+
die 2 "--fail-on-regression requires --baseline"
|
|
216
|
+
fi
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
require_tools() {
|
|
220
|
+
command -v jq >/dev/null 2>&1 || die 2 "jq is required"
|
|
221
|
+
command -v awk >/dev/null 2>&1 || die 2 "awk is required"
|
|
222
|
+
command -v node >/dev/null 2>&1 || die 2 "node is required"
|
|
223
|
+
[[ -f "./bin/gruff-ts" ]] || die 2 "missing ./bin/gruff-ts"
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
detect_time_cmd() {
|
|
227
|
+
local probe_file="$TMP_DIR/time-probe.txt"
|
|
228
|
+
if [[ -x /usr/bin/time ]] && /usr/bin/time -v bash -c 'exit 0' >/dev/null 2>"$probe_file"; then
|
|
229
|
+
TIME_CMD="/usr/bin/time"
|
|
230
|
+
return
|
|
231
|
+
fi
|
|
232
|
+
if command -v gtime >/dev/null 2>&1 && gtime -v bash -c 'exit 0' >/dev/null 2>"$probe_file"; then
|
|
233
|
+
TIME_CMD="gtime"
|
|
234
|
+
return
|
|
235
|
+
fi
|
|
236
|
+
die 2 "GNU time is required; install gnu-time so /usr/bin/time -v or gtime -v is available"
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
elapsed_to_seconds() {
|
|
240
|
+
awk -v elapsed="$1" '
|
|
241
|
+
BEGIN {
|
|
242
|
+
count = split(elapsed, parts, ":")
|
|
243
|
+
if (count == 2) {
|
|
244
|
+
printf "%.6f\n", (parts[1] * 60) + parts[2]
|
|
245
|
+
} else if (count == 3) {
|
|
246
|
+
printf "%.6f\n", (parts[1] * 3600) + (parts[2] * 60) + parts[3]
|
|
247
|
+
} else {
|
|
248
|
+
exit 1
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
'
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
assert_elapsed_parser() {
|
|
255
|
+
local one
|
|
256
|
+
local two
|
|
257
|
+
one="$(elapsed_to_seconds "0:01.50")" || die 2 "elapsed parser rejected m:ss sample"
|
|
258
|
+
two="$(elapsed_to_seconds "1:02:03.25")" || die 2 "elapsed parser rejected h:mm:ss sample"
|
|
259
|
+
[[ "$one" == "1.500000" ]] || die 2 "elapsed parser returned $one for 0:01.50"
|
|
260
|
+
[[ "$two" == "3723.250000" ]] || die 2 "elapsed parser returned $two for 1:02:03.25"
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
parse_elapsed_file() {
|
|
264
|
+
local file="$1"
|
|
265
|
+
local elapsed
|
|
266
|
+
elapsed="$(awk -F': ' '/Elapsed \(wall clock\) time/ { print $NF; found = 1; exit } END { if (!found) exit 1 }' "$file")" \
|
|
267
|
+
|| die 2 "could not parse elapsed wall-clock time from $file"
|
|
268
|
+
elapsed_to_seconds "$elapsed" || die 2 "could not convert elapsed wall-clock time: $elapsed"
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
parse_rss_file() {
|
|
272
|
+
local file="$1"
|
|
273
|
+
awk -F': ' '/Maximum resident set size/ { print $NF; found = 1; exit } END { if (!found) exit 1 }' "$file" \
|
|
274
|
+
|| die 2 "could not parse max RSS from $file"
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
parse_text_findings() {
|
|
278
|
+
local file="$1"
|
|
279
|
+
awk '
|
|
280
|
+
/Findings:/ {
|
|
281
|
+
line = $0
|
|
282
|
+
sub(/^.*Findings: /, "", line)
|
|
283
|
+
total = 0
|
|
284
|
+
while (match(line, /[0-9]+/)) {
|
|
285
|
+
total += substr(line, RSTART, RLENGTH)
|
|
286
|
+
line = substr(line, RSTART + RLENGTH)
|
|
287
|
+
}
|
|
288
|
+
print total
|
|
289
|
+
found = 1
|
|
290
|
+
exit
|
|
291
|
+
}
|
|
292
|
+
END { if (!found) exit 1 }
|
|
293
|
+
' "$file" || die 2 "could not parse finding count from text output"
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
parse_findings() {
|
|
297
|
+
local format="$1"
|
|
298
|
+
local file="$2"
|
|
299
|
+
case "$format" in
|
|
300
|
+
json)
|
|
301
|
+
jq -e '.findings | length' "$file" || die 2 "could not parse finding count from JSON output"
|
|
302
|
+
;;
|
|
303
|
+
text)
|
|
304
|
+
parse_text_findings "$file"
|
|
305
|
+
;;
|
|
306
|
+
*)
|
|
307
|
+
die 2 "unsupported format: $format"
|
|
308
|
+
;;
|
|
309
|
+
esac
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
aggregate_samples() {
|
|
313
|
+
local file="$1"
|
|
314
|
+
awk '
|
|
315
|
+
{
|
|
316
|
+
value = $1 + 0
|
|
317
|
+
values[NR] = value
|
|
318
|
+
sum += value
|
|
319
|
+
if (NR == 1 || value < min) {
|
|
320
|
+
min = value
|
|
321
|
+
}
|
|
322
|
+
if (NR == 1 || value > max) {
|
|
323
|
+
max = value
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
END {
|
|
327
|
+
if (NR == 0) {
|
|
328
|
+
exit 1
|
|
329
|
+
}
|
|
330
|
+
mean = sum / NR
|
|
331
|
+
for (i = 1; i <= NR; i++) {
|
|
332
|
+
diff = values[i] - mean
|
|
333
|
+
variance += diff * diff
|
|
334
|
+
samples = samples (i == 1 ? "" : ",") sprintf("%.6f", values[i])
|
|
335
|
+
}
|
|
336
|
+
stddev = sqrt(variance / NR)
|
|
337
|
+
printf "{\"mean\":%.6f,\"min\":%.6f,\"max\":%.6f,\"stddev\":%.6f,\"samples\":[%s]}\n", mean, min, max, stddev, samples
|
|
338
|
+
}
|
|
339
|
+
' "$file" || die 2 "could not aggregate samples from $file"
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
deterministic_finding_count() {
|
|
343
|
+
local file="$1"
|
|
344
|
+
awk '
|
|
345
|
+
NR == 1 { first = $1 }
|
|
346
|
+
$1 != first { deterministic = 0 }
|
|
347
|
+
NR == 1 { deterministic = 1 }
|
|
348
|
+
END {
|
|
349
|
+
if (NR == 0 || !deterministic) {
|
|
350
|
+
exit 1
|
|
351
|
+
}
|
|
352
|
+
print first
|
|
353
|
+
}
|
|
354
|
+
' "$file" || die 2 "finding count changed across runs"
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
run_cell() {
|
|
358
|
+
local workload="$1"
|
|
359
|
+
local target_path="$2"
|
|
360
|
+
local config_label="$3"
|
|
361
|
+
local config_flag="$4"
|
|
362
|
+
local format="$5"
|
|
363
|
+
local cell_name="${workload//[^[:alnum:]._-]/_}-${config_label//[^[:alnum:]._-]/_}-${format//[^[:alnum:]._-]/_}"
|
|
364
|
+
local cell_tmp="$TMP_DIR/cell-${cell_name}"
|
|
365
|
+
local wall_file="$cell_tmp-wall.txt"
|
|
366
|
+
local rss_file="$cell_tmp-rss.txt"
|
|
367
|
+
local findings_file="$cell_tmp-findings.txt"
|
|
368
|
+
local run
|
|
369
|
+
local run_output
|
|
370
|
+
local run_time
|
|
371
|
+
local elapsed_seconds
|
|
372
|
+
local rss_kb
|
|
373
|
+
local finding_count
|
|
374
|
+
local wall_json
|
|
375
|
+
local rss_json
|
|
376
|
+
local deterministic_count
|
|
377
|
+
local args
|
|
378
|
+
|
|
379
|
+
[[ -e "$target_path" ]] || die 2 "target does not exist: $target_path"
|
|
380
|
+
|
|
381
|
+
: > "$wall_file"
|
|
382
|
+
: > "$rss_file"
|
|
383
|
+
: > "$findings_file"
|
|
384
|
+
|
|
385
|
+
# Warmup policy: include every run; the src spike stayed below 25% variance without dropping sample 1.
|
|
386
|
+
for ((run = 1; run <= RUNS; run++)); do
|
|
387
|
+
run_output="$cell_tmp-run-$run.out"
|
|
388
|
+
run_time="$cell_tmp-run-$run.time"
|
|
389
|
+
args=(analyse "$target_path" "--format=$format" --no-baseline --fail-on=none)
|
|
390
|
+
if [[ -n "$config_flag" ]]; then
|
|
391
|
+
args+=("$config_flag")
|
|
392
|
+
fi
|
|
393
|
+
|
|
394
|
+
if ! "$TIME_CMD" -v ./bin/gruff-ts "${args[@]}" >"$run_output" 2>"$run_time"; then
|
|
395
|
+
die 2 "analyse failed for workload=$workload config=$config_label format=$format"
|
|
396
|
+
fi
|
|
397
|
+
|
|
398
|
+
elapsed_seconds="$(parse_elapsed_file "$run_time")"
|
|
399
|
+
rss_kb="$(parse_rss_file "$run_time")"
|
|
400
|
+
finding_count="$(parse_findings "$format" "$run_output")"
|
|
401
|
+
printf '%s\n' "$elapsed_seconds" >> "$wall_file"
|
|
402
|
+
printf '%s\n' "$rss_kb" >> "$rss_file"
|
|
403
|
+
printf '%s\n' "$finding_count" >> "$findings_file"
|
|
404
|
+
done
|
|
405
|
+
|
|
406
|
+
wall_json="$(aggregate_samples "$wall_file")"
|
|
407
|
+
rss_json="$(aggregate_samples "$rss_file")"
|
|
408
|
+
deterministic_count="$(deterministic_finding_count "$findings_file")"
|
|
409
|
+
|
|
410
|
+
jq -n \
|
|
411
|
+
--arg workload "$workload" \
|
|
412
|
+
--arg target "$target_path" \
|
|
413
|
+
--arg config "$config_label" \
|
|
414
|
+
--arg format "$format" \
|
|
415
|
+
--argjson runs "$RUNS" \
|
|
416
|
+
--argjson wall "$wall_json" \
|
|
417
|
+
--argjson rss "$rss_json" \
|
|
418
|
+
--argjson count "$deterministic_count" \
|
|
419
|
+
'{
|
|
420
|
+
workload: $workload,
|
|
421
|
+
target: $target,
|
|
422
|
+
config: $config,
|
|
423
|
+
format: $format,
|
|
424
|
+
runs: $runs,
|
|
425
|
+
wall_seconds: $wall,
|
|
426
|
+
max_rss_kb: $rss,
|
|
427
|
+
findings: { count: $count, deterministic: true }
|
|
428
|
+
}'
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
ensure_parent_dir() {
|
|
432
|
+
local path="$1"
|
|
433
|
+
local parent
|
|
434
|
+
parent="$(dirname -- "$path")"
|
|
435
|
+
mkdir -p -- "$parent"
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
write_single_json() {
|
|
439
|
+
local cell_file="$1"
|
|
440
|
+
ensure_parent_dir "$OUT_PATH"
|
|
441
|
+
jq -n --slurpfile cell "$cell_file" --arg tool "gruff-ts" --arg target "$TARGET" --argjson runs "$RUNS" '
|
|
442
|
+
{
|
|
443
|
+
tool: $tool,
|
|
444
|
+
target: $target,
|
|
445
|
+
runs: $runs,
|
|
446
|
+
wall_seconds: $cell[0].wall_seconds,
|
|
447
|
+
max_rss_kb: $cell[0].max_rss_kb,
|
|
448
|
+
findings: $cell[0].findings
|
|
449
|
+
}
|
|
450
|
+
' > "$OUT_PATH"
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
print_single_summary() {
|
|
454
|
+
jq -r '
|
|
455
|
+
def r3: (. * 1000 | round / 1000);
|
|
456
|
+
def mb: (. / 1024 * 10 | round / 10);
|
|
457
|
+
"gruff-ts performance",
|
|
458
|
+
"Target: \(.target)",
|
|
459
|
+
"Runs: \(.runs)",
|
|
460
|
+
"Wall: \(.wall_seconds.mean | r3)s +/- \(.wall_seconds.stddev | r3)s (min \(.wall_seconds.min | r3)s, max \(.wall_seconds.max | r3)s)",
|
|
461
|
+
"Max RSS: \(.max_rss_kb.mean | mb) MB +/- \(.max_rss_kb.stddev | mb) MB (min \(.max_rss_kb.min | mb) MB, max \(.max_rss_kb.max | mb) MB)",
|
|
462
|
+
"Findings: \(.findings.count) deterministic",
|
|
463
|
+
"JSON: \($path)"
|
|
464
|
+
' --arg path "$OUT_PATH" "$OUT_PATH"
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
generate_large_fixture() {
|
|
468
|
+
local target_dir="$1"
|
|
469
|
+
local file_count="$2"
|
|
470
|
+
local index
|
|
471
|
+
local line
|
|
472
|
+
local file
|
|
473
|
+
|
|
474
|
+
mkdir -p -- "$target_dir"
|
|
475
|
+
for ((index = 1; index <= file_count; index++)); do
|
|
476
|
+
file="$target_dir/perf-fixture-$index.ts"
|
|
477
|
+
{
|
|
478
|
+
printf 'export class PerfFixture%s {\n' "$index"
|
|
479
|
+
printf ' public name = "fixture-%s";\n' "$index"
|
|
480
|
+
printf ' public process%s(input: string[]): number {\n' "$index"
|
|
481
|
+
printf ' let total = 0;\n'
|
|
482
|
+
for ((line = 1; line <= 192; line++)); do
|
|
483
|
+
printf ' total += input[%s]?.length ?? %s;\n' "$((line % 7))" "$((index + line))"
|
|
484
|
+
done
|
|
485
|
+
printf ' return total;\n'
|
|
486
|
+
printf ' }\n'
|
|
487
|
+
printf '}\n'
|
|
488
|
+
} > "$file"
|
|
489
|
+
done
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
run_matrix() {
|
|
493
|
+
local cells_file="$TMP_DIR/cells.jsonl"
|
|
494
|
+
local matrix_file="$TMP_DIR/matrix.json"
|
|
495
|
+
local generated_at
|
|
496
|
+
local WORKLOADS=("fixtures/sample.ts" "src" "synthetic-large")
|
|
497
|
+
local WORKLOADS_TARGETS
|
|
498
|
+
local config_labels=("--no-config" "default")
|
|
499
|
+
local config_flags=("--no-config" "")
|
|
500
|
+
local formats=("json" "text")
|
|
501
|
+
local wi
|
|
502
|
+
local ci
|
|
503
|
+
local format
|
|
504
|
+
|
|
505
|
+
if [[ "$CLEANUP" -eq 1 ]]; then
|
|
506
|
+
cleanup_old_fixtures
|
|
507
|
+
fi
|
|
508
|
+
|
|
509
|
+
CURRENT_FIXTURE_DIR="/tmp/gruff-perf-fixture-$$"
|
|
510
|
+
generate_large_fixture "$CURRENT_FIXTURE_DIR" 100
|
|
511
|
+
WORKLOADS_TARGETS=("fixtures/sample.ts" "src" "$CURRENT_FIXTURE_DIR")
|
|
512
|
+
: > "$cells_file"
|
|
513
|
+
|
|
514
|
+
for wi in "${!WORKLOADS[@]}"; do
|
|
515
|
+
for ci in "${!config_labels[@]}"; do
|
|
516
|
+
for format in "${formats[@]}"; do
|
|
517
|
+
run_cell "${WORKLOADS[$wi]}" "${WORKLOADS_TARGETS[$wi]}" "${config_labels[$ci]}" "${config_flags[$ci]}" "$format" >> "$cells_file"
|
|
518
|
+
done
|
|
519
|
+
done
|
|
520
|
+
done
|
|
521
|
+
|
|
522
|
+
generated_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
|
523
|
+
jq -s \
|
|
524
|
+
--arg tool "gruff-ts" \
|
|
525
|
+
--arg schemaVersion "$SCHEMA_VERSION" \
|
|
526
|
+
--arg generatedAt "$generated_at" \
|
|
527
|
+
--argjson runs "$RUNS" \
|
|
528
|
+
'{ tool: $tool, schemaVersion: $schemaVersion, generatedAt: $generatedAt, runs: $runs, cells: . }' \
|
|
529
|
+
"$cells_file" > "$matrix_file"
|
|
530
|
+
|
|
531
|
+
ensure_parent_dir "$OUT_PATH"
|
|
532
|
+
cp -- "$matrix_file" "$OUT_PATH"
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
validate_baseline_schema() {
|
|
536
|
+
local schema
|
|
537
|
+
[[ -f "$BASELINE_PATH" ]] || die 2 "baseline file does not exist: $BASELINE_PATH"
|
|
538
|
+
schema="$(jq -r '.schemaVersion // ""' "$BASELINE_PATH")" || die 2 "invalid baseline JSON: $BASELINE_PATH"
|
|
539
|
+
[[ "$schema" == "$SCHEMA_VERSION" ]] || die 2 "unsupported perf baseline schema: ${schema:-missing}"
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
write_baseline() {
|
|
543
|
+
ensure_parent_dir "$WRITE_BASELINE_PATH"
|
|
544
|
+
if [[ -e "$WRITE_BASELINE_PATH" && "$FORCE" -ne 1 ]]; then
|
|
545
|
+
die 2 "baseline already exists: $WRITE_BASELINE_PATH (use --force to overwrite)"
|
|
546
|
+
fi
|
|
547
|
+
cp -- "$OUT_PATH" "$WRITE_BASELINE_PATH"
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
empty_baseline_file() {
|
|
551
|
+
local path="$TMP_DIR/empty-baseline.json"
|
|
552
|
+
printf '{"cells":[]}\n' > "$path"
|
|
553
|
+
printf '%s\n' "$path"
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
write_markdown_report() {
|
|
557
|
+
local current_json="$1"
|
|
558
|
+
local baseline_json="$2"
|
|
559
|
+
local destination="$3"
|
|
560
|
+
local report_tmp="$TMP_DIR/perf-report.md"
|
|
561
|
+
|
|
562
|
+
jq -r --slurpfile baseline "$baseline_json" '
|
|
563
|
+
def key($cell): $cell.workload + "\u0000" + $cell.config + "\u0000" + $cell.format;
|
|
564
|
+
def r3: (. * 1000 | round / 1000);
|
|
565
|
+
def mb: (. / 1024 * 10 | round / 10);
|
|
566
|
+
def pct($now; $then):
|
|
567
|
+
if ($then == null or $then == 0) then "-"
|
|
568
|
+
else
|
|
569
|
+
(((($now - $then) / $then) * 100 * 10 | round / 10) as $value
|
|
570
|
+
| (if ($value > -0.05 and $value < 0.05) then 0 else $value end) as $display
|
|
571
|
+
| (if $display > 0 then "+" else "" end) + ($display | tostring) + "%")
|
|
572
|
+
end;
|
|
573
|
+
($baseline[0].cells // [] | map({key: key(.), value: .}) | from_entries) as $base
|
|
574
|
+
| [
|
|
575
|
+
"# gruff-ts perf",
|
|
576
|
+
"",
|
|
577
|
+
"| workload | config | format | wall mean (s) | wall σ | RSS mean (MB) | findings | Δwall vs baseline | Δrss vs baseline |",
|
|
578
|
+
"|---|---|---|---:|---:|---:|---:|---:|---:|",
|
|
579
|
+
(.cells[] as $cell
|
|
580
|
+
| ($base[(key($cell))] // null) as $old
|
|
581
|
+
| "| \($cell.workload) | \($cell.config) | \($cell.format) | \($cell.wall_seconds.mean | r3) | \($cell.wall_seconds.stddev | r3) | \($cell.max_rss_kb.mean | mb) | \($cell.findings.count) | \(pct($cell.wall_seconds.mean; ($old.wall_seconds.mean // null))) | \(pct($cell.max_rss_kb.mean; ($old.max_rss_kb.mean // null))) |")
|
|
582
|
+
]
|
|
583
|
+
| .[]
|
|
584
|
+
' "$current_json" > "$report_tmp"
|
|
585
|
+
|
|
586
|
+
if [[ -n "$destination" ]]; then
|
|
587
|
+
ensure_parent_dir "$destination"
|
|
588
|
+
cp -- "$report_tmp" "$destination"
|
|
589
|
+
elif [[ "$QUIET" -ne 1 ]]; then
|
|
590
|
+
cat "$report_tmp"
|
|
591
|
+
fi
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
check_missing_baseline_cells() {
|
|
595
|
+
local current_json="$1"
|
|
596
|
+
local missing_file="$TMP_DIR/missing-baseline-cells.txt"
|
|
597
|
+
jq -r --slurpfile baseline "$BASELINE_PATH" '
|
|
598
|
+
def key($cell): $cell.workload + "\u0000" + $cell.config + "\u0000" + $cell.format;
|
|
599
|
+
($baseline[0].cells // [] | map({key: key(.), value: true}) | from_entries) as $base
|
|
600
|
+
| .cells[]
|
|
601
|
+
| select(($base[(key(.))] // false) | not)
|
|
602
|
+
| "\(.workload) \(.config) \(.format)"
|
|
603
|
+
' "$current_json" > "$missing_file"
|
|
604
|
+
if [[ -s "$missing_file" ]]; then
|
|
605
|
+
printf 'baseline is missing perf cells:\n' >&2
|
|
606
|
+
while IFS= read -r line; do
|
|
607
|
+
printf -- '- %s\n' "$line" >&2
|
|
608
|
+
done < "$missing_file"
|
|
609
|
+
exit 2
|
|
610
|
+
fi
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
check_regressions() {
|
|
614
|
+
local current_json="$1"
|
|
615
|
+
local regressions_file="$TMP_DIR/perf-regressions.txt"
|
|
616
|
+
jq -r --argjson tolerance "$REGRESSION_TOLERANCE" --slurpfile baseline "$BASELINE_PATH" '
|
|
617
|
+
def key($cell): $cell.workload + "\u0000" + $cell.config + "\u0000" + $cell.format;
|
|
618
|
+
def delta($now; $then):
|
|
619
|
+
if ($then == null or $then == 0) then 0
|
|
620
|
+
else (($now - $then) / $then) * 100
|
|
621
|
+
end;
|
|
622
|
+
def r1: (. * 10 | round / 10);
|
|
623
|
+
($baseline[0].cells // [] | map({key: key(.), value: .}) | from_entries) as $base
|
|
624
|
+
| .cells[] as $cell
|
|
625
|
+
| ($base[(key($cell))] // null) as $old
|
|
626
|
+
| select($old != null)
|
|
627
|
+
| (delta($cell.wall_seconds.mean; $old.wall_seconds.mean)) as $wall
|
|
628
|
+
| (delta($cell.max_rss_kb.mean; $old.max_rss_kb.mean)) as $rss
|
|
629
|
+
| select($wall > $tolerance or $rss > $tolerance)
|
|
630
|
+
| "\($cell.workload) \($cell.config) \($cell.format): wall \($wall | r1)% rss \($rss | r1)%"
|
|
631
|
+
' "$current_json" > "$regressions_file"
|
|
632
|
+
|
|
633
|
+
if [[ -s "$regressions_file" ]]; then
|
|
634
|
+
printf 'performance regression detected (tolerance %s%%):\n' "$REGRESSION_TOLERANCE" >&2
|
|
635
|
+
while IFS= read -r line; do
|
|
636
|
+
printf -- '- %s\n' "$line" >&2
|
|
637
|
+
done < "$regressions_file"
|
|
638
|
+
exit 1
|
|
639
|
+
fi
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
run_single() {
|
|
643
|
+
local cell_file="$TMP_DIR/single-cell.json"
|
|
644
|
+
run_cell "$TARGET" "$TARGET" "--no-config" "--no-config" "json" > "$cell_file"
|
|
645
|
+
write_single_json "$cell_file"
|
|
646
|
+
if [[ "$QUIET" -eq 1 ]]; then
|
|
647
|
+
printf '%s\n' "$OUT_PATH"
|
|
648
|
+
else
|
|
649
|
+
print_single_summary
|
|
650
|
+
fi
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
main() {
|
|
654
|
+
local root
|
|
655
|
+
local report_baseline
|
|
656
|
+
|
|
657
|
+
parse_args "$@"
|
|
658
|
+
validate_args
|
|
659
|
+
root="$(repo_root)"
|
|
660
|
+
cd "$root"
|
|
661
|
+
TMP_DIR="/tmp/gruff-perf-work-$$"
|
|
662
|
+
mkdir -p -- "$TMP_DIR"
|
|
663
|
+
trap cleanup EXIT
|
|
664
|
+
require_tools
|
|
665
|
+
detect_time_cmd
|
|
666
|
+
assert_elapsed_parser
|
|
667
|
+
|
|
668
|
+
if [[ -n "$BASELINE_PATH" ]]; then
|
|
669
|
+
validate_baseline_schema
|
|
670
|
+
fi
|
|
671
|
+
|
|
672
|
+
if [[ "$MATRIX" -eq 1 ]]; then
|
|
673
|
+
run_matrix
|
|
674
|
+
if [[ "$WRITE_BASELINE" -eq 1 ]]; then
|
|
675
|
+
write_baseline
|
|
676
|
+
fi
|
|
677
|
+
if [[ -n "$BASELINE_PATH" ]]; then
|
|
678
|
+
check_missing_baseline_cells "$OUT_PATH"
|
|
679
|
+
report_baseline="$BASELINE_PATH"
|
|
680
|
+
else
|
|
681
|
+
report_baseline="$(empty_baseline_file)"
|
|
682
|
+
fi
|
|
683
|
+
write_markdown_report "$OUT_PATH" "$report_baseline" "$REPORT_PATH"
|
|
684
|
+
if [[ "$QUIET" -eq 1 ]]; then
|
|
685
|
+
printf '%s\n' "$OUT_PATH"
|
|
686
|
+
fi
|
|
687
|
+
if [[ "$FAIL_ON_REGRESSION" -eq 1 ]]; then
|
|
688
|
+
check_regressions "$OUT_PATH"
|
|
689
|
+
fi
|
|
690
|
+
else
|
|
691
|
+
run_single
|
|
692
|
+
fi
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
main "$@"
|