@scalequality/check 0.5.1 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -8
- package/bin/check.mjs +4 -88
- package/bin/coverage.mjs +117 -0
- package/bin/durability/GitReader.mjs +49 -26
- package/bin/durability/PROVENANCE.md +17 -8
- package/bin/durability/provenance.json +6 -3
- package/bin/durability/runtime.mjs +31 -3
- package/bin/durability/worker.mjs +4 -3
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -87,15 +87,15 @@ source, coverage, and a bounded durability report (at most 1 MiB); it never uplo
|
|
|
87
87
|
the raw `.git` object database. Years of old binary history therefore do not add
|
|
88
88
|
to the archive size. It does not truncate or change your repository.
|
|
89
89
|
|
|
90
|
-
Use a complete checkout for that measurement.
|
|
91
|
-
|
|
90
|
+
Use a complete checkout for that measurement. A shallow or incomplete checkout
|
|
91
|
+
is rejected locally before upload; it is not treated as a full 90-day measurement.
|
|
92
92
|
|
|
93
93
|
```yaml
|
|
94
94
|
# GitHub Actions
|
|
95
95
|
- uses: actions/checkout@v4
|
|
96
96
|
with:
|
|
97
97
|
fetch-depth: 0
|
|
98
|
-
- run: npx @scalequality/check@0.5.
|
|
98
|
+
- run: npx @scalequality/check@0.5.3 --project <projectId>
|
|
99
99
|
env:
|
|
100
100
|
SQ_TOKEN: ${{ secrets.SQ_TOKEN }}
|
|
101
101
|
```
|
|
@@ -109,8 +109,8 @@ clone:
|
|
|
109
109
|
|
|
110
110
|
The current-code scanners still run on the server. Historical secret scanning and
|
|
111
111
|
commit-convention analysis are not included in this report; the result shows those
|
|
112
|
-
limits and identifies durability as reported by CI.
|
|
113
|
-
report
|
|
112
|
+
limits and identifies durability as reported by CI. A failed Git collection or
|
|
113
|
+
invalid report stops the CLI before upload, without submitting a replacement score.
|
|
114
114
|
|
|
115
115
|
Version 0.5.0 needs the compatible ScaleQuality server release. The CLI checks the
|
|
116
116
|
server's acknowledgement of this exact report before declaring a passing gate.
|
|
@@ -126,7 +126,7 @@ that you expect to measure.
|
|
|
126
126
|
|
|
127
127
|
```bash
|
|
128
128
|
# Local verification; does not need a token and sends nothing to ScaleQuality.
|
|
129
|
-
npx @scalequality/check@0.5.
|
|
129
|
+
npx @scalequality/check@0.5.3 --dry-run
|
|
130
130
|
```
|
|
131
131
|
|
|
132
132
|
The CLI checks compressed size before allocating the upload buffer or sending a
|
|
@@ -158,8 +158,8 @@ blame output without increasing the worker's memory or time budgets.
|
|
|
158
158
|
Archive exclusions, including `--exclude`, do not change Git durability's tracked
|
|
159
159
|
file scope. A tracked dependency can still contribute to the historical metric.
|
|
160
160
|
Changing that methodology requires a separate coordinated version; this patch
|
|
161
|
-
does not silently rewrite prior scores.
|
|
162
|
-
|
|
161
|
+
does not silently rewrite prior scores. Coverage may remain absent when the job
|
|
162
|
+
does not generate it; Git collection failures stop the check before upload.
|
|
163
163
|
|
|
164
164
|
Track-only completion now says `measurement completed (track only; no cutoff
|
|
165
165
|
configured)`. Its exit code remains zero; completion does not certify that every
|
|
@@ -168,3 +168,36 @@ dimension was measured or that a maturity threshold was passed.
|
|
|
168
168
|
The 60 MiB compressed-source limit still applies. A checkout with more than that
|
|
169
169
|
amount of first-party source or ambiguous generated assets is rejected locally;
|
|
170
170
|
this release does not truncate source to force a successful upload.
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
## 0.5.2: complete Git collection before submitting a score
|
|
174
|
+
|
|
175
|
+
The complete HEAD object inventory now streams through the same bounded reader
|
|
176
|
+
as blame. Large old histories no longer have to fit into a 16 MiB output buffer
|
|
177
|
+
for that check. Missing objects remain errors. The 90-day window, 14-day settlement,
|
|
178
|
+
scoring calculations and archive limit are unchanged.
|
|
179
|
+
|
|
180
|
+
Unexpected Git command failures stop collection; they cannot become zero surviving
|
|
181
|
+
lines or an upload with durability omitted. A temporary OS `EAGAIN` failure to
|
|
182
|
+
start Git gets one automatic retry within the existing measurement deadline.
|
|
183
|
+
Timeouts, missing or corrupt history, and resource-limit failures do not trigger
|
|
184
|
+
unbounded retries. A technical failure exits with code 2 and sends no new score.
|
|
185
|
+
|
|
186
|
+
Tracked blob names are preserved exactly, including Git-quoted characters.
|
|
187
|
+
Submodule gitlinks are not source blobs and are not sent to blame; ordinary files,
|
|
188
|
+
symlinks and binary blobs retain their existing measurement treatment.
|
|
189
|
+
|
|
190
|
+
A valid report with zero settled or AI-attributed lines still proceeds. It does
|
|
191
|
+
not invent 100% AI survival. Missing coverage alone does not block submission.
|
|
192
|
+
A completed check below a configured maturity cutoff still exits with code 1;
|
|
193
|
+
track-only completion still exits with code 0.
|
|
194
|
+
|
|
195
|
+
## 0.5.3: complete coverage report collection
|
|
196
|
+
|
|
197
|
+
Coverage discovery and staging now process all matching reports incrementally,
|
|
198
|
+
including deep modules and individual reports larger than 25 MiB. Report bytes
|
|
199
|
+
and paths are preserved; the CLI no longer silently stops after 40 reports.
|
|
200
|
+
A file that changes or cannot be copied stops submission before any upload.
|
|
201
|
+
The existing 60 MiB compressed upload limit, Git durability calculation and
|
|
202
|
+
maturity gates are unchanged. This release requires the matching server reader
|
|
203
|
+
that accepts the complete staged report index.
|
package/bin/check.mjs
CHANGED
|
@@ -4,12 +4,11 @@
|
|
|
4
4
|
// non-zero when maturity is below the configured cutoff, so a CI step can fail the
|
|
5
5
|
// build. Pure Node 18+ (global fetch/FormData/Blob), no npm dependencies.
|
|
6
6
|
|
|
7
|
-
import { tmpdir } from "node:os"
|
|
8
|
-
import { join, dirname, relative, sep } from "node:path"
|
|
9
7
|
import { spawnSync } from "node:child_process"
|
|
10
|
-
import { readFileSync
|
|
8
|
+
import { readFileSync } from "node:fs"
|
|
11
9
|
import { prepareCheckoutArchive, normalizeExcludes } from "./archive.mjs"
|
|
12
10
|
import { measureDurability } from "./durability.mjs"
|
|
11
|
+
import { stageCoverageReports } from "./coverage.mjs"
|
|
13
12
|
|
|
14
13
|
const args = process.argv.slice(2)
|
|
15
14
|
const flag = (name) => {
|
|
@@ -128,95 +127,12 @@ function slimReport(stdout, ref) {
|
|
|
128
127
|
return json
|
|
129
128
|
}
|
|
130
129
|
|
|
131
|
-
// Coverage reports live in exactly the directories the archive excludes: JaCoCo
|
|
132
|
-
// writes to target/ and build/, Istanbul to coverage/. So the one moment where a
|
|
133
|
-
// REAL coverage number is sitting on disk — right after this job ran the tests —
|
|
134
|
-
// was also the one moment we threw it away, and every Maven project came back
|
|
135
|
-
// with no measured coverage even with a green suite behind it.
|
|
136
|
-
//
|
|
137
|
-
// The reports are copied into .sqreports/ under FLAT, indexed names, with an
|
|
138
|
-
// index.json holding each one's original path. Mirroring the original directory
|
|
139
|
-
// tree looked tidier and did not work: bsdtar does not anchor `--exclude=./target`
|
|
140
|
-
// to the root, so a staged copy at `.scalequality/coverage/target/...` was dropped
|
|
141
|
-
// on macOS while surviving under GNU tar. Measured, not assumed — and a layout
|
|
142
|
-
// whose behaviour depends on which tar the runner ships is not a layout.
|
|
143
|
-
// Every staged name here is `<n>-<basename>`, which no exclude pattern can match.
|
|
144
|
-
const COVERAGE_NAMES = [
|
|
145
|
-
{ re: /^jacoco(TestReport)?\.xml$/i, format: "jacoco" },
|
|
146
|
-
{ re: /^lcov\.info$/i, format: "lcov" },
|
|
147
|
-
{ re: /^coverage-summary\.json$/i, format: "istanbul" },
|
|
148
|
-
{ re: /^coverage\.cobertura\.xml$/i, format: "cobertura" },
|
|
149
|
-
{ re: /^cobertura(-coverage)?\.xml$/i, format: "cobertura" },
|
|
150
|
-
{ re: /^coverage\.xml$/i, format: "cobertura" },
|
|
151
|
-
{ re: /^clover\.xml$/i, format: "clover" },
|
|
152
|
-
{ re: /^(coverage|cover)\.out$/i, format: "go" },
|
|
153
|
-
]
|
|
154
|
-
const COVERAGE_SKIP = new Set(["node_modules", ".git", ".venv", "venv", "vendor", "__pycache__", ".sqreports"])
|
|
155
|
-
const COVERAGE_STAGE = ".sqreports"
|
|
156
|
-
const MAX_REPORT_BYTES = 25 * 1024 * 1024
|
|
157
|
-
const MAX_REPORTS = 40
|
|
158
|
-
|
|
159
|
-
function stageCoverageReports() {
|
|
160
|
-
const root = process.cwd()
|
|
161
|
-
const stagingRoot = mkdtempSync(join(tmpdir(), "sq-coverage-"))
|
|
162
|
-
const stage = join(stagingRoot, COVERAGE_STAGE)
|
|
163
|
-
// The staged copies must not outlive this process under ANY exit path. A
|
|
164
|
-
// cancelled job or a network failure that left them behind would show up as
|
|
165
|
-
// untracked files in the customer's checkout, and a pipeline that runs
|
|
166
|
-
// `git diff --exit-code` after the gate would fail because of us.
|
|
167
|
-
const clean = () => { try { rmSync(stagingRoot, { recursive: true, force: true }) } catch { /* best effort */ } }
|
|
168
|
-
process.on("exit", clean)
|
|
169
|
-
for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"]) {
|
|
170
|
-
process.on(sig, () => { clean(); process.exit(130) })
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
const index = []
|
|
174
|
-
let bytes = 0
|
|
175
|
-
const walk = (dir, depth) => {
|
|
176
|
-
if (depth > 7 || index.length >= MAX_REPORTS || bytes >= MAX_REPORT_BYTES) return
|
|
177
|
-
let entries
|
|
178
|
-
try { entries = readdirSync(dir, { withFileTypes: true }) } catch { return }
|
|
179
|
-
for (const e of entries) {
|
|
180
|
-
if (index.length >= MAX_REPORTS || bytes >= MAX_REPORT_BYTES) return
|
|
181
|
-
const abs = join(dir, e.name)
|
|
182
|
-
if (e.isDirectory()) {
|
|
183
|
-
if (!COVERAGE_SKIP.has(e.name)) walk(abs, depth + 1)
|
|
184
|
-
continue
|
|
185
|
-
}
|
|
186
|
-
if (!e.isFile()) continue
|
|
187
|
-
const hit = COVERAGE_NAMES.find((c) => c.re.test(e.name))
|
|
188
|
-
if (!hit) continue
|
|
189
|
-
let size = 0
|
|
190
|
-
try { size = statSync(abs).size } catch { continue }
|
|
191
|
-
if (size === 0 || bytes + size > MAX_REPORT_BYTES) continue
|
|
192
|
-
const staged = `${index.length}-${e.name}`
|
|
193
|
-
try {
|
|
194
|
-
mkdirSync(stage, { recursive: true })
|
|
195
|
-
copyFileSync(abs, join(stage, staged))
|
|
196
|
-
} catch { continue }
|
|
197
|
-
bytes += size
|
|
198
|
-
index.push({ file: staged, path: relative(root, abs).split(sep).join("/"), format: hit.format })
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
walk(root, 0)
|
|
202
|
-
|
|
203
|
-
if (index.length) {
|
|
204
|
-
try {
|
|
205
|
-
writeFileSync(join(stage, "index.json"), JSON.stringify({ version: 1, reports: index }))
|
|
206
|
-
} catch { /* the archive just goes without them */ }
|
|
207
|
-
const shown = index.slice(0, 3).map((r) => r.path).join(", ")
|
|
208
|
-
log(`ScaleQuality: found ${index.length} coverage report(s) — ${shown}${index.length > 3 ? " …" : ""}`)
|
|
209
|
-
} else {
|
|
210
|
-
log("ScaleQuality: no coverage report found in this job. Run your tests with coverage before this step to get a measured %.")
|
|
211
|
-
}
|
|
212
|
-
return { stage, root: stagingRoot, count: index.length, cleanup: clean }
|
|
213
|
-
}
|
|
214
|
-
|
|
215
130
|
async function main() {
|
|
216
131
|
const auth = { Authorization: `Bearer ${token}` }
|
|
217
132
|
const excludes = normalizeExcludes(args.flatMap((arg, i) => arg === "--exclude" ? [args[i + 1]] : []))
|
|
218
133
|
const durabilityReport = await measureDurability({ root: process.cwd(), log: jsonOut ? (message) => console.error(message) : log })
|
|
219
|
-
|
|
134
|
+
if (!durabilityReport) fail("Git durability collection failed. No upload was sent and no new score was submitted. Resolve the Git error above before rerunning this step.")
|
|
135
|
+
const coverage = await stageCoverageReports({ log })
|
|
220
136
|
let packed, buf
|
|
221
137
|
try {
|
|
222
138
|
packed = prepareCheckoutArchive({ root: process.cwd(), coverageRoot: coverage.root, durabilityReport, excludes, log })
|
package/bin/coverage.mjs
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { constants, rmSync } from "node:fs"
|
|
2
|
+
import { open, opendir, mkdir, mkdtemp } from "node:fs/promises"
|
|
3
|
+
import { tmpdir } from "node:os"
|
|
4
|
+
import { join, relative, sep } from "node:path"
|
|
5
|
+
|
|
6
|
+
const NAMES = [
|
|
7
|
+
{ re: /^jacoco(TestReport)?\.xml$/i, format: "jacoco" },
|
|
8
|
+
{ re: /^lcov\.info$/i, format: "lcov" },
|
|
9
|
+
{ re: /^coverage-summary\.json$/i, format: "istanbul" },
|
|
10
|
+
{ re: /^coverage\.cobertura\.xml$/i, format: "cobertura" },
|
|
11
|
+
{ re: /^cobertura(-coverage)?\.xml$/i, format: "cobertura" },
|
|
12
|
+
{ re: /^coverage\.xml$/i, format: "cobertura" },
|
|
13
|
+
{ re: /^clover\.xml$/i, format: "clover" },
|
|
14
|
+
{ re: /^(coverage|cover)\.out$/i, format: "go" },
|
|
15
|
+
]
|
|
16
|
+
const SKIP = new Set(["node_modules", ".git", ".venv", "venv", "vendor", "__pycache__", ".sqreports"])
|
|
17
|
+
const STAGE = ".sqreports"
|
|
18
|
+
|
|
19
|
+
// Copy sequentially using a fixed buffer. Report count and uncompressed bytes
|
|
20
|
+
// do not determine which sources survive. The final compressed checkout still
|
|
21
|
+
// has the existing upload limit, checked by archive.mjs before any HTTP request.
|
|
22
|
+
async function copyReport(source, destination) {
|
|
23
|
+
let input, output
|
|
24
|
+
try {
|
|
25
|
+
input = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK)
|
|
26
|
+
const before = await input.stat({ bigint: true })
|
|
27
|
+
if (!before.isFile() || before.size === 0n) throw new Error("report is empty or is not a regular file")
|
|
28
|
+
output = await open(destination, "wx")
|
|
29
|
+
const buffer = Buffer.allocUnsafe(64 * 1024)
|
|
30
|
+
let copied = 0n
|
|
31
|
+
while (true) {
|
|
32
|
+
const { bytesRead } = await input.read(buffer, 0, buffer.length, null)
|
|
33
|
+
if (!bytesRead) break
|
|
34
|
+
let written = 0
|
|
35
|
+
while (written < bytesRead) {
|
|
36
|
+
const { bytesWritten } = await output.write(buffer, written, bytesRead - written, null)
|
|
37
|
+
if (!bytesWritten) throw new Error("report copy made no progress")
|
|
38
|
+
written += bytesWritten
|
|
39
|
+
}
|
|
40
|
+
copied += BigInt(bytesRead)
|
|
41
|
+
if (copied > before.size) throw new Error("report changed during collection")
|
|
42
|
+
}
|
|
43
|
+
const after = await input.stat({ bigint: true })
|
|
44
|
+
if (copied !== before.size || after.size !== before.size || after.mtimeNs !== before.mtimeNs || after.ctimeNs !== before.ctimeNs) {
|
|
45
|
+
throw new Error("report changed during collection")
|
|
46
|
+
}
|
|
47
|
+
} finally {
|
|
48
|
+
await Promise.all([output?.close(), input?.close()])
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Preserve every discovered original report in a flat v1 index. Flat names
|
|
53
|
+
* keep GNU/BSD tar exclusions from dropping nested build/target reports.
|
|
54
|
+
* Generated
|
|
55
|
+
* directories are scanned too: they are exactly where test runners write.
|
|
56
|
+
* No customer files are created or changed. A failed scan/copy/index never
|
|
57
|
+
* returns a partial set that could be presented as a complete measurement. */
|
|
58
|
+
export async function stageCoverageReports({ root = process.cwd(), log = () => {} } = {}) {
|
|
59
|
+
const stagingRoot = await mkdtemp(join(tmpdir(), "sq-coverage-"))
|
|
60
|
+
const stage = join(stagingRoot, STAGE)
|
|
61
|
+
const remove = () => { try { rmSync(stagingRoot, { recursive: true, force: true }) } catch { /* best effort on process exit */ } }
|
|
62
|
+
const signals = ["SIGINT", "SIGTERM", "SIGHUP"]
|
|
63
|
+
const onSignal = () => { remove(); process.exit(130) }
|
|
64
|
+
process.once("exit", remove)
|
|
65
|
+
for (const signal of signals) process.once(signal, onSignal)
|
|
66
|
+
const cleanup = () => {
|
|
67
|
+
process.removeListener("exit", remove)
|
|
68
|
+
for (const signal of signals) process.removeListener(signal, onSignal)
|
|
69
|
+
remove()
|
|
70
|
+
}
|
|
71
|
+
let index
|
|
72
|
+
let count = 0
|
|
73
|
+
const shown = []
|
|
74
|
+
let current = "."
|
|
75
|
+
const walk = async (directory) => {
|
|
76
|
+
current = relative(root, directory).split(sep).join("/") || "."
|
|
77
|
+
const entries = await opendir(directory)
|
|
78
|
+
for await (const entry of entries) {
|
|
79
|
+
const absolute = join(directory, entry.name)
|
|
80
|
+
if (entry.isDirectory()) {
|
|
81
|
+
if (!SKIP.has(entry.name)) await walk(absolute)
|
|
82
|
+
continue
|
|
83
|
+
}
|
|
84
|
+
if (!entry.isFile()) continue // Never follow report or directory symlinks.
|
|
85
|
+
const match = NAMES.find(candidate => candidate.re.test(entry.name))
|
|
86
|
+
if (!match) continue
|
|
87
|
+
current = relative(root, absolute).split(sep).join("/")
|
|
88
|
+
if (!index) {
|
|
89
|
+
await mkdir(stage)
|
|
90
|
+
index = await open(join(stage, "index.json"), "wx")
|
|
91
|
+
await index.writeFile('{"version":1,"reports":[')
|
|
92
|
+
}
|
|
93
|
+
const file = `${count}-${entry.name}`
|
|
94
|
+
await copyReport(absolute, join(stage, file))
|
|
95
|
+
await index.writeFile((count ? "," : "") + JSON.stringify({ file, path: current, format: match.format }))
|
|
96
|
+
count++
|
|
97
|
+
if (shown.length < 3) shown.push(current)
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
try {
|
|
101
|
+
await walk(root)
|
|
102
|
+
if (index) {
|
|
103
|
+
await index.writeFile("]}\n")
|
|
104
|
+
await index.close()
|
|
105
|
+
index = undefined
|
|
106
|
+
log(`ScaleQuality: found ${count} coverage report(s) — ${shown.join(", ")}${count > 3 ? " …" : ""}`)
|
|
107
|
+
} else {
|
|
108
|
+
log("ScaleQuality: no coverage report found in this job. Run your tests with coverage before this step to get a measured %.")
|
|
109
|
+
}
|
|
110
|
+
return { stage, root: stagingRoot, count, cleanup }
|
|
111
|
+
} catch (error) {
|
|
112
|
+
await index?.close().catch(() => {})
|
|
113
|
+
cleanup()
|
|
114
|
+
const reason = typeof error?.code === "string" ? error.code : error?.message || "I/O error"
|
|
115
|
+
throw new Error(`Coverage collection could not finish at ${JSON.stringify(current)} (${reason}). No upload was sent and no partial coverage was submitted.`)
|
|
116
|
+
}
|
|
117
|
+
}
|
|
@@ -1,7 +1,27 @@
|
|
|
1
|
-
import { runGit as git, runGitLines,
|
|
1
|
+
import { runGit as git, runGitLines, GitMeasurementError } from "./runtime.mjs";
|
|
2
2
|
const US = "\x1f"; // unit separator (field)
|
|
3
3
|
const RS = "\x1e"; // record separator (commit)
|
|
4
4
|
const CO_AUTHOR_RE = /co-authored-by:\s*(.*?)\s*<([^>]+)>/gi;
|
|
5
|
+
/** `git log --numstat` C-quotes path bytes, independently of the UTF-8 source.
|
|
6
|
+
* Decode only this transport representation; leave rename/statistics semantics
|
|
7
|
+
* unchanged. `ls-tree -z` and blame both use the resulting literal pathname. */
|
|
8
|
+
function decodeGitPath(path) {
|
|
9
|
+
if (!path.startsWith('"') || !path.endsWith('"')) return path;
|
|
10
|
+
const parts = [];
|
|
11
|
+
let literal = '';
|
|
12
|
+
const escapes = { a: 7, b: 8, t: 9, n: 10, v: 11, f: 12, r: 13, '\\': 92, '"': 34 };
|
|
13
|
+
for (let i = 1; i < path.length - 1; i++) {
|
|
14
|
+
if (path[i] !== '\\') { literal += path[i]; continue; }
|
|
15
|
+
parts.push(Buffer.from(literal)); literal = '';
|
|
16
|
+
const escaped = path[++i];
|
|
17
|
+
if (Object.hasOwn(escapes, escaped)) parts.push(Buffer.from([escapes[escaped]]));
|
|
18
|
+
else if (/^[0-7]{3}$/.test(path.slice(i, i + 3))) {
|
|
19
|
+
parts.push(Buffer.from([parseInt(path.slice(i, i + 3), 8)])); i += 2;
|
|
20
|
+
} else throw new GitMeasurementError('Git file path could not be parsed');
|
|
21
|
+
}
|
|
22
|
+
parts.push(Buffer.from(literal));
|
|
23
|
+
return Buffer.concat(parts).toString('utf8');
|
|
24
|
+
}
|
|
5
25
|
function parseCoAuthors(body) {
|
|
6
26
|
const out = [];
|
|
7
27
|
let m;
|
|
@@ -77,15 +97,24 @@ export async function readCommits(repoPath, sinceDays, untilDays = 0) {
|
|
|
77
97
|
const del = Number.parseInt(cols[1], 10);
|
|
78
98
|
rec.additions += Number.isFinite(add) ? add : 0;
|
|
79
99
|
rec.deletions += Number.isFinite(del) ? del : 0;
|
|
80
|
-
rec.files.push(cols.slice(2).join("\t"));
|
|
100
|
+
rec.files.push(decodeGitPath(cols.slice(2).join("\t")));
|
|
81
101
|
}
|
|
82
102
|
}
|
|
83
103
|
return [...bySha.values()];
|
|
84
104
|
}
|
|
85
|
-
/**
|
|
105
|
+
/** Blameable entries currently at HEAD. Gitlinks point to another repository;
|
|
106
|
+
* native blame already contributed zero for them. Keep all blobs, including
|
|
107
|
+
* symlinks and binaries, without changing the commit/additions denominator. */
|
|
86
108
|
export async function listHeadFiles(repoPath) {
|
|
87
|
-
const out = await git(repoPath, ["ls-tree", "-r", "
|
|
88
|
-
|
|
109
|
+
const out = await git(repoPath, ["ls-tree", "-r", "-z", "HEAD"]);
|
|
110
|
+
const files = new Set();
|
|
111
|
+
for (const line of out.split("\0")) {
|
|
112
|
+
if (!line) continue;
|
|
113
|
+
const entry = /^[0-7]{6} (blob|tree|commit) [0-9a-f]{40}\t([\s\S]+)$/.exec(line);
|
|
114
|
+
if (!entry) throw new GitMeasurementError('Git tree output could not be parsed');
|
|
115
|
+
if (entry[1] === 'blob') files.add(entry[2]);
|
|
116
|
+
}
|
|
117
|
+
return files;
|
|
89
118
|
}
|
|
90
119
|
const BLAME_GROUP = /^([0-9a-f]{40}) [1-9][0-9]* [1-9][0-9]* ([1-9][0-9]*)$/;
|
|
91
120
|
/**
|
|
@@ -93,29 +122,23 @@ const BLAME_GROUP = /^([0-9a-f]{40}) [1-9][0-9]* [1-9][0-9]* ([1-9][0-9]*)$/;
|
|
|
93
122
|
* CURRENT lines it owns. Incremental output describes identical attribution
|
|
94
123
|
* in groups, without source lines or repeated per-line commit metadata.
|
|
95
124
|
* Stream those groups so large output does not require a large buffer.
|
|
96
|
-
*
|
|
125
|
+
* Callers select files present at HEAD. A failed command is incomplete evidence,
|
|
126
|
+
* never proof that this file has no surviving lines.
|
|
97
127
|
*/
|
|
98
128
|
export async function blameFile(repoPath, file) {
|
|
99
129
|
const counts = new Map();
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
catch (error) {
|
|
116
|
-
if (isFatalGitError(error)) throw error;
|
|
117
|
-
// deleted at HEAD / binary / unreadable → no surviving lines counted
|
|
118
|
-
counts.clear();
|
|
119
|
-
}
|
|
130
|
+
await runGitLines(repoPath, [
|
|
131
|
+
"blame",
|
|
132
|
+
"--incremental",
|
|
133
|
+
"HEAD",
|
|
134
|
+
"--",
|
|
135
|
+
file,
|
|
136
|
+
], line => {
|
|
137
|
+
const m = BLAME_GROUP.exec(line);
|
|
138
|
+
if (!m) return;
|
|
139
|
+
const total = (counts.get(m[1]) ?? 0) + Number(m[2]);
|
|
140
|
+
if (!Number.isSafeInteger(total)) throw new GitMeasurementError('Git blame line count exceeded the numeric limit');
|
|
141
|
+
counts.set(m[1], total);
|
|
142
|
+
});
|
|
120
143
|
return counts;
|
|
121
144
|
}
|
|
@@ -15,9 +15,14 @@ extensions. The engine's calculations and classifier remain unchanged. In the
|
|
|
15
15
|
reader, replace its promisified `execFile` wrapper with `runtime.mjs`'s `runGit`.
|
|
16
16
|
For blame, use `runGitLines` with `--incremental` and sum each group's line count
|
|
17
17
|
instead of counting individual `--line-porcelain` headers. Attribution is unchanged;
|
|
18
|
-
source text and repeated per-line metadata are not emitted or retained.
|
|
19
|
-
|
|
20
|
-
|
|
18
|
+
source text and repeated per-line metadata are not emitted or retained. Any failed
|
|
19
|
+
blame aborts the report instead of returning empty or partial counts. Read the
|
|
20
|
+
object type from `ls-tree -rz` and select only blobs, preserving literal names
|
|
21
|
+
without trimming. Decode Git's C-quoted path bytes in `log --numstat` so tabs,
|
|
22
|
+
quotes, Unicode and leading/trailing whitespace match these same blob names.
|
|
23
|
+
This skips only submodule gitlinks,
|
|
24
|
+
whose native blame contributed zero; binary and symlink blobs remain eligible.
|
|
25
|
+
Commit statistics and the introduced-lines denominator remain unchanged.
|
|
21
26
|
Recompute provenance hashes and run the
|
|
22
27
|
parity and cancellation tests. The reference modules under
|
|
23
28
|
`test/native-durability-reference/` preserve the original reader for small fixture
|
|
@@ -32,11 +37,15 @@ No clone, fetch, checkout or Git write is performed. Shallow or partial historie
|
|
|
32
37
|
missing objects, changing HEAD and invalid reports produce no measurement.
|
|
33
38
|
The separate worker has a 120-second total budget and 512 MiB V8 heap ceiling;
|
|
34
39
|
each Git command has at most 30 seconds. Buffered commands retain at most 16 MiB
|
|
35
|
-
of output. Incremental blame
|
|
36
|
-
and at most 16 MiB of stderr, rather than
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
+
of output. Incremental blame and the complete HEAD object inventory stream stdout,
|
|
41
|
+
retaining one line (at most 16 MiB) and at most 16 MiB of stderr, rather than
|
|
42
|
+
limiting total streamed output. One failed OS spawn with `EAGAIN` can recover after
|
|
43
|
+
100 ms, within the original total deadline; the recovery allowance is shared by
|
|
44
|
+
all Git commands. Command exits, signals, timeouts, missing/incomplete history and
|
|
45
|
+
deterministic resource failures are never automatically retried.
|
|
46
|
+
Collection failures abort the CLI before packaging or upload and cannot submit a
|
|
47
|
+
replacement score. A validated report with zero settled or AI-eligible lines is
|
|
48
|
+
still a successful collection; an absent optional coverage report is permitted.
|
|
40
49
|
Only the measurement worker's own process group and Git children are cleaned up
|
|
41
50
|
on cancellation or parent IPC disconnection. Git command output, author details,
|
|
42
51
|
credentials and customer environment tokens are not logged or sent to the worker.
|
|
@@ -29,8 +29,11 @@
|
|
|
29
29
|
"runtimeChanges": [
|
|
30
30
|
"ESM relative import extensions only in engine/classifier",
|
|
31
31
|
"GitReader process wrapper replaced by bounded local-only runner",
|
|
32
|
-
"GitReader blame
|
|
33
|
-
"GitReader
|
|
32
|
+
"GitReader blame aborts the report on any failed command; partial counts never become a successful measurement",
|
|
33
|
+
"GitReader selects literal blob paths from NUL-delimited ls-tree and decodes C-quoted numstat filenames; submodule gitlinks retain zero native contribution without changing introduced-line counts",
|
|
34
|
+
"GitReader blame streams incremental groups and sums their line counts with exact native attribution; bounded lines and stderr preserve resource limits without capping total streamed stdout",
|
|
35
|
+
"Worker streams the complete HEAD object inventory and rejects missing objects without buffering total output",
|
|
36
|
+
"Runtime permits one recovery from OS spawn EAGAIN within the original deadline; all other collection failures abort the CLI before upload"
|
|
34
37
|
],
|
|
35
38
|
"generatedFiles": [
|
|
36
39
|
{
|
|
@@ -39,7 +42,7 @@
|
|
|
39
42
|
},
|
|
40
43
|
{
|
|
41
44
|
"file": "GitReader.mjs",
|
|
42
|
-
"sha256": "
|
|
45
|
+
"sha256": "e86e2e8251f5d9e8f50f671805d8926346aeb574bd2d6a115e5cd324555858f3"
|
|
43
46
|
},
|
|
44
47
|
{
|
|
45
48
|
"file": "OriginClassifier.mjs",
|
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
import { execFile, spawn } from 'node:child_process'
|
|
2
|
+
import { setTimeout as delay } from 'node:timers/promises'
|
|
2
3
|
|
|
3
4
|
export const MAX_GIT_BUFFER = 16 * 1024 * 1024
|
|
4
5
|
export const MAX_GIT_TIME_MS = 30_000
|
|
5
6
|
let deadline = 0
|
|
7
|
+
let spawnRecoveryUsed = false
|
|
6
8
|
const children = new Set()
|
|
7
9
|
|
|
8
10
|
export class GitMeasurementError extends Error {
|
|
9
11
|
constructor(reason) { super(reason); this.reason = reason; this.fatalMeasurement = true }
|
|
10
12
|
}
|
|
11
13
|
export const isFatalGitError = (error) => error?.fatalMeasurement === true
|
|
12
|
-
export function setDeadline(value) { deadline = value }
|
|
14
|
+
export function setDeadline(value) { deadline = value; spawnRecoveryUsed = false }
|
|
13
15
|
export function remainingTime() {
|
|
14
16
|
const remaining = deadline - Date.now()
|
|
15
17
|
if (remaining <= 0) throw new GitMeasurementError('time budget exceeded')
|
|
@@ -36,6 +38,13 @@ function gitArgs(repoPath, args) {
|
|
|
36
38
|
}
|
|
37
39
|
|
|
38
40
|
function gitError(error, stderr) {
|
|
41
|
+
// EAGAIN is a failed OS spawn, before Git ran or emitted any result. A child
|
|
42
|
+
// exit status, timeout or deterministic Git/resource error is not retryable.
|
|
43
|
+
if (error.code === 'EAGAIN') {
|
|
44
|
+
const failure = new GitMeasurementError('Git process could not start because system resources are temporarily unavailable')
|
|
45
|
+
failure.retryableSpawn = true
|
|
46
|
+
return failure
|
|
47
|
+
}
|
|
39
48
|
if (error.code === 'ENOENT') return new GitMeasurementError('Git is unavailable')
|
|
40
49
|
if (error.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER' || error.code === 'ENOBUFS') return new GitMeasurementError('Git output exceeded the 16 MiB resource limit')
|
|
41
50
|
if (error.killed || error.signal || error.code === 'ETIMEDOUT') return new GitMeasurementError('Git command exceeded the time budget')
|
|
@@ -43,12 +52,27 @@ function gitError(error, stderr) {
|
|
|
43
52
|
return new GitMeasurementError('Git history is incomplete or unavailable locally')
|
|
44
53
|
}
|
|
45
54
|
// Never propagate command output, author identities or credentials to logs.
|
|
46
|
-
const safeError = new
|
|
55
|
+
const safeError = new GitMeasurementError('Git command failed')
|
|
47
56
|
safeError.code = error.code
|
|
48
57
|
return safeError
|
|
49
58
|
}
|
|
50
59
|
|
|
51
|
-
|
|
60
|
+
async function withSpawnRecovery(operation) {
|
|
61
|
+
try { return await operation() } catch (error) {
|
|
62
|
+
if (!error?.retryableSpawn || spawnRecoveryUsed) throw error
|
|
63
|
+
spawnRecoveryUsed = true
|
|
64
|
+
remainingTime()
|
|
65
|
+
await delay(Math.min(100, remainingTime()))
|
|
66
|
+
remainingTime()
|
|
67
|
+
return operation()
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export async function runGit(repoPath, args, options = {}) {
|
|
72
|
+
return withSpawnRecovery(() => runGitOnce(repoPath, args, options))
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function runGitOnce(repoPath, args, { allowMissingConfig = false } = {}) {
|
|
52
76
|
const timeout = Math.min(MAX_GIT_TIME_MS, remainingTime())
|
|
53
77
|
return new Promise((resolve, reject) => {
|
|
54
78
|
const child = execFile('git', gitArgs(repoPath, args), { encoding: 'utf8', maxBuffer: MAX_GIT_BUFFER, timeout, killSignal: 'SIGKILL', windowsHide: true, env: localGitEnvironment() }, (error, stdout, stderr) => {
|
|
@@ -63,6 +87,10 @@ export async function runGit(repoPath, args, { allowMissingConfig = false } = {}
|
|
|
63
87
|
|
|
64
88
|
/** Consume stdout a line at a time; retain at most one bounded line, not the full output. */
|
|
65
89
|
export async function runGitLines(repoPath, args, onLine) {
|
|
90
|
+
return withSpawnRecovery(() => runGitLinesOnce(repoPath, args, onLine))
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function runGitLinesOnce(repoPath, args, onLine) {
|
|
66
94
|
const timeout = Math.min(MAX_GIT_TIME_MS, remainingTime())
|
|
67
95
|
return new Promise((resolve, reject) => {
|
|
68
96
|
let pending = ''
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { realpath } from 'node:fs/promises'
|
|
2
2
|
import { analyzeDurability } from './GitDurabilityEngine.mjs'
|
|
3
3
|
import { ENGINE_COMMIT, validateReport } from './validate.mjs'
|
|
4
|
-
import { runGit, setDeadline, remainingTime, stopGitChildren, GitMeasurementError } from './runtime.mjs'
|
|
4
|
+
import { runGit, runGitLines, setDeadline, remainingTime, stopGitChildren, GitMeasurementError } from './runtime.mjs'
|
|
5
5
|
|
|
6
6
|
process.on('SIGTERM', () => { stopGitChildren(); process.exit(2) })
|
|
7
7
|
let completed = false
|
|
@@ -20,8 +20,9 @@ process.once('message', async ({ root, deadline }) => {
|
|
|
20
20
|
if (partial.split('\n').some(line => /^extensions\.partialclone\s+\S/i.test(line) || /^remote\..*\.promisor\s+(true|yes|on|1)\s*$/i.test(line))) throw new GitMeasurementError('partial Git history cannot be measured without fetching objects')
|
|
21
21
|
const headSha = (await runGit(root, ['rev-parse', '--verify', 'HEAD'])).trim()
|
|
22
22
|
if (!/^[0-9a-f]{40}$/.test(headSha)) throw new GitMeasurementError('a valid SHA-1 Git HEAD is required')
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
await runGitLines(root, ['rev-list', '--objects', '--no-object-names', '--missing=print', 'HEAD'], line => {
|
|
24
|
+
if (/^\?[0-9a-f]+$/.test(line)) throw new GitMeasurementError('Git history is missing local objects')
|
|
25
|
+
})
|
|
25
26
|
const measuredAt = new Date().toISOString()
|
|
26
27
|
const result = await analyzeDurability(root, {
|
|
27
28
|
windowDays: 90, settleDays: 14,
|
package/package.json
CHANGED