@scalequality/check 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -2
- package/bin/archive.mjs +10 -4
- package/bin/check.mjs +1 -0
- package/bin/composer-vendor.mjs +198 -0
- package/bin/durability/GitReader.mjs +16 -13
- package/bin/durability/PROVENANCE.md +10 -4
- package/bin/durability/provenance.json +3 -2
- package/bin/durability/runtime.mjs +82 -17
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -95,7 +95,7 @@ unavailable; it is not treated as a full 90-day measurement.
|
|
|
95
95
|
- uses: actions/checkout@v4
|
|
96
96
|
with:
|
|
97
97
|
fetch-depth: 0
|
|
98
|
-
- run: npx @scalequality/check@0.5.
|
|
98
|
+
- run: npx @scalequality/check@0.5.1 --project <projectId>
|
|
99
99
|
env:
|
|
100
100
|
SQ_TOKEN: ${{ secrets.SQ_TOKEN }}
|
|
101
101
|
```
|
|
@@ -126,9 +126,45 @@ that you expect to measure.
|
|
|
126
126
|
|
|
127
127
|
```bash
|
|
128
128
|
# Local verification; does not need a token and sends nothing to ScaleQuality.
|
|
129
|
-
npx @scalequality/check@0.5.
|
|
129
|
+
npx @scalequality/check@0.5.1 --dry-run
|
|
130
130
|
```
|
|
131
131
|
|
|
132
132
|
The CLI checks compressed size before allocating the upload buffer or sending a
|
|
133
133
|
request. If current source alone cannot fit, it lists the largest included files
|
|
134
134
|
and exits with an actionable error. No oversized upload is sent.
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
## 0.5.1: Composer dependencies and large Git blame output
|
|
138
|
+
|
|
139
|
+
The CLI recognizes installed Composer dependencies using `composer.json` and a
|
|
140
|
+
valid `composer.lock`. It excludes the default `vendor/`, or a safe relative
|
|
141
|
+
`config.vendor-dir`, while preserving both files for dependency analysis. Nested
|
|
142
|
+
Composer projects are recognized too. A directory named `vendor` alone is not
|
|
143
|
+
sufficient evidence to exclude it. Ambiguous configuration, source paths that
|
|
144
|
+
overlap the dependency directory, and unavailable local package origins keep the
|
|
145
|
+
directory in the archive. The CLI reports which directories it excluded or kept.
|
|
146
|
+
It reads JSON only; it does not run Composer, plugins or scripts.
|
|
147
|
+
|
|
148
|
+
`node_modules/` and the existing generated-output directories remain excluded.
|
|
149
|
+
Coverage reports from application tests are staged separately before packaging.
|
|
150
|
+
Excluding installed dependencies does not certify that their locked versions are
|
|
151
|
+
free of vulnerabilities.
|
|
152
|
+
|
|
153
|
+
Git durability reads incremental blame ranges as a stream instead of buffering
|
|
154
|
+
all per-line metadata and source content. This preserves the native calculation,
|
|
155
|
+
90-day window, 14-day settlement and tracked-file selection. It handles large
|
|
156
|
+
blame output without increasing the worker's memory or time budgets.
|
|
157
|
+
|
|
158
|
+
Archive exclusions, including `--exclude`, do not change Git durability's tracked
|
|
159
|
+
file scope. A tracked dependency can still contribute to the historical metric.
|
|
160
|
+
Changing that methodology requires a separate coordinated version; this patch
|
|
161
|
+
does not silently rewrite prior scores. Missing coverage or history is still
|
|
162
|
+
reported as unavailable.
|
|
163
|
+
|
|
164
|
+
Track-only completion now says `measurement completed (track only; no cutoff
|
|
165
|
+
configured)`. Its exit code remains zero; completion does not certify that every
|
|
166
|
+
dimension was measured or that a maturity threshold was passed.
|
|
167
|
+
|
|
168
|
+
The 60 MiB compressed-source limit still applies. A checkout with more than that
|
|
169
|
+
amount of first-party source or ambiguous generated assets is rejected locally;
|
|
170
|
+
this release does not truncate source to force a successful upload.
|
package/bin/archive.mjs
CHANGED
|
@@ -2,6 +2,7 @@ import { spawnSync } from "node:child_process"
|
|
|
2
2
|
import { mkdtempSync, mkdirSync, readdirSync, lstatSync, existsSync, statSync, writeFileSync, rmSync } from "node:fs"
|
|
3
3
|
import { tmpdir } from "node:os"
|
|
4
4
|
import { join, resolve, posix } from "node:path"
|
|
5
|
+
import { planComposerExclusions } from "./composer-vendor.mjs"
|
|
5
6
|
|
|
6
7
|
export const ARCHIVE_LIMIT_BYTES = 60 * 1024 * 1024
|
|
7
8
|
const MAX_DURABILITY_REPORT_BYTES = 1024 * 1024
|
|
@@ -23,8 +24,12 @@ export function normalizeExcludes(excludes = []) {
|
|
|
23
24
|
})
|
|
24
25
|
}
|
|
25
26
|
|
|
26
|
-
function inventory(root, { reports = false, excludes = [] } = {}) {
|
|
27
|
+
function inventory(root, { reports = false, excludes = [], log = () => {} } = {}) {
|
|
27
28
|
const paths = [], largest = []
|
|
29
|
+
const composerExcluded = reports ? [] : planComposerExclusions(root, excludes, SKIP, log)
|
|
30
|
+
for (const candidate of composerExcluded) {
|
|
31
|
+
log(`ScaleQuality: excluding installed Composer dependencies at ${JSON.stringify(candidate.path)}; retaining ${JSON.stringify(candidate.manifest)} and ${JSON.stringify(candidate.lockfile)} for dependency analysis. Git durability scope is unchanged.`)
|
|
32
|
+
}
|
|
28
33
|
const walk = (rel) => {
|
|
29
34
|
const abs = join(root, rel)
|
|
30
35
|
const st = lstatSync(abs)
|
|
@@ -36,12 +41,12 @@ function inventory(root, { reports = false, excludes = [] } = {}) {
|
|
|
36
41
|
}
|
|
37
42
|
}
|
|
38
43
|
const visit = (rel, name) => {
|
|
39
|
-
if (!reports && (name === ".git" || rel === ".sqreports" || excludes.some((p) => rel === p || rel.startsWith(`${p}/`)))) return
|
|
44
|
+
if (!reports && (name === ".git" || rel === ".sqreports" || [...excludes, ...composerExcluded.map(item => item.path)].some((p) => rel === p || rel.startsWith(`${p}/`)))) return
|
|
40
45
|
if (!reports && SKIP.has(name) && lstatSync(join(root, rel)).isDirectory()) return
|
|
41
46
|
walk(rel)
|
|
42
47
|
}
|
|
43
48
|
for (const entry of readdirSync(root).sort()) visit(entry, entry)
|
|
44
|
-
return { paths, largest: largest.sort((a, b) => b.bytes - a.bytes).slice(0, 5) }
|
|
49
|
+
return { paths, largest: largest.sort((a, b) => b.bytes - a.bytes).slice(0, 5), composerExcluded }
|
|
45
50
|
}
|
|
46
51
|
|
|
47
52
|
/**
|
|
@@ -58,7 +63,7 @@ export function prepareCheckoutArchive({ root, coverageRoot, durabilityReport, l
|
|
|
58
63
|
process.once("exit", onExit)
|
|
59
64
|
const cleanup = () => { process.removeListener("exit", onExit); rmSync(temp, { recursive: true, force: true }) }
|
|
60
65
|
try {
|
|
61
|
-
const source = inventory(root, { excludes })
|
|
66
|
+
const source = inventory(root, { excludes, log })
|
|
62
67
|
const sourceList = join(temp, "source.list")
|
|
63
68
|
writeFileSync(sourceList, source.paths.length ? `${source.paths.join("\0")}\0` : "")
|
|
64
69
|
const metadataRoot = join(temp, "metadata")
|
|
@@ -66,6 +71,7 @@ export function prepareCheckoutArchive({ root, coverageRoot, durabilityReport, l
|
|
|
66
71
|
const hasGit = existsSync(join(root, ".git"))
|
|
67
72
|
const history = { mode: hasGit ? "omitted" : "unavailable", reason: durabilityReport ? "ci_report" : hasGit ? "invalid_git" : "missing_git" }
|
|
68
73
|
writeFileSync(join(metadataRoot, ".sqreports/checkout.json"), JSON.stringify({ version: 1, history, archiveLimitBytes: ARCHIVE_LIMIT_BYTES }))
|
|
74
|
+
if (source.composerExcluded.length) writeFileSync(join(metadataRoot, ".sqreports/source.json"), JSON.stringify({ version: 1, composerDependencyExclusions: source.composerExcluded }))
|
|
69
75
|
if (durabilityReport) {
|
|
70
76
|
const json = JSON.stringify(durabilityReport)
|
|
71
77
|
if (Buffer.byteLength(json) > MAX_DURABILITY_REPORT_BYTES) throw new Error("Durability report exceeds the 1 MiB report limit. No upload was sent.")
|
package/bin/check.mjs
CHANGED
|
@@ -298,6 +298,7 @@ async function main() {
|
|
|
298
298
|
const cut = last.cutoff != null ? ` (cutoff ${last.cutoff})` : " (track only)"
|
|
299
299
|
log(`ScaleQuality: maturity ${last.maturity ?? "—"}${cut}`)
|
|
300
300
|
if (last.passed === false) console.error("ScaleQuality: below the line — failing the build.")
|
|
301
|
+
else if (last.cutoff == null) log("ScaleQuality: measurement completed (track only; no cutoff configured).")
|
|
301
302
|
else log("ScaleQuality: passed.")
|
|
302
303
|
}
|
|
303
304
|
process.exit(last.passed === false ? 1 : 0)
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import { constants, closeSync, fstatSync, lstatSync, openSync, readSync, readdirSync } from 'node:fs'
|
|
2
|
+
import { join, posix } from 'node:path'
|
|
3
|
+
|
|
4
|
+
const MAX_JSON_BYTES = 4 * 1024 * 1024
|
|
5
|
+
const object = value => value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
6
|
+
const overlaps = (a, b) => a === b || a.startsWith(`${b}/`) || b.startsWith(`${a}/`) || a === '.' || b === '.'
|
|
7
|
+
|
|
8
|
+
// Inspect data only. Composer plugins/scripts and customer symlinks are never run/followed.
|
|
9
|
+
function readJson(path) {
|
|
10
|
+
let fd
|
|
11
|
+
try {
|
|
12
|
+
fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK)
|
|
13
|
+
const stat = fstatSync(fd)
|
|
14
|
+
if (!stat.isFile() || stat.size > MAX_JSON_BYTES) return null
|
|
15
|
+
const raw = Buffer.alloc(stat.size + 1)
|
|
16
|
+
let bytes = 0
|
|
17
|
+
while (bytes < raw.length) {
|
|
18
|
+
const count = readSync(fd, raw, bytes, raw.length - bytes, null)
|
|
19
|
+
if (!count) break
|
|
20
|
+
bytes += count
|
|
21
|
+
}
|
|
22
|
+
if (bytes !== stat.size) return null
|
|
23
|
+
const value = JSON.parse(raw.subarray(0, bytes).toString('utf8'))
|
|
24
|
+
return object(value) ? value : null
|
|
25
|
+
} catch { return null }
|
|
26
|
+
finally { if (fd !== undefined) closeSync(fd) }
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function safeRelative(value) {
|
|
30
|
+
if (typeof value !== 'string' || !value || /[\\\0*?\[\]$~]/.test(value) || value.startsWith('/') || /^[A-Za-z]:/.test(value)) return null
|
|
31
|
+
if (value.split('/').includes('..')) return null
|
|
32
|
+
const path = posix.normalize(value).replace(/\/$/, '')
|
|
33
|
+
return path === '.' || path.split('/').some(part => ['.git', '.sqreports'].includes(part)) ? null : path
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function sourcePath(module, value) {
|
|
37
|
+
if (typeof value !== 'string' || /[\\\0$~]/.test(value) || value.startsWith('/') || /^[A-Za-z]:/.test(value)) return null
|
|
38
|
+
// A glob's fixed directory prefix bounds every possible source it might select.
|
|
39
|
+
const glob = value.search(/[*?\[]/)
|
|
40
|
+
const prefix = glob < 0 ? value : value.slice(0, glob).replace(/[^/]*$/, '')
|
|
41
|
+
const path = posix.normalize(posix.join(module || '.', prefix || '.')).replace(/\/$/, '')
|
|
42
|
+
return path === '..' || path.startsWith('../') ? null : path
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function autoloadPaths(value) {
|
|
46
|
+
if (value === undefined) return []
|
|
47
|
+
if (!object(value)) return [null]
|
|
48
|
+
const paths = []
|
|
49
|
+
for (const key of ['psr-4', 'psr-0', 'classmap', 'files']) {
|
|
50
|
+
const entry = value[key]
|
|
51
|
+
if (entry === undefined) continue
|
|
52
|
+
const values = ['psr-4', 'psr-0'].includes(key) ? object(entry) ? Object.values(entry) : [null] : Array.isArray(entry) ? entry : [null]
|
|
53
|
+
for (const item of values) paths.push(...(Array.isArray(item) ? item : [item]))
|
|
54
|
+
}
|
|
55
|
+
return paths
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function regularDirectory(root, path, consume = () => {}) {
|
|
59
|
+
let current = root
|
|
60
|
+
for (const part of path.split('/')) {
|
|
61
|
+
consume()
|
|
62
|
+
current = join(current, part)
|
|
63
|
+
try { if (!lstatSync(current).isDirectory()) return false }
|
|
64
|
+
catch { return false }
|
|
65
|
+
}
|
|
66
|
+
return true
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** A transport exclusion, never a change to the native Git durability scope. */
|
|
70
|
+
export function composerVendorDirectory(root, module = '', excludes = [], ignoredDirectories = new Set(), consume = () => {}) {
|
|
71
|
+
const manifest = readJson(join(root, module, 'composer.json'))
|
|
72
|
+
if (!manifest) return null
|
|
73
|
+
const label = posix.join(module, 'composer.json')
|
|
74
|
+
const keep = reason => ({ kept: true, manifest: label, reason })
|
|
75
|
+
const lock = readJson(join(root, module, 'composer.lock'))
|
|
76
|
+
if (!lock || !Array.isArray(lock.packages) || (lock['packages-dev'] !== undefined && !Array.isArray(lock['packages-dev']))) return keep('a valid composer.lock is required')
|
|
77
|
+
const packages = [...lock.packages, ...(lock['packages-dev'] || [])]
|
|
78
|
+
if (!packages.length || packages.some(pkg => !object(pkg) || typeof pkg.name !== 'string' || !pkg.name || typeof pkg.version !== 'string' || !pkg.version)) return keep('locked dependency versions could not be confirmed')
|
|
79
|
+
if (manifest.config !== undefined && !object(manifest.config)) return keep('Composer config is ambiguous')
|
|
80
|
+
const vendor = safeRelative(manifest.config && Object.hasOwn(manifest.config, 'vendor-dir') ? manifest.config['vendor-dir'] : 'vendor')
|
|
81
|
+
if (!vendor) return keep('vendor-dir must be a literal directory within the Composer project')
|
|
82
|
+
const path = posix.join(module, vendor)
|
|
83
|
+
// If either evidence file is explicitly excluded, do not infer a safe replacement for installed dependencies.
|
|
84
|
+
if (['composer.json', 'composer.lock'].some(file => excludes.some(ex => overlaps(ex, posix.join(module, file))))) return keep('the manifest or lockfile was explicitly excluded')
|
|
85
|
+
|
|
86
|
+
const repositories = manifest.repositories === undefined ? [] : Array.isArray(manifest.repositories) ? manifest.repositories : object(manifest.repositories) ? Object.values(manifest.repositories) : [null]
|
|
87
|
+
if (repositories.some(repo => repo !== false && !object(repo))) return keep('Composer repositories are ambiguous')
|
|
88
|
+
const localOrigins = [
|
|
89
|
+
...repositories.filter(repo => repo && repo.type === 'path').map(repo => repo.url),
|
|
90
|
+
...packages.filter(pkg => pkg.dist?.type === 'path').map(pkg => pkg.dist.url),
|
|
91
|
+
]
|
|
92
|
+
// Resolving Composer's glob rules and each symlink target is outside this transport hotfix.
|
|
93
|
+
if (localOrigins.some(value => typeof value !== 'string' || /[*?\[]/.test(value))) return keep('local package origins use ambiguous paths or globs')
|
|
94
|
+
const sources = [...autoloadPaths(manifest.autoload), ...autoloadPaths(manifest['autoload-dev']), ...localOrigins]
|
|
95
|
+
if (sources.some(value => { consume(); const source = sourcePath(module, value); return source === null || overlaps(path, source) })) return keep('local source paths may overlap installed dependencies or lie outside the checkout')
|
|
96
|
+
if (localOrigins.some(value => {
|
|
97
|
+
const source = sourcePath(module, value)
|
|
98
|
+
consume()
|
|
99
|
+
return !regularDirectory(root, source, consume) || source.split('/').some(part => ignoredDirectories.has(part) || part === 'vendor') || excludes.some(ex => overlaps(ex, source))
|
|
100
|
+
})) return keep('a local package origin is missing, linked or excluded from the archive')
|
|
101
|
+
// A custom vendor-dir can have several components: reject a symlink in ANY component.
|
|
102
|
+
if (!regularDirectory(root, path, consume)) return null
|
|
103
|
+
return { path, manifest: label, lockfile: posix.join(module, 'composer.lock') }
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function declaredSources(root, module, consume) {
|
|
107
|
+
const manifest = readJson(join(root, module, 'composer.json'))
|
|
108
|
+
if (!manifest) return [null]
|
|
109
|
+
const repositories = manifest.repositories === undefined ? [] : Array.isArray(manifest.repositories) ? manifest.repositories : object(manifest.repositories) ? Object.values(manifest.repositories) : [null]
|
|
110
|
+
if (repositories.some(repo => repo !== false && !object(repo))) return [null]
|
|
111
|
+
const lock = readJson(join(root, module, 'composer.lock'))
|
|
112
|
+
let packages = []
|
|
113
|
+
try {
|
|
114
|
+
lstatSync(join(root, module, 'composer.lock'))
|
|
115
|
+
if (!lock || !Array.isArray(lock.packages) || (lock['packages-dev'] !== undefined && !Array.isArray(lock['packages-dev']))) return [null]
|
|
116
|
+
packages = [...lock.packages, ...(lock['packages-dev'] || [])]
|
|
117
|
+
if (packages.some(pkg => !object(pkg))) return [null]
|
|
118
|
+
} catch (error) { if (error.code !== 'ENOENT') return [null] }
|
|
119
|
+
return [
|
|
120
|
+
...autoloadPaths(manifest.autoload), ...autoloadPaths(manifest['autoload-dev']),
|
|
121
|
+
...repositories.filter(repo => repo && repo.type === 'path').map(repo => repo.url),
|
|
122
|
+
...packages.filter(pkg => pkg.dist?.type === 'path').map(pkg => pkg.dist.url),
|
|
123
|
+
].map(value => {
|
|
124
|
+
consume()
|
|
125
|
+
// Resolving a glob or symlink could reveal another project's only source copy.
|
|
126
|
+
if (typeof value !== 'string' || /[*?\[]/.test(value)) return null
|
|
127
|
+
const path = sourcePath(module, value)
|
|
128
|
+
if (path === null) return null
|
|
129
|
+
let current = root
|
|
130
|
+
for (const part of path.split('/')) {
|
|
131
|
+
consume()
|
|
132
|
+
current = join(current, part)
|
|
133
|
+
try { if (lstatSync(current).isSymbolicLink()) return null }
|
|
134
|
+
catch (error) { if (error.code === 'ENOENT') break; return null }
|
|
135
|
+
}
|
|
136
|
+
return path
|
|
137
|
+
})
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** Discover all projects before applying any Composer exclusion. References can cross modules. */
|
|
141
|
+
export function planComposerExclusions(root, excludes = [], ignoredDirectories = new Set(), log = () => {}) {
|
|
142
|
+
const limit = new Error('Composer discovery resource limit')
|
|
143
|
+
const deadline = Date.now() + 5_000
|
|
144
|
+
let work = 0
|
|
145
|
+
const consume = () => { if (++work > 1_000_000 || Date.now() > deadline) throw limit }
|
|
146
|
+
try {
|
|
147
|
+
const projects = []
|
|
148
|
+
const pending = ['']
|
|
149
|
+
let visited = 0
|
|
150
|
+
while (pending.length) {
|
|
151
|
+
consume()
|
|
152
|
+
const module = pending.pop()
|
|
153
|
+
const entries = readdirSync(join(root, module), { withFileTypes: true })
|
|
154
|
+
visited += entries.length
|
|
155
|
+
if (visited > 100_000 || projects.length > 5_000) throw limit
|
|
156
|
+
if (entries.some(entry => entry.name === 'composer.json')) projects.push({
|
|
157
|
+
module,
|
|
158
|
+
candidate: composerVendorDirectory(root, module, excludes, ignoredDirectories, consume),
|
|
159
|
+
sources: declaredSources(root, module, consume),
|
|
160
|
+
})
|
|
161
|
+
for (const entry of entries.sort((a, b) => b.name.localeCompare(a.name))) {
|
|
162
|
+
consume()
|
|
163
|
+
const path = posix.join(module, entry.name)
|
|
164
|
+
if (!entry.isDirectory() || entry.name === '.git' || path === '.sqreports' || ignoredDirectories.has(entry.name) || excludes.some(ex => path === ex || path.startsWith(`${ex}/`))) continue
|
|
165
|
+
pending.push(path)
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
const candidates = projects.flatMap(project => project.candidate?.path ? [project.candidate] : [])
|
|
169
|
+
const protectedPaths = new Set()
|
|
170
|
+
const isActive = project => !candidates.some(candidate => { consume(); return !protectedPaths.has(candidate.path) && (project.module === candidate.path || project.module.startsWith(`${candidate.path}/`)) })
|
|
171
|
+
// Dependency package manifests do not veto their containing vendor directory.
|
|
172
|
+
// Once an external project references it, it is retained and its declarations
|
|
173
|
+
// become relevant too. Iterate until no newly retained project protects another.
|
|
174
|
+
let changed
|
|
175
|
+
do {
|
|
176
|
+
changed = false
|
|
177
|
+
const sources = projects.filter(isActive).flatMap(project => project.sources)
|
|
178
|
+
for (const candidate of candidates) {
|
|
179
|
+
if (!protectedPaths.has(candidate.path) && sources.some(source => { consume(); return source === null || overlaps(candidate.path, source) })) {
|
|
180
|
+
protectedPaths.add(candidate.path)
|
|
181
|
+
changed = true
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
} while (changed)
|
|
185
|
+
for (const project of projects.filter(isActive)) {
|
|
186
|
+
if (project.candidate?.kept) log(`ScaleQuality: retaining possible Composer dependencies near ${JSON.stringify(project.candidate.manifest)}: ${project.candidate.reason}.`)
|
|
187
|
+
}
|
|
188
|
+
for (const candidate of candidates) {
|
|
189
|
+
if (protectedPaths.has(candidate.path)) log(`ScaleQuality: retaining ${JSON.stringify(candidate.path)}: a project declares source there or its source paths could not be resolved safely.`)
|
|
190
|
+
}
|
|
191
|
+
// Nested installed package manifests cannot create independent upload exclusions.
|
|
192
|
+
return projects.filter(isActive).flatMap(project => project.candidate?.path && !protectedPaths.has(project.candidate.path) ? [project.candidate] : [])
|
|
193
|
+
} catch (error) {
|
|
194
|
+
if (error !== limit) throw error
|
|
195
|
+
log('ScaleQuality: retaining Composer dependency directories: project discovery exceeded its resource limit.')
|
|
196
|
+
return []
|
|
197
|
+
}
|
|
198
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { runGit as git, isFatalGitError } from "./runtime.mjs";
|
|
1
|
+
import { runGit as git, runGitLines, isFatalGitError, GitMeasurementError } from "./runtime.mjs";
|
|
2
2
|
const US = "\x1f"; // unit separator (field)
|
|
3
3
|
const RS = "\x1e"; // record separator (commit)
|
|
4
4
|
const CO_AUTHOR_RE = /co-authored-by:\s*(.*?)\s*<([^>]+)>/gi;
|
|
@@ -87,32 +87,35 @@ export async function listHeadFiles(repoPath) {
|
|
|
87
87
|
const out = await git(repoPath, ["ls-tree", "-r", "--name-only", "HEAD"]);
|
|
88
88
|
return new Set(out.split("\n").map((l) => l.trim()).filter(Boolean));
|
|
89
89
|
}
|
|
90
|
-
const
|
|
90
|
+
const BLAME_GROUP = /^([0-9a-f]{40}) [1-9][0-9]* [1-9][0-9]* ([1-9][0-9]*)$/;
|
|
91
91
|
/**
|
|
92
92
|
* Blames a file at HEAD and returns, per introducing commit sha, how many
|
|
93
|
-
* CURRENT lines it owns
|
|
94
|
-
*
|
|
95
|
-
*
|
|
93
|
+
* CURRENT lines it owns. Incremental output describes identical attribution
|
|
94
|
+
* in groups, without source lines or repeated per-line commit metadata.
|
|
95
|
+
* Stream those groups so large output does not require a large buffer.
|
|
96
|
+
* Ordinary per-file failures return no counts; resource/history errors abort.
|
|
96
97
|
*/
|
|
97
98
|
export async function blameFile(repoPath, file) {
|
|
98
99
|
const counts = new Map();
|
|
99
100
|
try {
|
|
100
|
-
|
|
101
|
+
await runGitLines(repoPath, [
|
|
101
102
|
"blame",
|
|
102
|
-
"--
|
|
103
|
+
"--incremental",
|
|
103
104
|
"HEAD",
|
|
104
105
|
"--",
|
|
105
106
|
file,
|
|
106
|
-
]
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
107
|
+
], line => {
|
|
108
|
+
const m = BLAME_GROUP.exec(line);
|
|
109
|
+
if (!m) return;
|
|
110
|
+
const total = (counts.get(m[1]) ?? 0) + Number(m[2]);
|
|
111
|
+
if (!Number.isSafeInteger(total)) throw new GitMeasurementError('Git blame line count exceeded the numeric limit');
|
|
112
|
+
counts.set(m[1], total);
|
|
113
|
+
});
|
|
112
114
|
}
|
|
113
115
|
catch (error) {
|
|
114
116
|
if (isFatalGitError(error)) throw error;
|
|
115
117
|
// deleted at HEAD / binary / unreadable → no surviving lines counted
|
|
118
|
+
counts.clear();
|
|
116
119
|
}
|
|
117
120
|
return counts;
|
|
118
121
|
}
|
|
@@ -12,9 +12,13 @@ moduleResolution node, strict true, esModuleInterop true, skipLibCheck true, and
|
|
|
12
12
|
Node type declarations. No dependencies are needed by the resulting runtime.
|
|
13
13
|
Rename the generated files to `.mjs` and give their relative imports `.mjs`
|
|
14
14
|
extensions. The engine's calculations and classifier remain unchanged. In the
|
|
15
|
-
reader, replace its promisified `execFile` wrapper with `runtime.mjs`'s `runGit
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
reader, replace its promisified `execFile` wrapper with `runtime.mjs`'s `runGit`.
|
|
16
|
+
For blame, use `runGitLines` with `--incremental` and sum each group's line count
|
|
17
|
+
instead of counting individual `--line-porcelain` headers. Attribution is unchanged;
|
|
18
|
+
source text and repeated per-line metadata are not emitted or retained. Rethrow
|
|
19
|
+
`isFatalGitError(error)` in the per-file blame catch. Discard partial streamed
|
|
20
|
+
counts on ordinary per-file failures, retaining the native empty-map behavior.
|
|
21
|
+
Recompute provenance hashes and run the
|
|
18
22
|
parity and cancellation tests. The reference modules under
|
|
19
23
|
`test/native-durability-reference/` preserve the original reader for small fixture
|
|
20
24
|
comparisons only; they are excluded from the published package.
|
|
@@ -27,7 +31,9 @@ part of native Git measurement even if archive exclusions omit those directories
|
|
|
27
31
|
No clone, fetch, checkout or Git write is performed. Shallow or partial histories,
|
|
28
32
|
missing objects, changing HEAD and invalid reports produce no measurement.
|
|
29
33
|
The separate worker has a 120-second total budget and 512 MiB V8 heap ceiling;
|
|
30
|
-
each Git command has at most 30 seconds
|
|
34
|
+
each Git command has at most 30 seconds. Buffered commands retain at most 16 MiB
|
|
35
|
+
of output. Incremental blame streams stdout, retaining one line (at most 16 MiB)
|
|
36
|
+
and at most 16 MiB of stderr, rather than limiting total streamed output. The complete HEAD
|
|
31
37
|
object inventory uses this same limit, so sufficiently large object inventories
|
|
32
38
|
can be unavailable even when the 90-day change window is small. Limit failures
|
|
33
39
|
are reported as unavailable, never converted into successful zero measurements.
|
|
@@ -29,7 +29,8 @@
|
|
|
29
29
|
"runtimeChanges": [
|
|
30
30
|
"ESM relative import extensions only in engine/classifier",
|
|
31
31
|
"GitReader process wrapper replaced by bounded local-only runner",
|
|
32
|
-
"GitReader blame catch rethrows resource/incomplete-history failures; ordinary per-file errors retain native handling"
|
|
32
|
+
"GitReader blame catch rethrows resource/incomplete-history failures; ordinary per-file errors retain native handling",
|
|
33
|
+
"GitReader blame streams incremental groups and sums their line counts with exact native attribution; bounded lines and stderr preserve resource limits without capping total streamed stdout"
|
|
33
34
|
],
|
|
34
35
|
"generatedFiles": [
|
|
35
36
|
{
|
|
@@ -38,7 +39,7 @@
|
|
|
38
39
|
},
|
|
39
40
|
{
|
|
40
41
|
"file": "GitReader.mjs",
|
|
41
|
-
"sha256": "
|
|
42
|
+
"sha256": "727cc9dd9278ac29dd18d5d482918c20ee7c7550356376614051566ca2f4abd0"
|
|
42
43
|
},
|
|
43
44
|
{
|
|
44
45
|
"file": "OriginClassifier.mjs",
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { execFile } from 'node:child_process'
|
|
1
|
+
import { execFile, spawn } from 'node:child_process'
|
|
2
2
|
|
|
3
3
|
export const MAX_GIT_BUFFER = 16 * 1024 * 1024
|
|
4
4
|
export const MAX_GIT_TIME_MS = 30_000
|
|
@@ -27,29 +27,94 @@ export function localGitEnvironment() {
|
|
|
27
27
|
return { ...env, LC_ALL: 'C', GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: process.platform === 'win32' ? 'NUL' : '/dev/null', GIT_TERMINAL_PROMPT: '0', GIT_OPTIONAL_LOCKS: '0', GIT_NO_LAZY_FETCH: '1' }
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
+
function gitArgs(repoPath, args) {
|
|
31
|
+
return [
|
|
32
|
+
'--no-pager', '-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=' + (process.platform === 'win32' ? 'NUL' : '/dev/null'),
|
|
33
|
+
'-c', 'credential.helper=', '-c', 'protocol.allow=never', '-c', 'protocol.file.allow=never',
|
|
34
|
+
'-C', repoPath, ...args,
|
|
35
|
+
]
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function gitError(error, stderr) {
|
|
39
|
+
if (error.code === 'ENOENT') return new GitMeasurementError('Git is unavailable')
|
|
40
|
+
if (error.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER' || error.code === 'ENOBUFS') return new GitMeasurementError('Git output exceeded the 16 MiB resource limit')
|
|
41
|
+
if (error.killed || error.signal || error.code === 'ETIMEDOUT') return new GitMeasurementError('Git command exceeded the time budget')
|
|
42
|
+
if (/missing (?:blob|tree|commit|object)|bad object|unable to read|could not read|object .*corrupt|object file .*empty|invalid object|promisor|transport .*not allowed/i.test(stderr || '')) {
|
|
43
|
+
return new GitMeasurementError('Git history is incomplete or unavailable locally')
|
|
44
|
+
}
|
|
45
|
+
// Never propagate command output, author identities or credentials to logs.
|
|
46
|
+
const safeError = new Error('Git command failed')
|
|
47
|
+
safeError.code = error.code
|
|
48
|
+
return safeError
|
|
49
|
+
}
|
|
50
|
+
|
|
30
51
|
export async function runGit(repoPath, args, { allowMissingConfig = false } = {}) {
|
|
31
52
|
const timeout = Math.min(MAX_GIT_TIME_MS, remainingTime())
|
|
32
53
|
return new Promise((resolve, reject) => {
|
|
33
|
-
const child = execFile('git',
|
|
34
|
-
'--no-pager', '-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=' + (process.platform === 'win32' ? 'NUL' : '/dev/null'),
|
|
35
|
-
'-c', 'credential.helper=', '-c', 'protocol.allow=never', '-c', 'protocol.file.allow=never',
|
|
36
|
-
'-C', repoPath, ...args,
|
|
37
|
-
], { encoding: 'utf8', maxBuffer: MAX_GIT_BUFFER, timeout, killSignal: 'SIGKILL', windowsHide: true, env: localGitEnvironment() }, (error, stdout, stderr) => {
|
|
54
|
+
const child = execFile('git', gitArgs(repoPath, args), { encoding: 'utf8', maxBuffer: MAX_GIT_BUFFER, timeout, killSignal: 'SIGKILL', windowsHide: true, env: localGitEnvironment() }, (error, stdout, stderr) => {
|
|
38
55
|
children.delete(child)
|
|
39
56
|
if (!error) { try { remainingTime(); resolve(stdout) } catch (e) { reject(e) }; return }
|
|
40
57
|
if (allowMissingConfig && error.code === 1) { resolve(''); return }
|
|
41
|
-
|
|
42
|
-
if (error.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER' || error.code === 'ENOBUFS') { reject(new GitMeasurementError('Git output exceeded the 16 MiB resource limit')); return }
|
|
43
|
-
if (error.killed || error.signal || error.code === 'ETIMEDOUT') { reject(new GitMeasurementError('Git command exceeded the time budget')); return }
|
|
44
|
-
if (/missing (?:blob|tree|commit|object)|bad object|unable to read|could not read|object .*corrupt|object file .*empty|invalid object|promisor|transport .*not allowed/i.test(stderr || '')) {
|
|
45
|
-
reject(new GitMeasurementError('Git history is incomplete or unavailable locally')); return
|
|
46
|
-
}
|
|
47
|
-
// Preserve the native reader's handling of ordinary per-file blame errors.
|
|
48
|
-
// Never propagate command output, author identities or credentials to logs.
|
|
49
|
-
const safeError = new Error('Git command failed')
|
|
50
|
-
safeError.code = error.code
|
|
51
|
-
reject(safeError)
|
|
58
|
+
reject(gitError(error, stderr))
|
|
52
59
|
})
|
|
53
60
|
children.add(child)
|
|
54
61
|
})
|
|
55
62
|
}
|
|
63
|
+
|
|
64
|
+
/** Consume stdout a line at a time; retain at most one bounded line, not the full output. */
|
|
65
|
+
export async function runGitLines(repoPath, args, onLine) {
|
|
66
|
+
const timeout = Math.min(MAX_GIT_TIME_MS, remainingTime())
|
|
67
|
+
return new Promise((resolve, reject) => {
|
|
68
|
+
let pending = ''
|
|
69
|
+
let stderr = ''
|
|
70
|
+
let failure
|
|
71
|
+
const child = spawn('git', gitArgs(repoPath, args), {
|
|
72
|
+
stdio: ['ignore', 'pipe', 'pipe'], timeout, killSignal: 'SIGKILL', windowsHide: true, env: localGitEnvironment(),
|
|
73
|
+
})
|
|
74
|
+
children.add(child)
|
|
75
|
+
const fail = error => {
|
|
76
|
+
failure ??= error
|
|
77
|
+
child.kill('SIGKILL')
|
|
78
|
+
}
|
|
79
|
+
const emit = line => {
|
|
80
|
+
if (Buffer.byteLength(line) > MAX_GIT_BUFFER) throw gitError({ code: 'ENOBUFS' })
|
|
81
|
+
onLine(line)
|
|
82
|
+
}
|
|
83
|
+
child.stdout.setEncoding('utf8')
|
|
84
|
+
child.stdout.on('data', chunk => {
|
|
85
|
+
if (failure) return
|
|
86
|
+
try {
|
|
87
|
+
remainingTime()
|
|
88
|
+
const text = pending + chunk
|
|
89
|
+
let start = 0
|
|
90
|
+
for (let end = text.indexOf('\n'); end !== -1; end = text.indexOf('\n', start)) {
|
|
91
|
+
emit(text.slice(start, end))
|
|
92
|
+
start = end + 1
|
|
93
|
+
}
|
|
94
|
+
pending = text.slice(start)
|
|
95
|
+
if (Buffer.byteLength(pending) > MAX_GIT_BUFFER) throw gitError({ code: 'ENOBUFS' })
|
|
96
|
+
} catch (error) {
|
|
97
|
+
fail(isFatalGitError(error) ? error : new GitMeasurementError('Git output could not be parsed'))
|
|
98
|
+
}
|
|
99
|
+
})
|
|
100
|
+
child.stderr.setEncoding('utf8')
|
|
101
|
+
child.stderr.on('data', chunk => {
|
|
102
|
+
if (failure) return
|
|
103
|
+
if (Buffer.byteLength(stderr) + Buffer.byteLength(chunk) > MAX_GIT_BUFFER) fail(gitError({ code: 'ENOBUFS' }))
|
|
104
|
+
else stderr += chunk
|
|
105
|
+
})
|
|
106
|
+
child.on('error', error => { failure ??= gitError(error, stderr) })
|
|
107
|
+
child.on('close', (code, signal) => {
|
|
108
|
+
children.delete(child)
|
|
109
|
+
if (failure) { reject(failure); return }
|
|
110
|
+
if (code !== 0 || signal) { reject(gitError({ code, signal, killed: child.killed }, stderr)); return }
|
|
111
|
+
try {
|
|
112
|
+
remainingTime()
|
|
113
|
+
if (pending) emit(pending)
|
|
114
|
+
resolve()
|
|
115
|
+
} catch (error) {
|
|
116
|
+
reject(isFatalGitError(error) ? error : new GitMeasurementError('Git output could not be parsed'))
|
|
117
|
+
}
|
|
118
|
+
})
|
|
119
|
+
})
|
|
120
|
+
}
|
package/package.json
CHANGED