@scalequality/check 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +97 -3
- package/bin/archive.mjs +115 -0
- package/bin/check.mjs +41 -34
- package/bin/composer-vendor.mjs +198 -0
- package/bin/durability/GitDurabilityEngine.mjs +225 -0
- package/bin/durability/GitReader.mjs +121 -0
- package/bin/durability/OriginClassifier.mjs +48 -0
- package/bin/durability/PROVENANCE.md +42 -0
- package/bin/durability/provenance.json +53 -0
- package/bin/durability/runtime.mjs +120 -0
- package/bin/durability/types.mjs +5 -0
- package/bin/durability/validate.mjs +31 -0
- package/bin/durability/worker.mjs +44 -0
- package/bin/durability.mjs +65 -0
- package/package.json +4 -1
package/README.md
CHANGED
|
@@ -21,6 +21,8 @@ and **Turn on continuous measurement**. Store the token as a CI secret named
|
|
|
21
21
|
| `--host` | `SQ_HOST` | `https://app.scalequality.io` | API host |
|
|
22
22
|
| `--timeout` | — | `600` | Max seconds to wait for the measurement |
|
|
23
23
|
| `--json` | — | off | Print the raw result as JSON |
|
|
24
|
+
| `--exclude <path>` | — | — | Exclude a generated file/directory by literal checkout-relative path; repeatable |
|
|
25
|
+
| `--dry-run` | — | off | Measure locally and validate the package without uploading; no token needed |
|
|
24
26
|
|
|
25
27
|
Exit codes: `0` passed (or track-only), `1` below the cutoff, `2` error.
|
|
26
28
|
|
|
@@ -62,9 +64,9 @@ Recognised out of the box: JaCoCo (`jacoco.xml`), LCOV (`lcov.info`), Cobertura
|
|
|
62
64
|
(`coverage.xml`, `coverage.cobertura.xml`), Istanbul (`coverage-summary.json`),
|
|
63
65
|
Clover (`clover.xml`) and Go (`coverage.out`). Multi-module builds are summed.
|
|
64
66
|
|
|
65
|
-
The reports are copied
|
|
66
|
-
|
|
67
|
-
|
|
67
|
+
The reports are copied outside your checkout into temporary staging, then stored
|
|
68
|
+
as `.sqreports/` entries in the upload. Build-output exclusions do not discard
|
|
69
|
+
these measured reports, and your working tree is not modified.
|
|
68
70
|
|
|
69
71
|
Report somewhere unusual? Point at it in `scalequality.properties` at the repo
|
|
70
72
|
root:
|
|
@@ -74,3 +76,95 @@ sq.coverage.reportPaths=ops/reports/jacoco-aggregate.xml
|
|
|
74
76
|
```
|
|
75
77
|
|
|
76
78
|
Full guide: https://scalequality.io/docs/continuous-measurement
|
|
79
|
+
|
|
80
|
+
## Fixed history window and large repositories (0.5.0)
|
|
81
|
+
|
|
82
|
+
Durability is calculated **on the CI runner**, against its existing Git history,
|
|
83
|
+
using the same native engine as ScaleQuality: a fixed **90-day** window, with
|
|
84
|
+
**14 days** for introduced code to settle before survival is measured. The source
|
|
85
|
+
engine revision and report time travel with the result. The CLI sends current
|
|
86
|
+
source, coverage, and a bounded durability report (at most 1 MiB); it never uploads
|
|
87
|
+
the raw `.git` object database. Years of old binary history therefore do not add
|
|
88
|
+
to the archive size. It does not truncate or change your repository.
|
|
89
|
+
|
|
90
|
+
Use a complete checkout for that measurement. Shallow history is reported as
|
|
91
|
+
unavailable; it is not treated as a full 90-day measurement.
|
|
92
|
+
|
|
93
|
+
```yaml
|
|
94
|
+
# GitHub Actions
|
|
95
|
+
- uses: actions/checkout@v4
|
|
96
|
+
with:
|
|
97
|
+
fetch-depth: 0
|
|
98
|
+
- run: npx @scalequality/check@0.5.1 --project <projectId>
|
|
99
|
+
env:
|
|
100
|
+
SQ_TOKEN: ${{ secrets.SQ_TOKEN }}
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
For GitLab set `GIT_DEPTH: "0"`. For Bitbucket Pipelines set:
|
|
104
|
+
|
|
105
|
+
```yaml
|
|
106
|
+
clone:
|
|
107
|
+
depth: full
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
The current-code scanners still run on the server. Historical secret scanning and
|
|
111
|
+
commit-convention analysis are not included in this report; the result shows those
|
|
112
|
+
limits and identifies durability as reported by CI. CPU, time, malformed Git or
|
|
113
|
+
report-size limits yield unavailable history, never a fabricated clean result.
|
|
114
|
+
|
|
115
|
+
Version 0.5.0 needs the compatible ScaleQuality server release. The CLI checks the
|
|
116
|
+
server's acknowledgement of this exact report before declaring a passing gate.
|
|
117
|
+
|
|
118
|
+
The compressed **source-and-reports archive** remains limited to 60 MiB. A time
|
|
119
|
+
window cannot guarantee that arbitrary source assets fit: one current binary can
|
|
120
|
+
exceed the limit. Generated dependency/output directories (`node_modules`,
|
|
121
|
+
`dist`, `build`, `.next`, `out`, `target`, `.venv`, `venv`, `coverage`,
|
|
122
|
+
`.turbo`) are excluded at every depth, while lockfiles are retained.
|
|
123
|
+
Use `--exclude generated/assets` for other generated content. Paths are literal,
|
|
124
|
+
relative to the checkout, and do not use shell wildcards. Do not exclude source
|
|
125
|
+
that you expect to measure.
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
# Local verification; does not need a token and sends nothing to ScaleQuality.
|
|
129
|
+
npx @scalequality/check@0.5.1 --dry-run
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
The CLI checks compressed size before allocating the upload buffer or sending a
|
|
133
|
+
request. If current source alone cannot fit, it lists the largest included files
|
|
134
|
+
and exits with an actionable error. No oversized upload is sent.
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
## 0.5.1: Composer dependencies and large Git blame output
|
|
138
|
+
|
|
139
|
+
The CLI recognizes installed Composer dependencies using `composer.json` and a
|
|
140
|
+
valid `composer.lock`. It excludes the default `vendor/`, or a safe relative
|
|
141
|
+
`config.vendor-dir`, while preserving both files for dependency analysis. Nested
|
|
142
|
+
Composer projects are recognized too. A directory named `vendor` alone is not
|
|
143
|
+
sufficient evidence to exclude it. Ambiguous configuration, source paths that
|
|
144
|
+
overlap the dependency directory, and unavailable local package origins keep the
|
|
145
|
+
directory in the archive. The CLI reports which directories it excluded or kept.
|
|
146
|
+
It reads JSON only; it does not run Composer, plugins or scripts.
|
|
147
|
+
|
|
148
|
+
`node_modules/` and the existing generated-output directories remain excluded.
|
|
149
|
+
Coverage reports from application tests are staged separately before packaging.
|
|
150
|
+
Excluding installed dependencies does not certify that their locked versions are
|
|
151
|
+
free of vulnerabilities.
|
|
152
|
+
|
|
153
|
+
Git durability reads incremental blame ranges as a stream instead of buffering
|
|
154
|
+
all per-line metadata and source content. This preserves the native calculation,
|
|
155
|
+
90-day window, 14-day settlement and tracked-file selection. It handles large
|
|
156
|
+
blame output without increasing the worker's memory or time budgets.
|
|
157
|
+
|
|
158
|
+
Archive exclusions, including `--exclude`, do not change Git durability's tracked
|
|
159
|
+
file scope. A tracked dependency can still contribute to the historical metric.
|
|
160
|
+
Changing that methodology requires a separate coordinated version; this patch
|
|
161
|
+
does not silently rewrite prior scores. Missing coverage or history is still
|
|
162
|
+
reported as unavailable.
|
|
163
|
+
|
|
164
|
+
Track-only completion now says `measurement completed (track only; no cutoff
|
|
165
|
+
configured)`. Its exit code remains zero; completion does not certify that every
|
|
166
|
+
dimension was measured or that a maturity threshold was passed.
|
|
167
|
+
|
|
168
|
+
The 60 MiB compressed-source limit still applies. A checkout with more than that
|
|
169
|
+
amount of first-party source or ambiguous generated assets is rejected locally;
|
|
170
|
+
this release does not truncate source to force a successful upload.
|
package/bin/archive.mjs
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process"
|
|
2
|
+
import { mkdtempSync, mkdirSync, readdirSync, lstatSync, existsSync, statSync, writeFileSync, rmSync } from "node:fs"
|
|
3
|
+
import { tmpdir } from "node:os"
|
|
4
|
+
import { join, resolve, posix } from "node:path"
|
|
5
|
+
import { planComposerExclusions } from "./composer-vendor.mjs"
|
|
6
|
+
|
|
7
|
+
export const ARCHIVE_LIMIT_BYTES = 60 * 1024 * 1024
|
|
8
|
+
const MAX_DURABILITY_REPORT_BYTES = 1024 * 1024
|
|
9
|
+
// Directory names, at every depth. Explicit file manifests avoid GNU/BSD tar
|
|
10
|
+
// differences in matching nested output/dependency directories.
|
|
11
|
+
const SKIP = new Set(["node_modules", "dist", "build", ".next", "out", "target", ".venv", "venv", "coverage", ".turbo"])
|
|
12
|
+
const mib = (bytes) => (bytes / 1024 / 1024).toFixed(1)
|
|
13
|
+
|
|
14
|
+
export function normalizeExcludes(excludes = []) {
|
|
15
|
+
return excludes.map((value) => {
|
|
16
|
+
if (typeof value !== "string" || !value || value.startsWith("-") || value.includes("\0") || value.includes("\\") || /[*?\[\]]/.test(value)) {
|
|
17
|
+
throw new Error("--exclude needs a literal path relative to the checkout (no wildcards).")
|
|
18
|
+
}
|
|
19
|
+
const path = posix.normalize(value.replace(/^\.\//, "")).replace(/\/$/, "")
|
|
20
|
+
if (!path || path === "." || path.startsWith("/") || path === ".." || path.startsWith("../") || /^[A-Za-z]:/.test(path) || [".git", ".sqreports"].includes(path.split("/")[0])) {
|
|
21
|
+
throw new Error("--exclude must stay inside the checkout and cannot exclude .git or .sqreports.")
|
|
22
|
+
}
|
|
23
|
+
return path
|
|
24
|
+
})
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function inventory(root, { reports = false, excludes = [], log = () => {} } = {}) {
|
|
28
|
+
const paths = [], largest = []
|
|
29
|
+
const composerExcluded = reports ? [] : planComposerExclusions(root, excludes, SKIP, log)
|
|
30
|
+
for (const candidate of composerExcluded) {
|
|
31
|
+
log(`ScaleQuality: excluding installed Composer dependencies at ${JSON.stringify(candidate.path)}; retaining ${JSON.stringify(candidate.manifest)} and ${JSON.stringify(candidate.lockfile)} for dependency analysis. Git durability scope is unchanged.`)
|
|
32
|
+
}
|
|
33
|
+
const walk = (rel) => {
|
|
34
|
+
const abs = join(root, rel)
|
|
35
|
+
const st = lstatSync(abs)
|
|
36
|
+
paths.push(`./${rel}`)
|
|
37
|
+
if (st.isDirectory()) {
|
|
38
|
+
for (const entry of readdirSync(abs).sort()) visit(rel ? `${rel}/${entry}` : entry, entry)
|
|
39
|
+
} else if (st.isFile()) {
|
|
40
|
+
largest.push({ path: rel, bytes: st.size })
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
const visit = (rel, name) => {
|
|
44
|
+
if (!reports && (name === ".git" || rel === ".sqreports" || [...excludes, ...composerExcluded.map(item => item.path)].some((p) => rel === p || rel.startsWith(`${p}/`)))) return
|
|
45
|
+
if (!reports && SKIP.has(name) && lstatSync(join(root, rel)).isDirectory()) return
|
|
46
|
+
walk(rel)
|
|
47
|
+
}
|
|
48
|
+
for (const entry of readdirSync(root).sort()) visit(entry, entry)
|
|
49
|
+
return { paths, largest: largest.sort((a, b) => b.bytes - a.bytes).slice(0, 5), composerExcluded }
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Upload current source plus bounded measurement reports, never raw Git objects.
|
|
54
|
+
* The 90-day durability calculation happens against the original local history;
|
|
55
|
+
* no shallow clone, artificial root commit or changes to the customer's checkout.
|
|
56
|
+
*/
|
|
57
|
+
export function prepareCheckoutArchive({ root, coverageRoot, durabilityReport, log = () => {}, maxBytes = ARCHIVE_LIMIT_BYTES, excludes = [] }) {
|
|
58
|
+
root = resolve(root)
|
|
59
|
+
if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0 || maxBytes > ARCHIVE_LIMIT_BYTES) throw new Error("Invalid archive size limit.")
|
|
60
|
+
excludes = normalizeExcludes(excludes)
|
|
61
|
+
const temp = mkdtempSync(join(tmpdir(), "sq-checkout-"))
|
|
62
|
+
const onExit = () => { try { rmSync(temp, { recursive: true, force: true }) } catch { /* best effort on process exit */ } }
|
|
63
|
+
process.once("exit", onExit)
|
|
64
|
+
const cleanup = () => { process.removeListener("exit", onExit); rmSync(temp, { recursive: true, force: true }) }
|
|
65
|
+
try {
|
|
66
|
+
const source = inventory(root, { excludes, log })
|
|
67
|
+
const sourceList = join(temp, "source.list")
|
|
68
|
+
writeFileSync(sourceList, source.paths.length ? `${source.paths.join("\0")}\0` : "")
|
|
69
|
+
const metadataRoot = join(temp, "metadata")
|
|
70
|
+
mkdirSync(join(metadataRoot, ".sqreports"), { recursive: true })
|
|
71
|
+
const hasGit = existsSync(join(root, ".git"))
|
|
72
|
+
const history = { mode: hasGit ? "omitted" : "unavailable", reason: durabilityReport ? "ci_report" : hasGit ? "invalid_git" : "missing_git" }
|
|
73
|
+
writeFileSync(join(metadataRoot, ".sqreports/checkout.json"), JSON.stringify({ version: 1, history, archiveLimitBytes: ARCHIVE_LIMIT_BYTES }))
|
|
74
|
+
if (source.composerExcluded.length) writeFileSync(join(metadataRoot, ".sqreports/source.json"), JSON.stringify({ version: 1, composerDependencyExclusions: source.composerExcluded }))
|
|
75
|
+
if (durabilityReport) {
|
|
76
|
+
const json = JSON.stringify(durabilityReport)
|
|
77
|
+
if (Buffer.byteLength(json) > MAX_DURABILITY_REPORT_BYTES) throw new Error("Durability report exceeds the 1 MiB report limit. No upload was sent.")
|
|
78
|
+
writeFileSync(join(metadataRoot, ".sqreports/durability.json"), json)
|
|
79
|
+
}
|
|
80
|
+
const metadataList = join(temp, "metadata.list")
|
|
81
|
+
writeFileSync(metadataList, `${inventory(metadataRoot, { reports: true }).paths.join("\0")}\0`)
|
|
82
|
+
const tarPath = join(temp, "checkout.tar")
|
|
83
|
+
const archivePath = `${tarPath}.gz`
|
|
84
|
+
// BSD tar uses only the last -T list. Build/append one directory and list
|
|
85
|
+
// per invocation, then stream compression through Node (no extra dependency).
|
|
86
|
+
const archivePart = (operation, directory, list) => {
|
|
87
|
+
const tar = spawnSync("tar", [operation, tarPath, "--no-recursion", "--null", "-C", directory, "-T", list], { encoding: "utf8", timeout: 180000, maxBuffer: 1024 * 1024, env: { ...process.env, COPYFILE_DISABLE: "1" } })
|
|
88
|
+
if (tar.error || tar.status !== 0) throw new Error("Could not archive the checkout. Ensure GNU/BSD tar is installed, files are readable, and the build has finished changing files.")
|
|
89
|
+
}
|
|
90
|
+
archivePart("-cf", root, sourceList)
|
|
91
|
+
if (coverageRoot) {
|
|
92
|
+
const coverageList = join(temp, "coverage.list")
|
|
93
|
+
const entries = inventory(coverageRoot, { reports: true }).paths
|
|
94
|
+
writeFileSync(coverageList, entries.length ? `${entries.join("\0")}\0` : "")
|
|
95
|
+
if (entries.length) archivePart("-rf", coverageRoot, coverageList)
|
|
96
|
+
}
|
|
97
|
+
archivePart("-rf", metadataRoot, metadataList)
|
|
98
|
+
const compression = spawnSync(process.execPath, ["--input-type=module", "-e", "import {createReadStream,createWriteStream} from 'node:fs';import{createGzip}from'node:zlib';import{pipeline}from'node:stream/promises';await pipeline(createReadStream(process.argv[1]),createGzip(),createWriteStream(process.argv[2]));", tarPath, archivePath], { encoding: "utf8", timeout: 180000, maxBuffer: 1024 * 1024 })
|
|
99
|
+
// A nonempty file can still be truncated. Never upload an archive produced
|
|
100
|
+
// by failed tar or a checkout that changed while it was being archived.
|
|
101
|
+
if (compression.error || compression.status !== 0 || !existsSync(archivePath) || statSync(archivePath).size === 0) {
|
|
102
|
+
throw new Error("Could not archive the checkout. Ensure GNU/BSD tar is installed, files are readable, and the build has finished changing files.")
|
|
103
|
+
}
|
|
104
|
+
const bytes = statSync(archivePath).size
|
|
105
|
+
if (bytes > maxBytes) {
|
|
106
|
+
const files = source.largest.map((f) => `${JSON.stringify(f.path)} (${mib(f.bytes)} MiB uncompressed)`).join(", ")
|
|
107
|
+
throw new Error(`source archive alone is ${mib(bytes)} MiB, above the ${mib(maxBytes)} MiB limit even without Git history. No upload was sent. Exclude generated assets with --exclude <relative-path>. Largest included files: ${files || "none"}`)
|
|
108
|
+
}
|
|
109
|
+
log(`ScaleQuality: checkout archive ${mib(bytes)} MiB / ${mib(maxBytes)} MiB; raw Git history stays on this runner.`)
|
|
110
|
+
return { archivePath, bytes, history, cleanup }
|
|
111
|
+
} catch (error) {
|
|
112
|
+
cleanup()
|
|
113
|
+
throw error
|
|
114
|
+
}
|
|
115
|
+
}
|
package/bin/check.mjs
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// ScaleQuality continuous check. Uploads THIS CI checkout to ScaleQuality (re-measures
|
|
3
|
-
// the code the runner already has,
|
|
3
|
+
// the code the runner already has, with a local 90-day durability report) and exits
|
|
4
4
|
// non-zero when maturity is below the configured cutoff, so a CI step can fail the
|
|
5
5
|
// build. Pure Node 18+ (global fetch/FormData/Blob), no npm dependencies.
|
|
6
6
|
|
|
7
7
|
import { tmpdir } from "node:os"
|
|
8
8
|
import { join, dirname, relative, sep } from "node:path"
|
|
9
9
|
import { spawnSync } from "node:child_process"
|
|
10
|
-
import { readFileSync, writeFileSync,
|
|
10
|
+
import { readFileSync, writeFileSync, statSync, rmSync, readdirSync, mkdirSync, mkdtempSync, copyFileSync } from "node:fs"
|
|
11
|
+
import { prepareCheckoutArchive, normalizeExcludes } from "./archive.mjs"
|
|
12
|
+
import { measureDurability } from "./durability.mjs"
|
|
11
13
|
|
|
12
14
|
const args = process.argv.slice(2)
|
|
13
15
|
const flag = (name) => {
|
|
@@ -30,6 +32,8 @@ Options:
|
|
|
30
32
|
myapp:\${{ github.sha }}. Needs trivy on PATH; skipped with a
|
|
31
33
|
note if it is missing, never fails the build for it.
|
|
32
34
|
--timeout <s> Max seconds to wait for the measurement (default 600)
|
|
35
|
+
--exclude <p> Exclude a generated path relative to the checkout; repeatable
|
|
36
|
+
--dry-run Measure locally and check the package; do not upload (no token needed)
|
|
33
37
|
--json Print the raw result as JSON
|
|
34
38
|
-h, --help Show this help
|
|
35
39
|
|
|
@@ -41,6 +45,7 @@ const token = flag("--token") || process.env.SQ_TOKEN
|
|
|
41
45
|
const project = flag("--project") || process.env.SQ_PROJECT
|
|
42
46
|
const host = (flag("--host") || process.env.SQ_HOST || "https://app.scalequality.io").replace(/\/+$/, "")
|
|
43
47
|
const jsonOut = has("--json")
|
|
48
|
+
const dryRun = has("--dry-run")
|
|
44
49
|
const image = flag("--image") || process.env.SQ_IMAGE
|
|
45
50
|
const timeoutMs = Math.max(30, Number(flag("--timeout") || 600)) * 1000
|
|
46
51
|
|
|
@@ -49,7 +54,7 @@ const log = (m) => { if (!jsonOut) console.log(m) }
|
|
|
49
54
|
const fail = (m) => { console.error(`ScaleQuality: ${m}`); process.exit(2) }
|
|
50
55
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
|
|
51
56
|
|
|
52
|
-
if (!token || !project) {
|
|
57
|
+
if (!dryRun && (!token || !project)) {
|
|
53
58
|
fail("--token and --project are required (or env SQ_TOKEN / SQ_PROJECT). See --help.")
|
|
54
59
|
}
|
|
55
60
|
|
|
@@ -153,13 +158,13 @@ const MAX_REPORTS = 40
|
|
|
153
158
|
|
|
154
159
|
function stageCoverageReports() {
|
|
155
160
|
const root = process.cwd()
|
|
156
|
-
const
|
|
157
|
-
|
|
161
|
+
const stagingRoot = mkdtempSync(join(tmpdir(), "sq-coverage-"))
|
|
162
|
+
const stage = join(stagingRoot, COVERAGE_STAGE)
|
|
158
163
|
// The staged copies must not outlive this process under ANY exit path. A
|
|
159
164
|
// cancelled job or a network failure that left them behind would show up as
|
|
160
165
|
// untracked files in the customer's checkout, and a pipeline that runs
|
|
161
166
|
// `git diff --exit-code` after the gate would fail because of us.
|
|
162
|
-
const clean = () => { try { rmSync(
|
|
167
|
+
const clean = () => { try { rmSync(stagingRoot, { recursive: true, force: true }) } catch { /* best effort */ } }
|
|
163
168
|
process.on("exit", clean)
|
|
164
169
|
for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"]) {
|
|
165
170
|
process.on(sig, () => { clean(); process.exit(130) })
|
|
@@ -178,6 +183,7 @@ function stageCoverageReports() {
|
|
|
178
183
|
if (!COVERAGE_SKIP.has(e.name)) walk(abs, depth + 1)
|
|
179
184
|
continue
|
|
180
185
|
}
|
|
186
|
+
if (!e.isFile()) continue
|
|
181
187
|
const hit = COVERAGE_NAMES.find((c) => c.re.test(e.name))
|
|
182
188
|
if (!hit) continue
|
|
183
189
|
let size = 0
|
|
@@ -203,42 +209,38 @@ function stageCoverageReports() {
|
|
|
203
209
|
} else {
|
|
204
210
|
log("ScaleQuality: no coverage report found in this job. Run your tests with coverage before this step to get a measured %.")
|
|
205
211
|
}
|
|
206
|
-
return { stage, count: index.length }
|
|
212
|
+
return { stage, root: stagingRoot, count: index.length, cleanup: clean }
|
|
207
213
|
}
|
|
208
214
|
|
|
209
215
|
async function main() {
|
|
210
216
|
const auth = { Authorization: `Bearer ${token}` }
|
|
217
|
+
const excludes = normalizeExcludes(args.flatMap((arg, i) => arg === "--exclude" ? [args[i + 1]] : []))
|
|
218
|
+
const durabilityReport = await measureDurability({ root: process.cwd(), log: jsonOut ? (message) => console.error(message) : log })
|
|
211
219
|
const coverage = stageCoverageReports()
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
220
|
+
let packed, buf
|
|
221
|
+
try {
|
|
222
|
+
packed = prepareCheckoutArchive({ root: process.cwd(), coverageRoot: coverage.root, durabilityReport, excludes, log })
|
|
223
|
+
if (durabilityReport) {
|
|
224
|
+
const head = spawnSync("git", ["--no-optional-locks", "-C", process.cwd(), "rev-parse", "--verify", "HEAD"], { encoding: "utf8", timeout: 10000 })
|
|
225
|
+
if (head.error || head.status !== 0 || head.stdout.trim() !== durabilityReport.headSha) throw new Error("Git HEAD changed during measurement or packaging. No upload was sent; rerun after the checkout is stable.")
|
|
226
|
+
}
|
|
227
|
+
if (!dryRun) buf = readFileSync(packed.archivePath)
|
|
228
|
+
} finally {
|
|
229
|
+
packed?.cleanup()
|
|
230
|
+
coverage.cleanup()
|
|
231
|
+
}
|
|
232
|
+
const localCheckout = { version: 1, history: packed.history, archiveBytes: packed.bytes, durability: durabilityReport ? { source: "ci", windowDays: 90, settleDays: 14, headSha: durabilityReport.headSha, measuredAt: durabilityReport.measuredAt, engineCommit: durabilityReport.engineCommit } : undefined }
|
|
233
|
+
const limitation = "ScaleQuality: historical secret scanning and commit-convention signals are not included in this checkout report."
|
|
234
|
+
console.error(limitation)
|
|
235
|
+
if (dryRun) {
|
|
236
|
+
if (jsonOut) console.log(JSON.stringify({ dryRun: true, checkout: localCheckout, coverageReports: coverage.count }))
|
|
237
|
+
else log("ScaleQuality: local package verified; no upload sent.")
|
|
238
|
+
return
|
|
231
239
|
}
|
|
232
|
-
const buf = readFileSync(archivePath)
|
|
233
|
-
rmSync(archivePath, { force: true })
|
|
234
|
-
// The staged copies exist only to ride along in the archive. Leaving them
|
|
235
|
-
// behind would dirty the working tree of the job that ran us.
|
|
236
|
-
rmSync(coverage.stage, { recursive: true, force: true })
|
|
237
|
-
const sizeMB = buf.length / (1024 * 1024)
|
|
238
|
-
if (sizeMB > 60) fail(`checkout archive too large (${sizeMB.toFixed(0)}MB > 60MB limit). Reduce clone depth or exclude build output.`)
|
|
239
240
|
|
|
240
241
|
const form = new FormData()
|
|
241
242
|
form.append("project", project)
|
|
243
|
+
form.append("reportProtocol", "durability-ci-v1")
|
|
242
244
|
form.append("file", new Blob([buf]), "checkout.tar.gz")
|
|
243
245
|
const containerReport = scanImage()
|
|
244
246
|
if (containerReport) form.append("containerReport", containerReport)
|
|
@@ -251,7 +253,7 @@ async function main() {
|
|
|
251
253
|
const body = await r.text().catch(() => "")
|
|
252
254
|
if (r.status === 401) fail("invalid token.")
|
|
253
255
|
if (r.status === 402) fail("continuous measurement needs an active subscription.")
|
|
254
|
-
if (r.status === 409) fail("continuous is not enabled for this project. Turn it on in ScaleQuality.")
|
|
256
|
+
if (r.status === 409) fail(body.includes("measurement_already_running") ? "another measurement is already running for this project. Wait for it to finish and rerun this step." : "continuous is not enabled for this project. Turn it on in ScaleQuality.")
|
|
255
257
|
fail(`report failed (${r.status}) ${body}`)
|
|
256
258
|
}
|
|
257
259
|
const { runId } = await r.json()
|
|
@@ -285,6 +287,10 @@ async function main() {
|
|
|
285
287
|
|
|
286
288
|
if (!last || (last.status !== "COMPLETED" && last.status !== "FAILED")) fail("timed out waiting for the measurement.")
|
|
287
289
|
if (last.status === "FAILED") fail(`measurement failed: ${last.failureReason || "unknown"}`)
|
|
290
|
+
if (last.checkout?.version !== 1) fail("the server did not confirm the checkout report protocol. No passing gate is claimed; update to the compatible continuous-measurement server release.")
|
|
291
|
+
if (durabilityReport && (last.checkout?.durability?.source !== "ci" || last.checkout.durability.windowDays !== 90 || last.checkout.durability.settleDays !== 14 || last.checkout.durability.headSha !== durabilityReport.headSha || last.checkout.durability.measuredAt !== durabilityReport.measuredAt || last.checkout.durability.engineCommit !== durabilityReport.engineCommit)) {
|
|
292
|
+
fail("the server did not confirm this checkout's 90-day durability report. No passing gate is claimed; the server needs the compatible continuous-measurement release.")
|
|
293
|
+
}
|
|
288
294
|
|
|
289
295
|
if (jsonOut) {
|
|
290
296
|
console.log(JSON.stringify(last))
|
|
@@ -292,6 +298,7 @@ async function main() {
|
|
|
292
298
|
const cut = last.cutoff != null ? ` (cutoff ${last.cutoff})` : " (track only)"
|
|
293
299
|
log(`ScaleQuality: maturity ${last.maturity ?? "—"}${cut}`)
|
|
294
300
|
if (last.passed === false) console.error("ScaleQuality: below the line — failing the build.")
|
|
301
|
+
else if (last.cutoff == null) log("ScaleQuality: measurement completed (track only; no cutoff configured).")
|
|
295
302
|
else log("ScaleQuality: passed.")
|
|
296
303
|
}
|
|
297
304
|
process.exit(last.passed === false ? 1 : 0)
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import { constants, closeSync, fstatSync, lstatSync, openSync, readSync, readdirSync } from 'node:fs'
|
|
2
|
+
import { join, posix } from 'node:path'
|
|
3
|
+
|
|
4
|
+
const MAX_JSON_BYTES = 4 * 1024 * 1024
|
|
5
|
+
const object = value => value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
6
|
+
const overlaps = (a, b) => a === b || a.startsWith(`${b}/`) || b.startsWith(`${a}/`) || a === '.' || b === '.'
|
|
7
|
+
|
|
8
|
+
// Inspect data only. Composer plugins/scripts and customer symlinks are never run/followed.
|
|
9
|
+
function readJson(path) {
|
|
10
|
+
let fd
|
|
11
|
+
try {
|
|
12
|
+
fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK)
|
|
13
|
+
const stat = fstatSync(fd)
|
|
14
|
+
if (!stat.isFile() || stat.size > MAX_JSON_BYTES) return null
|
|
15
|
+
const raw = Buffer.alloc(stat.size + 1)
|
|
16
|
+
let bytes = 0
|
|
17
|
+
while (bytes < raw.length) {
|
|
18
|
+
const count = readSync(fd, raw, bytes, raw.length - bytes, null)
|
|
19
|
+
if (!count) break
|
|
20
|
+
bytes += count
|
|
21
|
+
}
|
|
22
|
+
if (bytes !== stat.size) return null
|
|
23
|
+
const value = JSON.parse(raw.subarray(0, bytes).toString('utf8'))
|
|
24
|
+
return object(value) ? value : null
|
|
25
|
+
} catch { return null }
|
|
26
|
+
finally { if (fd !== undefined) closeSync(fd) }
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function safeRelative(value) {
|
|
30
|
+
if (typeof value !== 'string' || !value || /[\\\0*?\[\]$~]/.test(value) || value.startsWith('/') || /^[A-Za-z]:/.test(value)) return null
|
|
31
|
+
if (value.split('/').includes('..')) return null
|
|
32
|
+
const path = posix.normalize(value).replace(/\/$/, '')
|
|
33
|
+
return path === '.' || path.split('/').some(part => ['.git', '.sqreports'].includes(part)) ? null : path
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function sourcePath(module, value) {
|
|
37
|
+
if (typeof value !== 'string' || /[\\\0$~]/.test(value) || value.startsWith('/') || /^[A-Za-z]:/.test(value)) return null
|
|
38
|
+
// A glob's fixed directory prefix bounds every possible source it might select.
|
|
39
|
+
const glob = value.search(/[*?\[]/)
|
|
40
|
+
const prefix = glob < 0 ? value : value.slice(0, glob).replace(/[^/]*$/, '')
|
|
41
|
+
const path = posix.normalize(posix.join(module || '.', prefix || '.')).replace(/\/$/, '')
|
|
42
|
+
return path === '..' || path.startsWith('../') ? null : path
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function autoloadPaths(value) {
|
|
46
|
+
if (value === undefined) return []
|
|
47
|
+
if (!object(value)) return [null]
|
|
48
|
+
const paths = []
|
|
49
|
+
for (const key of ['psr-4', 'psr-0', 'classmap', 'files']) {
|
|
50
|
+
const entry = value[key]
|
|
51
|
+
if (entry === undefined) continue
|
|
52
|
+
const values = ['psr-4', 'psr-0'].includes(key) ? object(entry) ? Object.values(entry) : [null] : Array.isArray(entry) ? entry : [null]
|
|
53
|
+
for (const item of values) paths.push(...(Array.isArray(item) ? item : [item]))
|
|
54
|
+
}
|
|
55
|
+
return paths
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function regularDirectory(root, path, consume = () => {}) {
|
|
59
|
+
let current = root
|
|
60
|
+
for (const part of path.split('/')) {
|
|
61
|
+
consume()
|
|
62
|
+
current = join(current, part)
|
|
63
|
+
try { if (!lstatSync(current).isDirectory()) return false }
|
|
64
|
+
catch { return false }
|
|
65
|
+
}
|
|
66
|
+
return true
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** A transport exclusion, never a change to the native Git durability scope. */
|
|
70
|
+
export function composerVendorDirectory(root, module = '', excludes = [], ignoredDirectories = new Set(), consume = () => {}) {
|
|
71
|
+
const manifest = readJson(join(root, module, 'composer.json'))
|
|
72
|
+
if (!manifest) return null
|
|
73
|
+
const label = posix.join(module, 'composer.json')
|
|
74
|
+
const keep = reason => ({ kept: true, manifest: label, reason })
|
|
75
|
+
const lock = readJson(join(root, module, 'composer.lock'))
|
|
76
|
+
if (!lock || !Array.isArray(lock.packages) || (lock['packages-dev'] !== undefined && !Array.isArray(lock['packages-dev']))) return keep('a valid composer.lock is required')
|
|
77
|
+
const packages = [...lock.packages, ...(lock['packages-dev'] || [])]
|
|
78
|
+
if (!packages.length || packages.some(pkg => !object(pkg) || typeof pkg.name !== 'string' || !pkg.name || typeof pkg.version !== 'string' || !pkg.version)) return keep('locked dependency versions could not be confirmed')
|
|
79
|
+
if (manifest.config !== undefined && !object(manifest.config)) return keep('Composer config is ambiguous')
|
|
80
|
+
const vendor = safeRelative(manifest.config && Object.hasOwn(manifest.config, 'vendor-dir') ? manifest.config['vendor-dir'] : 'vendor')
|
|
81
|
+
if (!vendor) return keep('vendor-dir must be a literal directory within the Composer project')
|
|
82
|
+
const path = posix.join(module, vendor)
|
|
83
|
+
// If either evidence file is explicitly excluded, do not infer a safe replacement for installed dependencies.
|
|
84
|
+
if (['composer.json', 'composer.lock'].some(file => excludes.some(ex => overlaps(ex, posix.join(module, file))))) return keep('the manifest or lockfile was explicitly excluded')
|
|
85
|
+
|
|
86
|
+
const repositories = manifest.repositories === undefined ? [] : Array.isArray(manifest.repositories) ? manifest.repositories : object(manifest.repositories) ? Object.values(manifest.repositories) : [null]
|
|
87
|
+
if (repositories.some(repo => repo !== false && !object(repo))) return keep('Composer repositories are ambiguous')
|
|
88
|
+
const localOrigins = [
|
|
89
|
+
...repositories.filter(repo => repo && repo.type === 'path').map(repo => repo.url),
|
|
90
|
+
...packages.filter(pkg => pkg.dist?.type === 'path').map(pkg => pkg.dist.url),
|
|
91
|
+
]
|
|
92
|
+
// Resolving Composer's glob rules and each symlink target is outside this transport hotfix.
|
|
93
|
+
if (localOrigins.some(value => typeof value !== 'string' || /[*?\[]/.test(value))) return keep('local package origins use ambiguous paths or globs')
|
|
94
|
+
const sources = [...autoloadPaths(manifest.autoload), ...autoloadPaths(manifest['autoload-dev']), ...localOrigins]
|
|
95
|
+
if (sources.some(value => { consume(); const source = sourcePath(module, value); return source === null || overlaps(path, source) })) return keep('local source paths may overlap installed dependencies or lie outside the checkout')
|
|
96
|
+
if (localOrigins.some(value => {
|
|
97
|
+
const source = sourcePath(module, value)
|
|
98
|
+
consume()
|
|
99
|
+
return !regularDirectory(root, source, consume) || source.split('/').some(part => ignoredDirectories.has(part) || part === 'vendor') || excludes.some(ex => overlaps(ex, source))
|
|
100
|
+
})) return keep('a local package origin is missing, linked or excluded from the archive')
|
|
101
|
+
// A custom vendor-dir can have several components: reject a symlink in ANY component.
|
|
102
|
+
if (!regularDirectory(root, path, consume)) return null
|
|
103
|
+
return { path, manifest: label, lockfile: posix.join(module, 'composer.lock') }
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function declaredSources(root, module, consume) {
|
|
107
|
+
const manifest = readJson(join(root, module, 'composer.json'))
|
|
108
|
+
if (!manifest) return [null]
|
|
109
|
+
const repositories = manifest.repositories === undefined ? [] : Array.isArray(manifest.repositories) ? manifest.repositories : object(manifest.repositories) ? Object.values(manifest.repositories) : [null]
|
|
110
|
+
if (repositories.some(repo => repo !== false && !object(repo))) return [null]
|
|
111
|
+
const lock = readJson(join(root, module, 'composer.lock'))
|
|
112
|
+
let packages = []
|
|
113
|
+
try {
|
|
114
|
+
lstatSync(join(root, module, 'composer.lock'))
|
|
115
|
+
if (!lock || !Array.isArray(lock.packages) || (lock['packages-dev'] !== undefined && !Array.isArray(lock['packages-dev']))) return [null]
|
|
116
|
+
packages = [...lock.packages, ...(lock['packages-dev'] || [])]
|
|
117
|
+
if (packages.some(pkg => !object(pkg))) return [null]
|
|
118
|
+
} catch (error) { if (error.code !== 'ENOENT') return [null] }
|
|
119
|
+
return [
|
|
120
|
+
...autoloadPaths(manifest.autoload), ...autoloadPaths(manifest['autoload-dev']),
|
|
121
|
+
...repositories.filter(repo => repo && repo.type === 'path').map(repo => repo.url),
|
|
122
|
+
...packages.filter(pkg => pkg.dist?.type === 'path').map(pkg => pkg.dist.url),
|
|
123
|
+
].map(value => {
|
|
124
|
+
consume()
|
|
125
|
+
// Resolving a glob or symlink could reveal another project's only source copy.
|
|
126
|
+
if (typeof value !== 'string' || /[*?\[]/.test(value)) return null
|
|
127
|
+
const path = sourcePath(module, value)
|
|
128
|
+
if (path === null) return null
|
|
129
|
+
let current = root
|
|
130
|
+
for (const part of path.split('/')) {
|
|
131
|
+
consume()
|
|
132
|
+
current = join(current, part)
|
|
133
|
+
try { if (lstatSync(current).isSymbolicLink()) return null }
|
|
134
|
+
catch (error) { if (error.code === 'ENOENT') break; return null }
|
|
135
|
+
}
|
|
136
|
+
return path
|
|
137
|
+
})
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** Discover all projects before applying any Composer exclusion. References can cross modules. */
|
|
141
|
+
export function planComposerExclusions(root, excludes = [], ignoredDirectories = new Set(), log = () => {}) {
|
|
142
|
+
const limit = new Error('Composer discovery resource limit')
|
|
143
|
+
const deadline = Date.now() + 5_000
|
|
144
|
+
let work = 0
|
|
145
|
+
const consume = () => { if (++work > 1_000_000 || Date.now() > deadline) throw limit }
|
|
146
|
+
try {
|
|
147
|
+
const projects = []
|
|
148
|
+
const pending = ['']
|
|
149
|
+
let visited = 0
|
|
150
|
+
while (pending.length) {
|
|
151
|
+
consume()
|
|
152
|
+
const module = pending.pop()
|
|
153
|
+
const entries = readdirSync(join(root, module), { withFileTypes: true })
|
|
154
|
+
visited += entries.length
|
|
155
|
+
if (visited > 100_000 || projects.length > 5_000) throw limit
|
|
156
|
+
if (entries.some(entry => entry.name === 'composer.json')) projects.push({
|
|
157
|
+
module,
|
|
158
|
+
candidate: composerVendorDirectory(root, module, excludes, ignoredDirectories, consume),
|
|
159
|
+
sources: declaredSources(root, module, consume),
|
|
160
|
+
})
|
|
161
|
+
for (const entry of entries.sort((a, b) => b.name.localeCompare(a.name))) {
|
|
162
|
+
consume()
|
|
163
|
+
const path = posix.join(module, entry.name)
|
|
164
|
+
if (!entry.isDirectory() || entry.name === '.git' || path === '.sqreports' || ignoredDirectories.has(entry.name) || excludes.some(ex => path === ex || path.startsWith(`${ex}/`))) continue
|
|
165
|
+
pending.push(path)
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
const candidates = projects.flatMap(project => project.candidate?.path ? [project.candidate] : [])
|
|
169
|
+
const protectedPaths = new Set()
|
|
170
|
+
const isActive = project => !candidates.some(candidate => { consume(); return !protectedPaths.has(candidate.path) && (project.module === candidate.path || project.module.startsWith(`${candidate.path}/`)) })
|
|
171
|
+
// Dependency package manifests do not veto their containing vendor directory.
|
|
172
|
+
// Once an external project references it, it is retained and its declarations
|
|
173
|
+
// become relevant too. Iterate until no newly retained project protects another.
|
|
174
|
+
let changed
|
|
175
|
+
do {
|
|
176
|
+
changed = false
|
|
177
|
+
const sources = projects.filter(isActive).flatMap(project => project.sources)
|
|
178
|
+
for (const candidate of candidates) {
|
|
179
|
+
if (!protectedPaths.has(candidate.path) && sources.some(source => { consume(); return source === null || overlaps(candidate.path, source) })) {
|
|
180
|
+
protectedPaths.add(candidate.path)
|
|
181
|
+
changed = true
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
} while (changed)
|
|
185
|
+
for (const project of projects.filter(isActive)) {
|
|
186
|
+
if (project.candidate?.kept) log(`ScaleQuality: retaining possible Composer dependencies near ${JSON.stringify(project.candidate.manifest)}: ${project.candidate.reason}.`)
|
|
187
|
+
}
|
|
188
|
+
for (const candidate of candidates) {
|
|
189
|
+
if (protectedPaths.has(candidate.path)) log(`ScaleQuality: retaining ${JSON.stringify(candidate.path)}: a project declares source there or its source paths could not be resolved safely.`)
|
|
190
|
+
}
|
|
191
|
+
// Nested installed package manifests cannot create independent upload exclusions.
|
|
192
|
+
return projects.filter(isActive).flatMap(project => project.candidate?.path && !protectedPaths.has(project.candidate.path) ? [project.candidate] : [])
|
|
193
|
+
} catch (error) {
|
|
194
|
+
if (error !== limit) throw error
|
|
195
|
+
log('ScaleQuality: retaining Composer dependency directories: project discovery exceeded its resource limit.')
|
|
196
|
+
return []
|
|
197
|
+
}
|
|
198
|
+
}
|