@scalequality/check 0.5.2 → 0.5.3

package/README.md

@@ -95,7 +95,7 @@ is rejected locally before upload; it is not treated as a full 90-day measuremen
 - uses: actions/checkout@v4
   with:
     fetch-depth: 0
-- run: npx @scalequality/check@0.5.2 --project <projectId>
+- run: npx @scalequality/check@0.5.3 --project <projectId>
   env:
     SQ_TOKEN: ${{ secrets.SQ_TOKEN }}
 ```
@@ -126,7 +126,7 @@ that you expect to measure.
 
 ```bash
 # Local verification; does not need a token and sends nothing to ScaleQuality.
-npx @scalequality/check@0.5.2 --dry-run
+npx @scalequality/check@0.5.3 --dry-run
 ```
 
 The CLI checks compressed size before allocating the upload buffer or sending a
@@ -191,3 +191,13 @@ A valid report with zero settled or AI-attributed lines still proceeds. It does
 not invent 100% AI survival. Missing coverage alone does not block submission.
 A completed check below a configured maturity cutoff still exits with code 1;
 track-only completion still exits with code 0.
+
+## 0.5.3: complete coverage report collection
+
+Coverage discovery and staging now process all matching reports incrementally,
+including deep modules and individual reports larger than 25 MiB. Report bytes
+and paths are preserved; the CLI no longer silently stops after 40 reports.
+A file that changes or cannot be copied stops submission before any upload.
+The existing 60 MiB compressed upload limit, Git durability calculation and
+maturity gates are unchanged. This release requires the matching server reader
+that accepts the complete staged report index.

package/bin/check.mjs

@@ -4,12 +4,11 @@
 // non-zero when maturity is below the configured cutoff, so a CI step can fail the
 // build. Pure Node 18+ (global fetch/FormData/Blob), no npm dependencies.
 
-import { tmpdir } from "node:os"
-import { join, dirname, relative, sep } from "node:path"
 import { spawnSync } from "node:child_process"
-import { readFileSync, writeFileSync, statSync, rmSync, readdirSync, mkdirSync, mkdtempSync, copyFileSync } from "node:fs"
+import { readFileSync } from "node:fs"
 import { prepareCheckoutArchive, normalizeExcludes } from "./archive.mjs"
 import { measureDurability } from "./durability.mjs"
+import { stageCoverageReports } from "./coverage.mjs"
 
 const args = process.argv.slice(2)
 const flag = (name) => {
@@ -128,96 +127,12 @@ function slimReport(stdout, ref) {
   return json
 }
 
-// Coverage reports live in exactly the directories the archive excludes: JaCoCo
-// writes to target/ and build/, Istanbul to coverage/. So the one moment where a
-// REAL coverage number is sitting on disk — right after this job ran the tests —
-// was also the one moment we threw it away, and every Maven project came back
-// with no measured coverage even with a green suite behind it.
-//
-// The reports are copied into .sqreports/ under FLAT, indexed names, with an
-// index.json holding each one's original path. Mirroring the original directory
-// tree looked tidier and did not work: bsdtar does not anchor `--exclude=./target`
-// to the root, so a staged copy at `.scalequality/coverage/target/...` was dropped
-// on macOS while surviving under GNU tar. Measured, not assumed — and a layout
-// whose behaviour depends on which tar the runner ships is not a layout.
-// Every staged name here is `<n>-<basename>`, which no exclude pattern can match.
-const COVERAGE_NAMES = [
-  { re: /^jacoco(TestReport)?\.xml$/i, format: "jacoco" },
-  { re: /^lcov\.info$/i, format: "lcov" },
-  { re: /^coverage-summary\.json$/i, format: "istanbul" },
-  { re: /^coverage\.cobertura\.xml$/i, format: "cobertura" },
-  { re: /^cobertura(-coverage)?\.xml$/i, format: "cobertura" },
-  { re: /^coverage\.xml$/i, format: "cobertura" },
-  { re: /^clover\.xml$/i, format: "clover" },
-  { re: /^(coverage|cover)\.out$/i, format: "go" },
-]
-const COVERAGE_SKIP = new Set(["node_modules", ".git", ".venv", "venv", "vendor", "__pycache__", ".sqreports"])
-const COVERAGE_STAGE = ".sqreports"
-const MAX_REPORT_BYTES = 25 * 1024 * 1024
-const MAX_REPORTS = 40
-
-function stageCoverageReports() {
-  const root = process.cwd()
-  const stagingRoot = mkdtempSync(join(tmpdir(), "sq-coverage-"))
-  const stage = join(stagingRoot, COVERAGE_STAGE)
-  // The staged copies must not outlive this process under ANY exit path. A
-  // cancelled job or a network failure that left them behind would show up as
-  // untracked files in the customer's checkout, and a pipeline that runs
-  // `git diff --exit-code` after the gate would fail because of us.
-  const clean = () => { try { rmSync(stagingRoot, { recursive: true, force: true }) } catch { /* best effort */ } }
-  process.on("exit", clean)
-  for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"]) {
-    process.on(sig, () => { clean(); process.exit(130) })
-  }
-
-  const index = []
-  let bytes = 0
-  const walk = (dir, depth) => {
-    if (depth > 7 || index.length >= MAX_REPORTS || bytes >= MAX_REPORT_BYTES) return
-    let entries
-    try { entries = readdirSync(dir, { withFileTypes: true }) } catch { return }
-    for (const e of entries) {
-      if (index.length >= MAX_REPORTS || bytes >= MAX_REPORT_BYTES) return
-      const abs = join(dir, e.name)
-      if (e.isDirectory()) {
-        if (!COVERAGE_SKIP.has(e.name)) walk(abs, depth + 1)
-        continue
-      }
-      if (!e.isFile()) continue
-      const hit = COVERAGE_NAMES.find((c) => c.re.test(e.name))
-      if (!hit) continue
-      let size = 0
-      try { size = statSync(abs).size } catch { continue }
-      if (size === 0 || bytes + size > MAX_REPORT_BYTES) continue
-      const staged = `${index.length}-${e.name}`
-      try {
-        mkdirSync(stage, { recursive: true })
-        copyFileSync(abs, join(stage, staged))
-      } catch { continue }
-      bytes += size
-      index.push({ file: staged, path: relative(root, abs).split(sep).join("/"), format: hit.format })
-    }
-  }
-  walk(root, 0)
-
-  if (index.length) {
-    try {
-      writeFileSync(join(stage, "index.json"), JSON.stringify({ version: 1, reports: index }))
-    } catch { /* the archive just goes without them */ }
-    const shown = index.slice(0, 3).map((r) => r.path).join(", ")
-    log(`ScaleQuality: found ${index.length} coverage report(s) — ${shown}${index.length > 3 ? " …" : ""}`)
-  } else {
-    log("ScaleQuality: no coverage report found in this job. Run your tests with coverage before this step to get a measured %.")
-  }
-  return { stage, root: stagingRoot, count: index.length, cleanup: clean }
-}
-
 async function main() {
   const auth = { Authorization: `Bearer ${token}` }
   const excludes = normalizeExcludes(args.flatMap((arg, i) => arg === "--exclude" ? [args[i + 1]] : []))
   const durabilityReport = await measureDurability({ root: process.cwd(), log: jsonOut ? (message) => console.error(message) : log })
   if (!durabilityReport) fail("Git durability collection failed. No upload was sent and no new score was submitted. Resolve the Git error above before rerunning this step.")
-  const coverage = stageCoverageReports()
+  const coverage = await stageCoverageReports({ log })
   let packed, buf
   try {
     packed = prepareCheckoutArchive({ root: process.cwd(), coverageRoot: coverage.root, durabilityReport, excludes, log })

package/bin/coverage.mjs

@@ -0,0 +1,117 @@
+import { constants, rmSync } from "node:fs"
+import { open, opendir, mkdir, mkdtemp } from "node:fs/promises"
+import { tmpdir } from "node:os"
+import { join, relative, sep } from "node:path"
+
+const NAMES = [
+  { re: /^jacoco(TestReport)?\.xml$/i, format: "jacoco" },
+  { re: /^lcov\.info$/i, format: "lcov" },
+  { re: /^coverage-summary\.json$/i, format: "istanbul" },
+  { re: /^coverage\.cobertura\.xml$/i, format: "cobertura" },
+  { re: /^cobertura(-coverage)?\.xml$/i, format: "cobertura" },
+  { re: /^coverage\.xml$/i, format: "cobertura" },
+  { re: /^clover\.xml$/i, format: "clover" },
+  { re: /^(coverage|cover)\.out$/i, format: "go" },
+]
+const SKIP = new Set(["node_modules", ".git", ".venv", "venv", "vendor", "__pycache__", ".sqreports"])
+const STAGE = ".sqreports"
+
+// Copy sequentially using a fixed buffer. Report count and uncompressed bytes
+// do not determine which sources survive. The final compressed checkout still
+// has the existing upload limit, checked by archive.mjs before any HTTP request.
+async function copyReport(source, destination) {
+  let input, output
+  try {
+    input = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK)
+    const before = await input.stat({ bigint: true })
+    if (!before.isFile() || before.size === 0n) throw new Error("report is empty or is not a regular file")
+    output = await open(destination, "wx")
+    const buffer = Buffer.allocUnsafe(64 * 1024)
+    let copied = 0n
+    while (true) {
+      const { bytesRead } = await input.read(buffer, 0, buffer.length, null)
+      if (!bytesRead) break
+      let written = 0
+      while (written < bytesRead) {
+        const { bytesWritten } = await output.write(buffer, written, bytesRead - written, null)
+        if (!bytesWritten) throw new Error("report copy made no progress")
+        written += bytesWritten
+      }
+      copied += BigInt(bytesRead)
+      if (copied > before.size) throw new Error("report changed during collection")
+    }
+    const after = await input.stat({ bigint: true })
+    if (copied !== before.size || after.size !== before.size || after.mtimeNs !== before.mtimeNs || after.ctimeNs !== before.ctimeNs) {
+      throw new Error("report changed during collection")
+    }
+  } finally {
+    await Promise.all([output?.close(), input?.close()])
+  }
+}
+
+/** Preserve every discovered original report in a flat v1 index. Flat names
+ * keep GNU/BSD tar exclusions from dropping nested build/target reports.
+ * Generated
+ * directories are scanned too: they are exactly where test runners write.
+ * No customer files are created or changed. A failed scan/copy/index never
+ * returns a partial set that could be presented as a complete measurement. */
+export async function stageCoverageReports({ root = process.cwd(), log = () => {} } = {}) {
+  const stagingRoot = await mkdtemp(join(tmpdir(), "sq-coverage-"))
+  const stage = join(stagingRoot, STAGE)
+  const remove = () => { try { rmSync(stagingRoot, { recursive: true, force: true }) } catch { /* best effort on process exit */ } }
+  const signals = ["SIGINT", "SIGTERM", "SIGHUP"]
+  const onSignal = () => { remove(); process.exit(130) }
+  process.once("exit", remove)
+  for (const signal of signals) process.once(signal, onSignal)
+  const cleanup = () => {
+    process.removeListener("exit", remove)
+    for (const signal of signals) process.removeListener(signal, onSignal)
+    remove()
+  }
+  let index
+  let count = 0
+  const shown = []
+  let current = "."
+  const walk = async (directory) => {
+    current = relative(root, directory).split(sep).join("/") || "."
+    const entries = await opendir(directory)
+    for await (const entry of entries) {
+      const absolute = join(directory, entry.name)
+      if (entry.isDirectory()) {
+        if (!SKIP.has(entry.name)) await walk(absolute)
+        continue
+      }
+      if (!entry.isFile()) continue // Never follow report or directory symlinks.
+      const match = NAMES.find(candidate => candidate.re.test(entry.name))
+      if (!match) continue
+      current = relative(root, absolute).split(sep).join("/")
+      if (!index) {
+        await mkdir(stage)
+        index = await open(join(stage, "index.json"), "wx")
+        await index.writeFile('{"version":1,"reports":[')
+      }
+      const file = `${count}-${entry.name}`
+      await copyReport(absolute, join(stage, file))
+      await index.writeFile((count ? "," : "") + JSON.stringify({ file, path: current, format: match.format }))
+      count++
+      if (shown.length < 3) shown.push(current)
+    }
+  }
+  try {
+    await walk(root)
+    if (index) {
+      await index.writeFile("]}\n")
+      await index.close()
+      index = undefined
+      log(`ScaleQuality: found ${count} coverage report(s) — ${shown.join(", ")}${count > 3 ? " …" : ""}`)
+    } else {
+      log("ScaleQuality: no coverage report found in this job. Run your tests with coverage before this step to get a measured %.")
+    }
+    return { stage, root: stagingRoot, count, cleanup }
+  } catch (error) {
+    await index?.close().catch(() => {})
+    cleanup()
+    const reason = typeof error?.code === "string" ? error.code : error?.message || "I/O error"
+    throw new Error(`Coverage collection could not finish at ${JSON.stringify(current)} (${reason}). No upload was sent and no partial coverage was submitted.`)
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scalequality/check",
-  "version": "0.5.2",
+  "version": "0.5.3",
   "description": "ScaleQuality continuous code-maturity check for CI. Re-measures your repo on every build and fails the pipeline below your maturity line.",
   "type": "module",
   "bin": {