@opengsd/gsd-core 1.2.0-rc.1

package/scripts/run-tests.cjs

@@ -0,0 +1,282 @@
+#!/usr/bin/env node
+// Cross-platform test runner — resolves test file globs via Node
+// instead of relying on shell expansion (which fails on Windows PowerShell/cmd).
+// Propagates NODE_V8_COVERAGE so c8 collects coverage from the child process.
+//
+// Suite filtering (issue #3597):
+//   node scripts/run-tests.cjs                 # default — runs ALL tests (backcompat)
+//   node scripts/run-tests.cjs --suite all     # explicit "everything"
+//   node scripts/run-tests.cjs --suite unit    # only files with no other suite marker
+//   node scripts/run-tests.cjs --suite security    # *.security.test.cjs
+//   node scripts/run-tests.cjs --suite integration # *.integration.test.cjs
+//   node scripts/run-tests.cjs --suite install     # *.install.test.cjs
+//   node scripts/run-tests.cjs --suite slow        # *.slow.test.cjs
+//   node scripts/run-tests.cjs --files "a.test.cjs b.test.cjs"
+//   node scripts/run-tests.cjs --files-from /tmp/selected-tests.txt
+//
+// Suite grouping convention: filename suffix marker before `.test.cjs`.
+// A file named `foo.security.test.cjs` belongs to the `security` suite.
+// A file named `foo.test.cjs` (no marker) belongs to the `unit` suite.
+// See docs/TESTING-SUITES.md for full grouping policy.
+'use strict';
+
+const { readdirSync } = require('fs');
+const { join } = require('path');
+const { execFileSync } = require('child_process');
+
+const SUITES = ['all', 'unit', 'integration', 'install', 'security', 'slow'];
+const MARKED_SUITES = ['integration', 'install', 'security', 'slow'];
+
+function parseArgs(argv) {
+  let suite = null;
+  let seen = false;
+  let files = null;
+  let filesFrom = null;
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === '--suite') {
+      if (seen) {
+        return { error: 'duplicate --suite flag' };
+      }
+      seen = true;
+      const v = argv[i + 1];
+      if (!v || v.startsWith('--')) {
+        return { error: '--suite requires a value' };
+      }
+      suite = v;
+      i++;
+    } else if (a.startsWith('--suite=')) {
+      if (seen) {
+        return { error: 'duplicate --suite flag' };
+      }
+      seen = true;
+      suite = a.slice('--suite='.length);
+      if (!suite) {
+        return { error: '--suite requires a value' };
+      }
+    } else if (a === '--files') {
+      if (files !== null) {
+        return { error: 'duplicate --files flag' };
+      }
+      const v = argv[i + 1];
+      if (!v || v.startsWith('--')) {
+        return { error: '--files requires a value' };
+      }
+      files = v;
+      i++;
+    } else if (a.startsWith('--files=')) {
+      if (files !== null) {
+        return { error: 'duplicate --files flag' };
+      }
+      files = a.slice('--files='.length);
+      if (!files) {
+        return { error: '--files requires a value' };
+      }
+    } else if (a === '--files-from') {
+      if (filesFrom !== null) {
+        return { error: 'duplicate --files-from flag' };
+      }
+      const v = argv[i + 1];
+      if (!v || v.startsWith('--')) {
+        return { error: '--files-from requires a value' };
+      }
+      filesFrom = v;
+      i++;
+    } else if (a.startsWith('--files-from=')) {
+      if (filesFrom !== null) {
+        return { error: 'duplicate --files-from flag' };
+      }
+      filesFrom = a.slice('--files-from='.length);
+      if (!filesFrom) {
+        return { error: '--files-from requires a value' };
+      }
+    } else {
+      return { error: `unknown argument: ${a}` };
+    }
+  }
+  if (files !== null && filesFrom !== null) {
+    return { error: '--files and --files-from cannot be combined' };
+  }
+  return { suite, files, filesFrom };
+}
+
+// Return the marked suite name embedded in a filename, or null if it's unmarked.
+// foo.security.test.cjs -> "security"
+// foo.test.cjs          -> null (unit)
+function suiteOf(filename) {
+  if (!filename.endsWith('.test.cjs')) return null;
+  const base = filename.slice(0, -'.test.cjs'.length);
+  const lastDot = base.lastIndexOf('.');
+  if (lastDot === -1) return null;
+  const marker = base.slice(lastDot + 1);
+  return MARKED_SUITES.includes(marker) ? marker : null;
+}
+
+function selectFiles(allFiles, suite) {
+  if (suite === null || suite === 'all') {
+    return allFiles;
+  }
+  if (suite === 'unit') {
+    return allFiles.filter(f => suiteOf(f) === null);
+  }
+  return allFiles.filter(f => suiteOf(f) === suite);
+}
+
+function splitFileList(value) {
+  if (!value) return [];
+  return value
+    .split(/[,\s]+/)
+    .map(v => v.trim())
+    .filter(Boolean)
+    .map(v => v.replace(/^tests[\\/]/, ''));
+}
+
+function selectExplicitFiles(allFiles, filesValue, filesFrom) {
+  const fs = require('fs');
+  const requested = filesFrom
+    ? splitFileList(fs.readFileSync(filesFrom, 'utf8'))
+    : splitFileList(filesValue);
+  const available = new Set(allFiles);
+  const selected = [];
+  const missing = [];
+  for (const file of requested) {
+    if (available.has(file)) {
+      selected.push(file);
+    } else {
+      missing.push(file);
+    }
+  }
+  if (missing.length > 0) {
+    return {
+      error: `requested test file(s) not found: ${missing.join(', ')}`,
+    };
+  }
+  return { files: [...new Set(selected)] };
+}
+
+function main() {
+  const args = process.argv.slice(2);
+  const parsed = parseArgs(args);
+  if (parsed.error) {
+    console.error(`run-tests: ${parsed.error}`);
+    console.error(`Valid suites: ${SUITES.join(', ')}`);
+    process.exit(2);
+  }
+  const suite = parsed.suite;
+  if (suite !== null && !SUITES.includes(suite)) {
+    console.error(`run-tests: unknown suite "${suite}"`);
+    console.error(`Valid suites: ${SUITES.join(', ')}`);
+    process.exit(2);
+  }
+
+  const testDir = process.env.GSD_TEST_DIR
+    ? process.env.GSD_TEST_DIR
+    : join(__dirname, '..', 'tests');
+
+  const allFiles = readdirSync(testDir)
+    .filter(f => f.endsWith('.test.cjs'))
+    .sort();
+
+  if (allFiles.length === 0) {
+    console.error(`No test files found in ${testDir}`);
+    process.exit(1);
+  }
+
+  let selectedNames;
+  if (parsed.files !== null || parsed.filesFrom !== null) {
+    const explicit = selectExplicitFiles(allFiles, parsed.files, parsed.filesFrom);
+    if (explicit.error) {
+      console.error(`run-tests: ${explicit.error}`);
+      process.exit(2);
+    }
+    selectedNames = explicit.files;
+  } else {
+    selectedNames = selectFiles(allFiles, suite);
+  }
+  const selected = selectedNames.map(f => join(testDir, f));
+
+  if (selected.length === 0) {
+    // Empty suite: report and exit 0 so empty lanes (e.g. `security` before
+    // adversarial tests land) don't gate CI. CI consumers wanting strictness
+    // can grep stderr for "no tests in suite".
+    console.error(`run-tests: no tests in suite "${suite || 'all'}"`);
+    process.exit(0);
+  }
+
+  // Log selected files to stderr for CI / harness-test visibility.
+  // node:test default reporter doesn't echo filenames, so this gives
+  // operators a single stable line they can grep.
+  console.error(
+    `run-tests: suite="${suite || 'all'}" files=${selected.length}: ${selected
+      .map(f => f.split(/[\\/]/).pop())
+      .join(' ')}`,
+  );
+
+  // Default concurrency: 4 on Linux/macOS, 2 on Windows.
+  //
+  // Windows has significantly higher per-subprocess overhead than Linux/macOS:
+  //   - Windows Defender scans each spawned process
+  //   - NTFS has higher file-system latency under concurrent access
+  //   - synckit worker_threads (used by the SDK bridge in gsd-tools.cjs) spawn
+  //     native threads that contend on SharedArrayBuffer + Atomics.wait; under
+  //     Node 24 on Windows, 4-way concurrent gsd-tools invocations (each spawning
+  //     a synckit worker) caused intermittent process crashes with empty stderr —
+  //     a signature of OS-level resource exhaustion killing worker threads before
+  //     they could flush. Reducing to 2 halves the peak concurrent worker count.
+  //
+  // Operator override via TEST_CONCURRENCY env var for local debugging.
+  const defaultConcurrency = process.platform === 'win32' ? 2 : 4;
+  const concurrency = process.env.TEST_CONCURRENCY
+    ? `--test-concurrency=${process.env.TEST_CONCURRENCY}`
+    : `--test-concurrency=${defaultConcurrency}`;
+
+  // Windows `CreateProcess` caps the full command line at 32,767 chars
+  // (lpCommandLine). With 500+ test paths the spawn fails instantly with no
+  // test output. Linux/macOS allow ~2 MB (ARG_MAX) so unchunked spawns are
+  // fine there. Split into chunks sized for the tightest target so behavior
+  // is identical across platforms. (#3597)
+  // Operator override (also used by tests to force chunking with short paths).
+  const MAX_CMDLINE_CHARS = process.env.RUN_TESTS_MAX_CMDLINE_CHARS
+    ? Number(process.env.RUN_TESTS_MAX_CMDLINE_CHARS)
+    : 28000; // headroom below the 32,767 Windows ceiling
+  const FIXED_OVERHEAD = process.execPath.length + '--test'.length + concurrency.length + 8;
+  const chunks = [];
+  let current = [];
+  let currentLen = FIXED_OVERHEAD;
+  for (const file of selected) {
+    const add = file.length + 1; // +1 for the inter-arg separator
+    if (current.length > 0 && currentLen + add > MAX_CMDLINE_CHARS) {
+      chunks.push(current);
+      current = [];
+      currentLen = FIXED_OVERHEAD;
+    }
+    current.push(file);
+    currentLen += add;
+  }
+  if (current.length > 0) chunks.push(current);
+
+  let firstFailureExit = 0;
+  for (let i = 0; i < chunks.length; i++) {
+    if (chunks.length > 1) {
+      console.error(`run-tests: chunk ${i + 1}/${chunks.length} — ${chunks[i].length} files`);
+    }
+    try {
+      execFileSync(process.execPath, ['--test', concurrency, ...chunks[i]], {
+        stdio: 'inherit',
+        env: { ...process.env },
+      });
+    } catch (err) {
+      const code = err.status || 1;
+      // Run every chunk so the operator sees all failures in one pass; report
+      // the first non-zero exit at the end.
+      if (firstFailureExit === 0) firstFailureExit = code;
+    }
+  }
+  if (firstFailureExit !== 0) process.exit(firstFailureExit);
+}
+
+if (require.main === module) {
+  main();
+}
+
+module.exports = { suiteOf };

package/scripts/secret-scan-lint.sh

@@ -0,0 +1,231 @@
+#!/usr/bin/env bash
+# secret-scan-lint.sh — Lint governance policy for .secretscanignore exclusions
+#
+# Usage:
+#   scripts/secret-scan-lint.sh --file <path-to-.secretscanignore>
+#   scripts/secret-scan-lint.sh --file <path-to-.secretscanignore> --strict
+#
+# Exit codes:
+#   0 = every exclusion has full annotation OR is grandfathered (with deprecation warning)
+#   1 = annotation violation: missing required key, expired date, or unguarded wildcard
+#       without rule-id; OR (under --strict) any grandfathered entry is present
+#   2 = usage/config error (file not found, invalid arguments)
+#
+# Annotation syntax (sidecar comment, must immediately precede the path line):
+#   # allow: <pattern>  reason="..."  owner="..."  expires="YYYY-MM-DD"  [rule-id="..."]
+#   <pattern>
+#
+# Required annotation keys: reason, owner, expires
+# Optional annotation key:  rule-id (REQUIRED when pattern contains '*' wildcards)
+#
+# Grandfathered entries: paths with any preceding plain comment (not a structured
+# annotation) are treated as grandfathered in default mode — exit 0 with a
+# deprecation warning to stderr. Under --strict, grandfathered entries cause exit 1.
+#
+# Design references:
+#   - GitGuardian exclusion annotation convention:
+#     https://docs.gitguardian.com/internal-repositories-monitoring/integrations/cli/secrets
+#   - CNCF Security TAG threat-model exception lifecycle:
+#     https://github.com/cncf/tag-security/blob/main/community/working-groups/threat-modeling/templates/threats.md
+#
+# Exit-code alignment with secret-scan.sh:
+#   Both scripts use 0=clean, 1=policy-violation/findings, 2=usage/config-error.
+#   The symmetry is intentional — CI can treat either non-zero as a gate failure.
+
+set -euo pipefail
+
+# ─── Argument Parsing ─────────────────────────────────────────────────────────
+
+STRICT=false
+IGNOREFILE=""
+
+usage() {
+  echo "Usage: $0 --file <path-to-.secretscanignore> [--strict]" >&2
+  exit 2
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --file)
+      shift
+      [[ $# -eq 0 ]] && usage
+      IGNOREFILE="$1"
+      shift
+      ;;
+    --strict)
+      STRICT=true
+      shift
+      ;;
+    -h|--help)
+      usage
+      ;;
+    *)
+      echo "Error: unknown argument: $1" >&2
+      usage
+      ;;
+  esac
+done
+
+if [[ -z "$IGNOREFILE" ]]; then
+  usage
+fi
+
+if [[ ! -f "$IGNOREFILE" ]]; then
+  echo "Error: file not found: $IGNOREFILE" >&2
+  exit 2
+fi
+
+# ─── Date Helpers ─────────────────────────────────────────────────────────────
+
+# Returns today's date as YYYY-MM-DD (portable across macOS and Linux)
+today_date() {
+  date +%Y-%m-%d
+}
+
+# Returns true (0) if date1 < date2 (both YYYY-MM-DD strings)
+date_is_past() {
+  local check_date="$1"
+  local today
+  today=$(today_date)
+  # Lexicographic comparison works for ISO-8601 dates
+  [[ "$check_date" < "$today" ]]
+}
+
+# ─── Annotation Parser ────────────────────────────────────────────────────────
+
+# Returns value of a key="value" or key='value' pair from a string.
+# Usage: extract_key <string> <key>
+extract_key() {
+  local str="$1"
+  local key="$2"
+  # Match key="value" or key='value'
+  local val
+  val=$(echo "$str" | grep -oE "${key}=['\"][^'\"]+['\"]" | head -1 | sed "s/${key}=['\"]//;s/['\"]$//") || true
+  echo "$val"
+}
+
+# Returns true (0) if the string contains a wildcard glob character (* or **)
+contains_wildcard() {
+  local pattern="$1"
+  [[ "$pattern" == *"*"* ]]
+}
+
+# Returns true (0) if a comment line is a structured annotation
+# (must start with "# allow:" prefix)
+is_structured_annotation() {
+  local comment="$1"
+  [[ "$comment" =~ ^#[[:space:]]+allow:[[:space:]] ]]
+}
+
+# ─── Main Lint Logic ──────────────────────────────────────────────────────────
+
+VIOLATIONS=0
+WARNINGS=0
+
+# We process the file line-by-line, tracking the comment immediately preceding
+# each path entry. If the preceding line was a comment, we inspect it.
+
+prev_comment=""
+lineno=0
+
+while IFS= read -r line || [[ -n "$line" ]]; do
+  lineno=$((lineno + 1))
+
+  # Skip empty lines (reset prev_comment to avoid false association)
+  if [[ -z "${line// }" ]]; then
+    prev_comment=""
+    continue
+  fi
+
+  # Accumulate comment lines
+  if [[ "$line" =~ ^[[:space:]]*# ]]; then
+    prev_comment="$line"
+    continue
+  fi
+
+  # This is a path/pattern entry.
+  local_path="$line"
+
+  # ── Case 1: No preceding comment at all ────────────────────────────────────
+  if [[ -z "$prev_comment" ]]; then
+    echo "VIOLATION (line $lineno): '$local_path' has no annotation comment." >&2
+    echo "  Required: # allow: <pattern>  reason=\"...\"  owner=\"...\"  expires=\"YYYY-MM-DD\"" >&2
+    VIOLATIONS=$((VIOLATIONS + 1))
+    prev_comment=""
+    continue
+  fi
+
+  # ── Case 2: Preceding comment exists — check if it's a structured annotation ─
+  if is_structured_annotation "$prev_comment"; then
+    # Extract required keys
+    reason=$(extract_key "$prev_comment" "reason")
+    owner=$(extract_key "$prev_comment" "owner")
+    expires=$(extract_key "$prev_comment" "expires")
+    rule_id=$(extract_key "$prev_comment" "rule-id")
+
+    local_ok=true
+
+    if [[ -z "$reason" ]]; then
+      echo "VIOLATION (line $lineno): '$local_path' annotation missing required key: reason" >&2
+      local_ok=false
+    fi
+
+    if [[ -z "$owner" ]]; then
+      echo "VIOLATION (line $lineno): '$local_path' annotation missing required key: owner" >&2
+      local_ok=false
+    fi
+
+    if [[ -z "$expires" ]]; then
+      echo "VIOLATION (line $lineno): '$local_path' annotation missing required key: expires" >&2
+      local_ok=false
+    elif date_is_past "$expires"; then
+      echo "VIOLATION (line $lineno): '$local_path' annotation 'expires' date is in the past: $expires" >&2
+      echo "  Review this exclusion and update or remove it." >&2
+      local_ok=false
+    fi
+
+    if contains_wildcard "$local_path" && [[ -z "$rule_id" ]]; then
+      echo "VIOLATION (line $lineno): '$local_path' uses a wildcard but is missing required key: rule-id" >&2
+      echo "  Wildcard exclusions (**  *.ext) require an explicit rule-id for auditability." >&2
+      local_ok=false
+    fi
+
+    if [[ "$local_ok" == false ]]; then
+      VIOLATIONS=$((VIOLATIONS + 1))
+    fi
+
+  else
+    # ── Case 3: Preceding comment is plain (not structured) — grandfathered ──
+    if [[ "$STRICT" == true ]]; then
+      echo "VIOLATION (line $lineno): '$local_path' is grandfathered (no structured annotation) — rejected under --strict mode." >&2
+      echo "  Add: # allow: $local_path  reason=\"...\"  owner=\"...\"  expires=\"YYYY-MM-DD\"" >&2
+      VIOLATIONS=$((VIOLATIONS + 1))
+    else
+      echo "WARNING (line $lineno): '$local_path' is grandfathered (missing structured annotation)." >&2
+      echo "  DEPRECATION: migrate to structured annotation before removing grandfather status." >&2
+      echo "  Required: # allow: $local_path  reason=\"...\"  owner=\"...\"  expires=\"YYYY-MM-DD\"" >&2
+      echo "  See: https://docs.gitguardian.com/internal-repositories-monitoring/integrations/cli/secrets" >&2
+      WARNINGS=$((WARNINGS + 1))
+    fi
+  fi
+
+  prev_comment=""
+
+done < "$IGNOREFILE"
+
+# ─── Summary ──────────────────────────────────────────────────────────────────
+
+if [[ $VIOLATIONS -gt 0 ]]; then
+  echo "secret-scan-lint: $VIOLATIONS violation(s) found" >&2
+  if [[ $STRICT == true ]]; then
+    echo "secret-scan-lint: --strict mode active — grandfathered entries are not permitted" >&2
+  fi
+  exit 1
+fi
+
+if [[ $WARNINGS -gt 0 ]]; then
+  echo "secret-scan-lint: $WARNINGS grandfathered entry/entries (deprecation warning)" >&2
+fi
+
+echo "secret-scan-lint: OK"
+exit 0