fixyoursecret 0.3.1-developer-preview.1 → 0.4.0

package/CHANGELOG.md

@@ -2,6 +2,26 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.4.0] - 2026-03-26
+
+### Added
+- Verification v2 provider-safe checks for:
+  - OpenAI
+  - GitHub
+  - Slack
+  - Stripe
+- 500-repo regression quality gate script:
+  - `scripts/check-tuning-regression.js`
+  - `fixtures/tuning/regression-thresholds.json`
+- Weekly CI now runs:
+  - `tune:500:quick`
+  - regression gate fail-on-quality-drop
+
+### Improved
+- Stronger suppression for obvious fake/placeholder secrets in tests/docs.
+- Reduced large-corpus noise while preserving high-risk detections.
+- Stable release channel metadata for npm publish.
+
 ## [0.3.1-developer-preview.1] - 2026-03-26
 
 ### Added

package/README.md

@@ -2,8 +2,6 @@
 
 # FixYourSecret
 
-**Developer Preview**
-
 An ESLint-style CLI that finds leaked credentials, flags frontend exposure, suggests fixes, and helps rotate keys safely.
 
 [![Node >= 20](https://img.shields.io/badge/node-%3E%3D20-2ea44f)](https://nodejs.org/)
@@ -25,7 +23,8 @@ FixYourSecret helps teams catch these mistakes early and fix them with clear nex
 - Expanded provider detector coverage significantly.
 - Added benchmark-driven quality gates (`npm run benchmark`) so quality is measured every release.
 - Added CI threshold enforcement for recall/precision.
-- Added optional local verification mode for higher-confidence findings.
+- Added Verification v2 provider-safe checks for higher-confidence findings.
+- Added weekly 500-repo regression quality gate.
 
 ---
 
@@ -112,6 +111,32 @@ Run multi-repo tuning report:
 npm run tune:multi
 ```
 
+Run large-scale corpus tuning (parallel clone + scan):
+
+```bash
+npm run tune:large
+```
+
+Generate and run 500-repo corpus:
+
+```bash
+npm run corpus:generate
+npm run tune:500
+```
+
+Quick large-scale pass:
+
+```bash
+npm run tune:large:quick
+```
+
+Weekly regression check sequence (same as CI):
+
+```bash
+npm run tune:500:quick
+npm run regression:check
+```
+
 CI quality gate thresholds (defaults):
 - Recall >= 0.95
 - Precision >= 0.95
@@ -122,6 +147,7 @@ These can be tuned via env vars:
 
 Tuning workflow docs:
 - [./docs/tuning/process.md](./docs/tuning/process.md)
+- Large corpus list: [./fixtures/tuning/repos.large.json](./fixtures/tuning/repos.large.json)
 
 ---
 
@@ -159,7 +185,7 @@ fixyoursecret scan [options]
 ---
 
 ## Verification Mode (Optional)
-`--verify safe` performs local structure checks for supported detectors (no external API calls).
+`--verify safe` performs provider-safe local structure checks for supported detectors (no external API calls), including tighter checks for OpenAI, GitHub, Slack, and Stripe.
 
 Use `--verify-strict` to drop findings that fail verification.
 
@@ -168,7 +194,7 @@ Use `--verify-strict` to drop findings that fail verification.
 ## Config (`.fixyoursecretrc.json`)
 ```json
 {
-  "ignorePaths": ["node_modules/**", ".git/**", "dist/**", "build/**", ".next/**", "coverage/**"],
+  "ignorePaths": ["node_modules/**", ".git/**", ".cache/**", "dist/**", "build/**", ".next/**", "coverage/**", "vendor/**", "tmp/**"],
   "allowedExtensions": [".js", ".ts", ".jsx", ".tsx", ".env", ".swift"],
   "maxFileSizeKB": 256,
   "entropyThreshold": 3.8,

package/detectors/generic.js

@@ -1,4 +1,4 @@
-const TOKEN_REGEX = /[A-Za-z0-9_\-]{24,}/g;
+const TOKEN_REGEX = /[A-Za-z0-9_\-+=]{28,}/g;
 
 export function detectGenericSecrets(content, options = {}) {
   const threshold = Number.isFinite(options.entropyThreshold) ? options.entropyThreshold : 3.8;
@@ -7,6 +7,8 @@ export function detectGenericSecrets(content, options = {}) {
     const value = match[0];
     if (!looksLikeHighEntropy(value, threshold)) continue;
     if (looksSafeCommonWord(value)) continue;
+    if (looksLikeCodeIdentifier(value)) continue;
+    if (!looksLikeSecretToken(value)) continue;
 
     findings.push({
       rule: "generic-high-entropy",
@@ -23,12 +25,14 @@ export function detectGenericSecrets(content, options = {}) {
 function looksLikeHighEntropy(value, threshold) {
   const entropy = shannonEntropy(value);
   const hasLower = /[a-z]/.test(value);
-  const hasUpperOrSymbol = /[A-Z]/.test(value) || /[_-]/.test(value);
+  const hasUpperOrSymbol = /[A-Z]/.test(value) || /[_\-+/=]/.test(value);
   const hasDigit = /\d/.test(value);
-  return entropy >= threshold && hasLower && hasUpperOrSymbol && hasDigit;
+  const adjustedThreshold = value.length >= 40 ? threshold + 0.25 : threshold;
+  return entropy >= adjustedThreshold && hasLower && hasUpperOrSymbol && hasDigit;
 }
 
 function looksSafeCommonWord(value) {
+  const lower = value.toLowerCase();
   return (
     value.startsWith("sk-") ||
     value.startsWith("AIza") ||
@@ -36,11 +40,59 @@ function looksSafeCommonWord(value) {
     value.startsWith("pk_live_") ||
     value.startsWith("http") ||
     value.includes("localhost") ||
-    value.toLowerCase().includes("component") ||
-    value.toLowerCase().includes("configuration")
+    lower.includes("component") ||
+    lower.includes("configuration") ||
+    lower.includes("diagnostics") ||
+    lower.includes("typescript")
   );
 }
 
+function looksLikeSecretToken(value) {
+  if (/[/.:]/.test(value)) return false;
+  if (value.includes("://")) return false;
+  if (value.startsWith("www")) return false;
+  if (/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value)) return false;
+
+  const classes = [
+    /[a-z]/.test(value),
+    /[A-Z]/.test(value),
+    /\d/.test(value),
+    /[_\-+/=]/.test(value),
+  ].filter(Boolean).length;
+
+  const digits = (value.match(/\d/g) || []).length;
+  const hasLongHexOnly = /^[a-f0-9]{24,}$/i.test(value);
+  const hasOnlyAlphaNum = /^[A-Za-z0-9]+$/.test(value);
+  const symbolCount = (value.match(/[_\-+=]/g) || []).length;
+
+  if (hasLongHexOnly && digits < 6) return false;
+  if (classes < 3) return false;
+  if (value.length >= 32 && digits < 2) return false;
+  if (hasOnlyAlphaNum && value.length < 36) return false;
+  if (value.length >= 36 && symbolCount === 0 && digits < 4) return false;
+  return true;
+}
+
+function looksLikeCodeIdentifier(value) {
+  if (/^[A-Za-z_][A-Za-z0-9_]{30,}$/.test(value)) return true;
+  if (/^[A-Z][A-Za-z0-9]{20,}$/.test(value)) return true;
+  if (/^[_A-Za-z0-9-]+$/.test(value) && value.includes("__")) return true;
+  if (/^[A-Za-z]+(?:[A-Z][a-z0-9]+){2,}\d+$/.test(value)) return true;
+  if (/(Api|Context|Request|Response|Migration|Oauth2|OAuth2|V1alpha1|Agentflow)/.test(value)) return true;
+
+  const parts = value.split(/[_-]/).filter(Boolean);
+  if (parts.length >= 4) {
+    const alphaWords = parts.filter((p) => /^[A-Za-z]{3,}$/.test(p)).length;
+    if (alphaWords / parts.length >= 0.6) return true;
+  }
+
+  const vowelCount = (value.match(/[aeiou]/gi) || []).length;
+  const vowelRatio = vowelCount / value.length;
+  if (vowelRatio > 0.42 && !/[_\-+/=]/.test(value)) return true;
+
+  return false;
+}
+
 function shannonEntropy(value) {
   const map = new Map();
   for (const ch of value) {

package/detectors/openai.js

@@ -1,13 +1,17 @@
-const OPENAI_REGEX = /sk-(?:proj-)?[A-Za-z0-9_-]{20,}/g;
+const OPENAI_REGEX = /(^|[^A-Za-z0-9_])(sk-(?:proj-)?[A-Za-z0-9_-]{24,})(?![A-Za-z0-9_-])/g;
 
 export function detectOpenAI(content) {
   const findings = [];
   for (const match of content.matchAll(OPENAI_REGEX)) {
+    const value = match[2];
+    if (!value) continue;
+    if (!/\d/.test(value)) continue;
+    if (value.includes("--")) continue;
     findings.push({
       rule: "openai-key",
       issue: "OpenAI key exposed",
-      index: match.index ?? 0,
-      value: match[0],
+      index: (match.index ?? 0) + (match[1]?.length || 0),
+      value,
       type: "openai",
       confidence: "high",
     });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "fixyoursecret",
-  "version": "0.3.1-developer-preview.1",
-  "description": "Developer Preview: CLI tool to detect leaked secrets, frontend exposure, and generate safe fixes.",
+  "version": "0.4.0",
+  "description": "CLI tool to detect leaked secrets, frontend exposure, and generate safe fixes.",
   "type": "module",
   "bin": {
     "fixyoursecret": "bin/index.js",
@@ -30,10 +30,16 @@
     "history": "node ./bin/index.js history 20",
     "ci": "node ./bin/index.js ci",
     "benchmark": "node ./scripts/benchmark.js",
+    "regression:check": "node ./scripts/check-tuning-regression.js --report ./docs/tuning/report-500.json --thresholds ./fixtures/tuning/regression-thresholds.json",
+    "corpus:generate": "node ./scripts/generate-corpus.js --output ./fixtures/tuning/repos.500.json --count 500",
     "quality": "npm test && npm run benchmark",
     "test": "node --test",
     "tune:multi": "node ./scripts/multi-repo-tune.js --repos-file ./fixtures/tuning/repos.default.json --output ./docs/tuning/multi-repo-report.json --fp-review ./docs/tuning/false-positive-review.md",
-    "tune:weekly": "npm run benchmark && npm run tune:multi"
+    "tune:large": "node ./scripts/multi-repo-tune.js --repos-file ./fixtures/tuning/repos.large.json --output ./docs/tuning/large-report.json --fp-review ./docs/tuning/false-positive-review-large.md --verify safe",
+    "tune:large:quick": "node ./scripts/multi-repo-tune.js --repos-file ./fixtures/tuning/repos.large.json --output ./docs/tuning/large-report.json --fp-review ./docs/tuning/false-positive-review-large.md --verify safe --max-repos 25",
+    "tune:500": "node ./scripts/multi-repo-tune.js --repos-file ./fixtures/tuning/repos.500.json --output ./docs/tuning/report-500.json --fp-review ./docs/tuning/false-positive-review-500.md --verify safe --concurrency 12 --max-repos 500",
+    "tune:500:quick": "node ./scripts/multi-repo-tune.js --repos-file ./fixtures/tuning/repos.500.json --output ./docs/tuning/report-500.json --fp-review ./docs/tuning/false-positive-review-500.md --verify safe --concurrency 8 --max-repos 100",
+    "tune:weekly": "npm run benchmark && npm run tune:500:quick && npm run regression:check"
   },
   "keywords": [
     "security",
@@ -53,8 +59,7 @@
   },
   "license": "MIT",
   "publishConfig": {
-    "access": "public",
-    "tag": "preview"
+    "access": "public"
   },
   "dependencies": {
     "chalk": "^5.4.1",

package/utils/config.js

@@ -5,7 +5,7 @@ export const CONFIG_FILENAMES = [".fixyoursecretrc.json", ".secretlintrc.json"];
 export const BASELINE_FILENAMES = [".fixyoursecret-baseline.json", ".secretlint-baseline.json"];
 
 export const DEFAULT_CONFIG = {
-  ignorePaths: ["node_modules/**", ".git/**", "dist/**", "build/**", ".next/**", "coverage/**"],
+  ignorePaths: ["node_modules/**", ".git/**", ".cache/**", "dist/**", "build/**", ".next/**", "coverage/**", "vendor/**", "tmp/**"],
   allowedExtensions: [".js", ".ts", ".jsx", ".tsx", ".env", ".swift"],
   maxFileSizeKB: 256,
   entropyThreshold: 3.8,

package/utils/verifier.js

@@ -1,35 +1,52 @@
 export function verifyFinding(match, fileContent = "", snippet = "") {
+  const value = String(match.value || "");
+  const lowerSnippet = String(snippet || "").toLowerCase();
+
   switch (match.rule) {
-    case "openai-key":
-      return formatResult(/^(?:sk-(?:proj-)?)?[A-Za-z0-9_-]{24,}$/.test(match.value) && /\d/.test(match.value), "format");
+    case "openai-key": {
+      const formatOk = /^sk-(?:proj-)?[A-Za-z0-9_-]{24,}$/.test(value);
+      const hasDiversity = hasDiversityScore(value, 3);
+      const verified = formatOk && hasDiversity && !isLikelyPlaceholderToken(value, lowerSnippet);
+      return formatResult(verified, "provider-safe-v2");
+    }
     case "google-key":
-      return formatResult(/^AIza[0-9A-Za-z_-]{35}$/.test(match.value), "format");
+      return formatResult(/^AIza[0-9A-Za-z_-]{35}$/.test(value), "provider-safe-v2");
     case "aws-access-key-id":
-      return formatResult(/^(AKIA|ASIA)[A-Z0-9]{16}$/.test(match.value), "format");
-    case "stripe-secret-key":
-      return formatResult(/^sk_live_[0-9A-Za-z]{16,}$/.test(match.value), "format");
-    case "slack-token":
-      return formatResult(/^xox(?:b|p|a|r|s)-[0-9A-Za-z-]{10,}$/.test(match.value), "format");
-    case "github-token":
-      return formatResult(/^(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{20,}$|^github_pat_[A-Za-z0-9_]{20,}$/.test(match.value), "format");
+      return formatResult(/^(AKIA|ASIA)[A-Z0-9]{16}$/.test(value), "provider-safe-v2");
+    case "stripe-secret-key": {
+      const formatOk = /^sk_live_[0-9A-Za-z]{20,}$/.test(value);
+      const verified = formatOk && !isLikelyPlaceholderToken(value, lowerSnippet);
+      return formatResult(verified, "provider-safe-v2");
+    }
+    case "slack-token": {
+      const formatOk = /^xox(?:b|p|a|r|s)-[0-9A-Za-z-]{10,}$/.test(value);
+      const hasSegments = value.split("-").length >= 3;
+      const verified = formatOk && hasSegments && !isLikelyPlaceholderToken(value, lowerSnippet);
+      return formatResult(verified, "provider-safe-v2");
+    }
+    case "github-token": {
+      const formatOk = /^(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{30,}$|^github_pat_[A-Za-z0-9_]{40,}$/.test(value);
+      const verified = formatOk && !isLikelyPlaceholderToken(value, lowerSnippet);
+      return formatResult(verified, "provider-safe-v2");
+    }
     case "gitlab-token":
-      return formatResult(/^glpat-[A-Za-z0-9_-]{20,}$/.test(match.value), "format");
+      return formatResult(/^glpat-[A-Za-z0-9_-]{20,}$/.test(value), "provider-safe-v2");
     case "twilio-api-key":
-      return formatResult(/^SK[0-9a-fA-F]{32}$/.test(match.value), "format");
+      return formatResult(/^SK[0-9a-fA-F]{32}$/.test(value), "provider-safe-v2");
     case "sendgrid-api-key":
-      return formatResult(/^SG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}$/.test(match.value), "format");
+      return formatResult(/^SG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}$/.test(value), "provider-safe-v2");
     case "mailgun-api-key":
-      return formatResult(/^key-[A-Za-z0-9]{32}$/.test(match.value), "format");
+      return formatResult(/^key-[A-Za-z0-9]{32}$/.test(value), "provider-safe-v2");
     case "anthropic-api-key":
-      return formatResult(/^sk-ant-[A-Za-z0-9_-]{20,}$/.test(match.value), "format");
+      return formatResult(/^sk-ant-[A-Za-z0-9_-]{20,}$/.test(value), "provider-safe-v2");
     case "cohere-api-key":
-      return formatResult(/^co_[A-Za-z0-9]{30,}$/.test(match.value), "format");
+      return formatResult(/^co_[A-Za-z0-9]{30,}$/.test(value), "provider-safe-v2");
     case "huggingface-token":
-      return formatResult(/^hf_[A-Za-z0-9]{30,}$/.test(match.value), "format");
+      return formatResult(/^hf_[A-Za-z0-9]{30,}$/.test(value), "provider-safe-v2");
     case "telegram-bot-token":
-      return formatResult(/^\d{8,10}:[A-Za-z0-9_-]{35}$/.test(match.value), "format");
+      return formatResult(/^\d{8,10}:[A-Za-z0-9_-]{35}$/.test(value), "provider-safe-v2");
     case "npm-token":
-      return formatResult(/^npm_[A-Za-z0-9]{36}$/.test(match.value), "format");
+      return formatResult(/^npm_[A-Za-z0-9]{36}$/.test(value), "provider-safe-v2");
     case "private-key-block": {
       const hasEnd = /-----END (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/.test(fileContent.slice(match.index));
       return formatResult(hasEnd, "pem-structure");
@@ -56,18 +73,91 @@ export function normalizeVerifyMode(mode) {
 export function shouldSkipAsNonSecret(match, snippet = "", filePath = "", hints = []) {
   const lowerSnippet = snippet.toLowerCase();
   const lowerPath = filePath.toLowerCase();
+  const value = String(match.value || "");
+  const isNonProdPath = ["/test/", "/tests/", "/__tests__/", "/fixtures/", "/docs/", "/examples/", "/spec/"]
+    .some((segment) => lowerPath.includes(segment));
 
   const builtinHints = ["example", "dummy", "fake", "sample", "not_secret", "replace_in_runtime_only", "docs_only"];
   const allHints = [...builtinHints, ...hints.map((h) => String(h).toLowerCase())];
 
   if (allHints.some((hint) => lowerSnippet.includes(hint))) return true;
+  if (isNonProdPath && isLikelyPlaceholderToken(value, lowerSnippet)) return true;
+
+  if (
+    isNonProdPath &&
+    /(?:ghp_|github_pat_|xox[bpars]-|sk_live_|sk-)/.test(value) &&
+    /(?:x{6,}|y{6,}|z{6,}|123456|example|dummy|placeholder|mock|test[_-]?key|your[_-]?key)/i.test(lowerSnippet + " " + value.toLowerCase())
+  ) {
+    return true;
+  }
 
   if (
     match.rule === "generic-high-entropy" &&
-    ["/test/", "/tests/", "/__tests__/", "/fixtures/", "/docs/"].some((segment) => lowerPath.includes(segment))
+    [
+      "/test/",
+      "/tests/",
+      "/__tests__/",
+      "/fixtures/",
+      "/docs/",
+      "/spec/",
+      "/bench/",
+      "/benchmark/",
+      "/examples/",
+      "/migrations/",
+      "/generated/",
+      "/api-client/",
+      "/fonts/",
+      "/vendor/"
+    ].some((segment) => lowerPath.includes(segment))
   ) {
     return true;
   }
 
+  if (match.rule === "generic-high-entropy") {
+    const genericNoiseHints = [
+      "canvasrenderingcontext2d",
+      "axios parameter creator",
+      "sourcemappingurl=data:",
+      "base64,",
+      "images.unsplash.com",
+      ".woff2",
+      "oauth2",
+      "requestparameters",
+      "data-cy=",
+      "uuid",
+      "v1alpha1",
+      "openapi",
+      "migration",
+      "model:",
+      "anthropiccontext1m",
+      "bigint64arraybytes_per_element",
+      "claude-sonnet",
+      "gemini-"
+    ];
+    if (genericNoiseHints.some((hint) => lowerSnippet.includes(hint))) return true;
+  }
+
   return false;
 }
+
+function isLikelyPlaceholderToken(value, lowerSnippet = "") {
+  const lowerValue = String(value || "").toLowerCase();
+  const joined = `${lowerValue} ${lowerSnippet}`;
+  if (!lowerValue) return false;
+
+  if (/(?:x{8,}|0{8,}|12345678)/.test(lowerValue)) return true;
+  if (/(?:example|dummy|placeholder|mock|redacted|sanitized|masked)/.test(joined)) return true;
+  if (/(?:ghp_x+|xox[bpars]-x+|sk_live_x+|sk-[a-z]*x{8,})/.test(lowerValue)) return true;
+  if (/(?:your[_-]?token|your[_-]?key|replace[_-]?me|insert[_-]?key)/.test(joined)) return true;
+  return false;
+}
+
+function hasDiversityScore(value, minClasses) {
+  const classes = [
+    /[a-z]/.test(value),
+    /[A-Z]/.test(value),
+    /\d/.test(value),
+    /[_-]/.test(value),
+  ].filter(Boolean).length;
+  return classes >= minClasses;
+}