@vellumai/assistant 0.7.0 → 0.7.1

package/src/__tests__/secret-scanner.test.ts

@@ -1,17 +1,10 @@
 import { describe, expect, test } from "bun:test";
 
 import {
-  _hasSecretContext,
   _isPlaceholder,
-  _redact,
-  _tryDecodeBase64,
-  _tryDecodeContinuousHex,
-  _tryDecodeHexEscapes,
-  _tryDecodePercentEncoded,
   redactSecrets,
   scanText,
   type SecretMatch,
-  shannonEntropy,
 } from "../security/secret-scanner.js";
 
 // ---------------------------------------------------------------------------
@@ -412,20 +405,6 @@ describe("placeholder detection", () => {
 // Redaction
 // ---------------------------------------------------------------------------
 describe("redaction", () => {
-  test("redact masks the middle of a string", () => {
-    const result = _redact("AKIAIOSFODNN7REALKEY");
-    // Should show first few + last few with stars in between
-    expect(result).toContain("*");
-    expect(result.length).toBeLessThanOrEqual(30);
-    // First chars visible
-    expect(result.startsWith("AKIA")).toBe(false); // only ~15% visible
-    expect(result.startsWith("AK")).toBe(true);
-  });
-
-  test("redact handles short strings", () => {
-    expect(_redact("short")).toBe("***");
-  });
-
   test("redactSecrets replaces secrets in text", () => {
     const input = `export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7REALKEY`;
     const result = redactSecrets(input);
@@ -546,154 +525,9 @@ describe("false positive resistance", () => {
 });
 
 // ---------------------------------------------------------------------------
-// Shannon entropy
-// ---------------------------------------------------------------------------
-describe("shannonEntropy", () => {
-  test("returns 0 for empty string", () => {
-    expect(shannonEntropy("")).toBe(0);
-  });
-
-  test("returns 0 for single repeated character", () => {
-    expect(shannonEntropy("aaaaaa")).toBe(0);
-  });
-
-  test("returns 1.0 for two equally distributed characters", () => {
-    expect(shannonEntropy("ababab")).toBeCloseTo(1.0, 5);
-  });
-
-  test("returns high entropy for random-looking strings", () => {
-    // A high-entropy hex string
-    const entropy = shannonEntropy("a3f8c1b2d9e4f5a6b7c8d9e0f1a2b3c4");
-    expect(entropy).toBeGreaterThan(3.0);
-  });
-
-  test("returns lower entropy for structured/repetitive content", () => {
-    const entropy = shannonEntropy("abcabcabcabcabcabc");
-    expect(entropy).toBeLessThan(2.0);
-  });
-});
-
-// ---------------------------------------------------------------------------
-// Entropy-based secret detection
+// Pass-through cases (allowlist-shaped strings that look secret-ish)
 // ---------------------------------------------------------------------------
-describe("entropy-based detection", () => {
-  test("detects high-entropy hex token near secret keyword", () => {
-    const hexSecret = "a3f8c1b2d9e4f5a6b7c8d9e0f1a2b3c4d5e6f7a8";
-    // Use context that triggers entropy detection but not generic assignment pattern
-    const input = `The signing_key is ${hexSecret}`;
-    const matches = scanText(input);
-    const entropyMatch = matches.find(
-      (m) => m.type === "High-Entropy Hex Token",
-    );
-    expect(entropyMatch).toBeDefined();
-    expect(entropyMatch!.startIndex).toBe(input.indexOf(hexSecret));
-  });
-
-  test("detects high-entropy base64 token near secret keyword", () => {
-    const b64Secret = "aB3cD4eF5gH6iJ7kL8mN+pQ/rS0tU1vW2xY3zA=";
-    // Use "is" instead of ": " to avoid triggering the generic assignment pattern
-    const input = `The token is ${b64Secret}`;
-    const matches = scanText(input);
-    const entropyMatch = matches.find(
-      (m) => m.type === "High-Entropy Base64 Token",
-    );
-    expect(entropyMatch).toBeDefined();
-  });
-
-  test("does not flag high-entropy hex without secret context", () => {
-    const hexString = "a3f8c1b2d9e4f5a6b7c8d9e0f1a2b3c4d5e6f7a8";
-    // No secret keyword nearby — just "checksum"
-    const input = `checksum: ${hexString}`;
-    const matches = scanText(input);
-    const entropyMatches = matches.filter((m) =>
-      m.type.startsWith("High-Entropy"),
-    );
-    expect(entropyMatches).toHaveLength(0);
-  });
-
-  test("does not flag low-entropy tokens even with context", () => {
-    // Repeated pattern = low entropy
-    const lowEntropy = "abcabcabcabcabcabcabcabc";
-    const input = `secret = ${lowEntropy}`;
-    const matches = scanText(input);
-    const entropyMatches = matches.filter((m) =>
-      m.type.startsWith("High-Entropy"),
-    );
-    expect(entropyMatches).toHaveLength(0);
-  });
-
-  test("does not flag tokens shorter than minLength", () => {
-    const shortToken = "a3f8c1b2d9e4f5a6";
-    const input = `api_key = ${shortToken}`;
-    const matches = scanText(input);
-    const entropyMatches = matches.filter((m) =>
-      m.type.startsWith("High-Entropy"),
-    );
-    expect(entropyMatches).toHaveLength(0);
-  });
-
-  test("can be disabled via config", () => {
-    const hexSecret = "a3f8c1b2d9e4f5a6b7c8d9e0f1a2b3c4d5e6f7a8";
-    const input = `api_key = ${hexSecret}`;
-    const matches = scanText(input, { enabled: false });
-    const entropyMatches = matches.filter((m) =>
-      m.type.startsWith("High-Entropy"),
-    );
-    expect(entropyMatches).toHaveLength(0);
-  });
-
-  test("respects custom threshold", () => {
-    const hexSecret = "a3f8c1b2d9e4f5a6b7c8d9e0f1a2b3c4d5e6f7a8";
-    const input = `The signing_key is ${hexSecret}`;
-
-    // With extremely high threshold, nothing matches
-    const matchesHigh = scanText(input, { hexThreshold: 10.0 });
-    const entropyHigh = matchesHigh.filter((m) =>
-      m.type.startsWith("High-Entropy"),
-    );
-    expect(entropyHigh).toHaveLength(0);
-
-    // With low threshold, it matches
-    const matchesLow = scanText(input, { hexThreshold: 1.0 });
-    const entropyLow = matchesLow.filter((m) =>
-      m.type.startsWith("High-Entropy"),
-    );
-    expect(entropyLow.length).toBeGreaterThan(0);
-  });
-
-  test("does not double-count tokens already matched by patterns", () => {
-    // An AWS access key should only appear once (pattern match), not again as entropy
-    const input = `api_key = AKIAIOSFODNN7REALKEY`;
-    const matches = scanText(input);
-    const awsMatches = matches.filter((m) => m.type === "AWS Access Key");
-    const entropyMatches = matches.filter((m) =>
-      m.type.startsWith("High-Entropy"),
-    );
-    expect(awsMatches).toHaveLength(1);
-    // Should not have an entropy match for the same range
-    expect(entropyMatches).toHaveLength(0);
-  });
-
-  test("recognizes various secret context keywords", () => {
-    const hexSecret = "a3f8c1b2d9e4f5a6b7c8d9e0f1a2b3c4d5e6f7a8";
-    // Use "is" instead of "=" to avoid matching the generic assignment pattern
-    const keywords = [
-      "bearer",
-      "credential",
-      "private_key",
-      "signing_key",
-      "encryption_key",
-    ];
-    for (const kw of keywords) {
-      const input = `The ${kw} is ${hexSecret}`;
-      const matches = scanText(input);
-      const entropyMatches = matches.filter((m) =>
-        m.type.startsWith("High-Entropy"),
-      );
-      expect(entropyMatches.length).toBeGreaterThanOrEqual(1);
-    }
-  });
-
+describe("pass-through behavior", () => {
   test("does not redact Telegram invite deep links", () => {
     const invite =
       "https://t.me/credence_the_bot?start=iv_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789-_ABCDE";
@@ -704,26 +538,6 @@ describe("entropy-based detection", () => {
   });
 });
 
-// ---------------------------------------------------------------------------
-// hasSecretContext helper
-// ---------------------------------------------------------------------------
-describe("hasSecretContext", () => {
-  test("detects keyword in prefix", () => {
-    const text = "api_key = abc123";
-    expect(_hasSecretContext(text, 10)).toBe(true);
-  });
-
-  test("returns false without keyword", () => {
-    const text = "username = abc123";
-    expect(_hasSecretContext(text, 11)).toBe(false);
-  });
-
-  test("detects keyword case-insensitively", () => {
-    const text = "API_KEY = abc123";
-    expect(_hasSecretContext(text, 10)).toBe(true);
-  });
-});
-
 // ---------------------------------------------------------------------------
 // Overlapping match handling in redactSecrets (#166 feedback)
 // ---------------------------------------------------------------------------
@@ -855,360 +669,3 @@ describe("unquoted generic secret assignments", () => {
     expect(generic).toHaveLength(0);
   });
 });
-
-// ---------------------------------------------------------------------------
-// Base64 padding in entropy detection (#169 feedback)
-// ---------------------------------------------------------------------------
-describe("base64 padding handling", () => {
-  test("includes trailing = padding in the match", () => {
-    const b64 = "aB3cD4eF5gH6iJ7kL8mN9pQrS0tU1vW2xY3zA=";
-    const input = `token: ${b64}`;
-    const matches = scanText(input);
-    const entropyMatch = matches.find(
-      (m) => m.type === "High-Entropy Base64 Token",
-    );
-    expect(entropyMatch).toBeDefined();
-    // endIndex should include the '='
-    expect(
-      input.slice(entropyMatch!.startIndex, entropyMatch!.endIndex),
-    ).toContain("=");
-  });
-
-  test("includes trailing == padding in the match", () => {
-    const b64 = "aB3cD4eF5gH6iJ7kL8mN9pQrS0tU1vW2x==";
-    const input = `token: ${b64}`;
-    const matches = scanText(input);
-    const entropyMatch = matches.find(
-      (m) => m.type === "High-Entropy Base64 Token",
-    );
-    expect(entropyMatch).toBeDefined();
-    expect(
-      input.slice(entropyMatch!.startIndex, entropyMatch!.endIndex),
-    ).toContain("==");
-  });
-
-  test("redactSecrets fully redacts padded base64 tokens", () => {
-    const b64 = "aB3cD4eF5gH6iJ7kL8mN9pQrS0tU1vW2xY3zA=";
-    const input = `token: "${b64}"`;
-    const result = redactSecrets(input);
-    // No trailing '=' should leak after redaction
-    expect(result).not.toMatch(/<redacted type="[^"]+" \/>=/);
-    expect(result).toContain('<redacted type="');
-  });
-});
-
-// ---------------------------------------------------------------------------
-// Word-boundary context keyword matching (#169 feedback)
-// ---------------------------------------------------------------------------
-describe("word-boundary context keywords", () => {
-  test('does not match "key" inside "monkey"', () => {
-    const hexSecret = "a3f8c1b2d9e4f5a6b7c8d9e0f1a2b3c4d5e6f7a8";
-    const input = `monkey: ${hexSecret}`;
-    const matches = scanText(input);
-    const entropy = matches.filter((m) => m.type.startsWith("High-Entropy"));
-    expect(entropy).toHaveLength(0);
-  });
-
-  test('does not match "key" inside "keyboard"', () => {
-    const hexSecret = "a3f8c1b2d9e4f5a6b7c8d9e0f1a2b3c4d5e6f7a8";
-    const input = `keyboard = ${hexSecret}`;
-    const matches = scanText(input);
-    const entropy = matches.filter((m) => m.type.startsWith("High-Entropy"));
-    expect(entropy).toHaveLength(0);
-  });
-
-  test('does not match "token" inside "tokenizer"', () => {
-    const hexSecret = "a3f8c1b2d9e4f5a6b7c8d9e0f1a2b3c4d5e6f7a8";
-    const input = `tokenizer: ${hexSecret}`;
-    const matches = scanText(input);
-    const entropy = matches.filter((m) => m.type.startsWith("High-Entropy"));
-    expect(entropy).toHaveLength(0);
-  });
-
-  test('still matches "key" as a standalone word', () => {
-    const hexSecret = "a3f8c1b2d9e4f5a6b7c8d9e0f1a2b3c4d5e6f7a8";
-    const input = `The key is ${hexSecret}`;
-    const matches = scanText(input);
-    const entropy = matches.filter((m) => m.type.startsWith("High-Entropy"));
-    expect(entropy.length).toBeGreaterThan(0);
-  });
-
-  test('still matches "api_key" with underscores', () => {
-    const hexSecret = "a3f8c1b2d9e4f5a6b7c8d9e0f1a2b3c4d5e6f7a8";
-    const input = `The api_key is ${hexSecret}`;
-    const matches = scanText(input);
-    const entropy = matches.filter((m) => m.type.startsWith("High-Entropy"));
-    expect(entropy.length).toBeGreaterThan(0);
-  });
-
-  test('still matches "api-key" with hyphens', () => {
-    const hexSecret = "a3f8c1b2d9e4f5a6b7c8d9e0f1a2b3c4d5e6f7a8";
-    const input = `The api-key is ${hexSecret}`;
-    const matches = scanText(input);
-    const entropy = matches.filter((m) => m.type.startsWith("High-Entropy"));
-    expect(entropy.length).toBeGreaterThan(0);
-  });
-});
-
-// ---------------------------------------------------------------------------
-// Encoded secret detection — decode + re-scan
-// ---------------------------------------------------------------------------
-describe("encoded secret detection", () => {
-  // -- Base64-encoded secrets --
-  describe("base64-encoded", () => {
-    test("detects base64-encoded Stripe key", () => {
-      const secret = "sk_live_abcdefghijklmnopqrstuvwx";
-      const encoded = Buffer.from(secret).toString("base64");
-      const input = `config: ${encoded}`;
-      const matches = scanText(input);
-      const found = matches.find(
-        (m) => m.type === "Stripe Secret Key (base64-encoded)",
-      );
-      expect(found).toBeDefined();
-    });
-
-    test("detects base64-encoded GitHub token", () => {
-      const secret = `ghp_${"A".repeat(36)}`;
-      const encoded = Buffer.from(secret).toString("base64");
-      const input = `value=${encoded}`;
-      const matches = scanText(input);
-      const found = matches.find(
-        (m) => m.type === "GitHub Token (base64-encoded)",
-      );
-      expect(found).toBeDefined();
-    });
-
-    test("detects base64-encoded private key header", () => {
-      const secret = "-----BEGIN RSA PRIVATE KEY-----";
-      const encoded = Buffer.from(secret).toString("base64");
-      const input = `data: ${encoded}`;
-      const matches = scanText(input);
-      const found = matches.find(
-        (m) => m.type === "Private Key (base64-encoded)",
-      );
-      expect(found).toBeDefined();
-    });
-
-    test("does not flag base64 that decodes to non-secret text", () => {
-      const encoded = Buffer.from("Hello, this is just normal text!").toString(
-        "base64",
-      );
-      const input = `data: ${encoded}`;
-      const matches = scanText(input);
-      const encoded_matches = matches.filter((m) =>
-        m.type.includes("base64-encoded"),
-      );
-      expect(encoded_matches).toHaveLength(0);
-    });
-
-    test("does not flag base64 that decodes to binary data", () => {
-      // Create a base64 string that decodes to non-printable bytes
-      const binary = Buffer.from([
-        0x00, 0x01, 0x02, 0x80, 0xff, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
-        0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15,
-      ]);
-      const encoded = binary.toString("base64");
-      const input = `data: ${encoded}`;
-      const matches = scanText(input);
-      const encoded_matches = matches.filter((m) =>
-        m.type.includes("base64-encoded"),
-      );
-      expect(encoded_matches).toHaveLength(0);
-    });
-
-    test("does not double-count secrets already detected by raw patterns", () => {
-      // A JWT is already detected directly — should not be re-detected as base64-encoded
-      const header = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9";
-      const payload =
-        "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ";
-      const signature = "SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c";
-      const jwt = `${header}.${payload}.${signature}`;
-      const input = `token: ${jwt}`;
-      const matches = scanText(input);
-      const jwtMatches = matches.filter((m) => m.type === "JSON Web Token");
-      const encodedMatches = matches.filter((m) =>
-        m.type.includes("base64-encoded"),
-      );
-      expect(jwtMatches).toHaveLength(1);
-      expect(encodedMatches).toHaveLength(0);
-    });
-  });
-
-  // -- Percent-encoded secrets --
-  describe("percent-encoded", () => {
-    test("detects percent-encoded database connection string", () => {
-      const secret = "postgres://user:secret@db.example.com:5432/mydb";
-      const encoded = encodeURIComponent(secret);
-      const input = `url=${encoded}`;
-      const matches = scanText(input);
-      const found = matches.find(
-        (m) => m.type === "Database Connection String (percent-encoded)",
-      );
-      expect(found).toBeDefined();
-    });
-
-    test("detects percent-encoded secret assignment", () => {
-      const encoded = "password%3D%22SuperSecret123%21%22";
-      const input = `data=${encoded}`;
-      const matches = scanText(input);
-      const found = matches.find((m) => m.type.includes("percent-encoded"));
-      expect(found).toBeDefined();
-    });
-
-    test("does not flag percent-encoded non-secret text", () => {
-      const encoded = "hello%20world%20this%20is%20normal";
-      const input = `text=${encoded}`;
-      const matches = scanText(input);
-      const encoded_matches = matches.filter((m) =>
-        m.type.includes("percent-encoded"),
-      );
-      expect(encoded_matches).toHaveLength(0);
-    });
-  });
-
-  // -- Hex-escaped secrets --
-  describe("hex-escaped", () => {
-    test("detects hex-escaped Stripe key", () => {
-      const secret = "sk_live_abcdefghijklmnopqrstuvwx";
-      const escaped = Array.from(secret)
-        .map((c) => `\\x${c.charCodeAt(0).toString(16).padStart(2, "0")}`)
-        .join("");
-      const input = `value = "${escaped}"`;
-      const matches = scanText(input);
-      const found = matches.find(
-        (m) => m.type === "Stripe Secret Key (hex-escaped)",
-      );
-      expect(found).toBeDefined();
-    });
-
-    test("does not flag hex-escaped non-secret text", () => {
-      const escaped = "\\x48\\x65\\x6c\\x6c\\x6f";
-      const input = `value = "${escaped}"`;
-      const matches = scanText(input);
-      const encoded_matches = matches.filter((m) =>
-        m.type.includes("hex-escaped"),
-      );
-      expect(encoded_matches).toHaveLength(0);
-    });
-  });
-
-  // -- Continuous hex-encoded secrets --
-  describe("hex-encoded (continuous)", () => {
-    test("detects hex-encoded GitHub token", () => {
-      const secret = `ghp_${"A".repeat(36)}`;
-      const hexEncoded = Buffer.from(secret).toString("hex");
-      const input = `payload: ${hexEncoded}`;
-      const matches = scanText(input);
-      const found = matches.find(
-        (m) => m.type === "GitHub Token (hex-encoded)",
-      );
-      expect(found).toBeDefined();
-    });
-
-    test("detects hex-encoded AWS access key", () => {
-      const secret = "AKIAIOSFODNN7REALKEY";
-      const hexEncoded = Buffer.from(secret).toString("hex");
-      const input = `data: ${hexEncoded}`;
-      const matches = scanText(input);
-      const found = matches.find(
-        (m) => m.type === "AWS Access Key (hex-encoded)",
-      );
-      expect(found).toBeDefined();
-    });
-
-    test("does not flag hex strings that decode to non-secret text", () => {
-      const hexEncoded = Buffer.from(
-        "This is just normal harmless text",
-      ).toString("hex");
-      const input = `data: ${hexEncoded}`;
-      const matches = scanText(input);
-      const encoded_matches = matches.filter((m) =>
-        m.type.includes("hex-encoded"),
-      );
-      expect(encoded_matches).toHaveLength(0);
-    });
-
-    test("does not flag git SHAs or similar hex hashes", () => {
-      // 40-char hex SHA — too short to decode to a meaningful secret
-      const sha = "4b825dc642cb6eb9a060e54bf899d15f13fe1d7a";
-      const input = `commit: ${sha}`;
-      const matches = scanText(input);
-      const encoded_matches = matches.filter((m) =>
-        m.type.includes("hex-encoded"),
-      );
-      expect(encoded_matches).toHaveLength(0);
-    });
-  });
-
-  // -- Redaction of encoded secrets --
-  describe("redaction of encoded secrets", () => {
-    test("redactSecrets replaces base64-encoded secrets", () => {
-      const secret = "sk_live_abcdefghijklmnopqrstuvwx";
-      const encoded = Buffer.from(secret).toString("base64");
-      const input = `config: ${encoded}`;
-      const result = redactSecrets(input);
-      expect(result).toContain(
-        '<redacted type="Stripe Secret Key (base64-encoded)" />',
-      );
-      expect(result).not.toContain(encoded);
-    });
-
-    test("redactSecrets replaces hex-encoded secrets", () => {
-      const secret = `ghp_${"A".repeat(36)}`;
-      const hexEncoded = Buffer.from(secret).toString("hex");
-      const input = `data: ${hexEncoded}`;
-      const result = redactSecrets(input);
-      expect(result).toContain(
-        '<redacted type="GitHub Token (hex-encoded)" />',
-      );
-      expect(result).not.toContain(hexEncoded);
-    });
-  });
-});
-
-// ---------------------------------------------------------------------------
-// Decode helper unit tests
-// ---------------------------------------------------------------------------
-describe("decode helpers", () => {
-  test("tryDecodeBase64 returns decoded text for valid base64", () => {
-    const encoded = Buffer.from("sk_live_abcdefghijklmnopqrstuvwx").toString(
-      "base64",
-    );
-    expect(_tryDecodeBase64(encoded)).toBe("sk_live_abcdefghijklmnopqrstuvwx");
-  });
-
-  test("tryDecodeBase64 returns null for binary content", () => {
-    const binary = Buffer.from([0x00, 0x01, 0x80, 0xff]).toString("base64");
-    expect(_tryDecodeBase64(binary)).toBeNull();
-  });
-
-  test("tryDecodeBase64 returns null for invalid base64", () => {
-    expect(_tryDecodeBase64("not!!valid!!base64!!")).toBeNull();
-  });
-
-  test("tryDecodePercentEncoded returns decoded text", () => {
-    expect(_tryDecodePercentEncoded("hello%20world%21")).toBe("hello world!");
-  });
-
-  test("tryDecodePercentEncoded returns null when nothing to decode", () => {
-    expect(_tryDecodePercentEncoded("no-encoding-here")).toBeNull();
-  });
-
-  test("tryDecodeHexEscapes returns decoded text", () => {
-    expect(_tryDecodeHexEscapes("\\x48\\x65\\x6c\\x6c\\x6f")).toBe("Hello");
-  });
-
-  test("tryDecodeHexEscapes returns null when no escapes", () => {
-    expect(_tryDecodeHexEscapes("plain text")).toBeNull();
-  });
-
-  test("tryDecodeContinuousHex returns decoded text", () => {
-    const hex = Buffer.from("Hello").toString("hex");
-    expect(_tryDecodeContinuousHex(hex)).toBe("Hello");
-  });
-
-  test("tryDecodeContinuousHex returns null for non-printable result", () => {
-    // Hex that decodes to binary
-    expect(_tryDecodeContinuousHex("0001ff80")).toBeNull();
-  });
-});