@disco_trooper/apple-notes-mcp 1.2.0 → 1.4.0

package/src/utils/content-filter.test.ts

@@ -0,0 +1,225 @@
+import { describe, it, expect } from "vitest";
+import {
+  calculateEntropy,
+  isLikelyBase64,
+  getBase64Ratio,
+  hasBinaryContent,
+  removeBase64Blocks,
+  redactSecrets,
+  filterContent,
+  shouldIndexContent,
+} from "./content-filter.js";
+
+describe("content-filter", () => {
+  describe("calculateEntropy", () => {
+    it("returns 0 for empty string", () => {
+      expect(calculateEntropy("")).toBe(0);
+    });
+
+    it("returns low entropy for repetitive text", () => {
+      const entropy = calculateEntropy("aaaaaaaaaa");
+      expect(entropy).toBe(0);
+    });
+
+    it("returns moderate entropy for normal text", () => {
+      const entropy = calculateEntropy("Hello, this is normal text.");
+      expect(entropy).toBeGreaterThan(2);
+      expect(entropy).toBeLessThan(5);
+    });
+
+    it("returns high entropy for Base64 content", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5";
+      const entropy = calculateEntropy(base64);
+      expect(entropy).toBeGreaterThan(4.5);
+    });
+  });
+
+  describe("isLikelyBase64", () => {
+    it("returns false for short strings", () => {
+      expect(isLikelyBase64("abc123")).toBe(false);
+    });
+
+    it("returns false for normal text", () => {
+      expect(isLikelyBase64("This is normal text with spaces and punctuation!")).toBe(false);
+    });
+
+    it("returns true for Base64 encoded content", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5eyJpc3MiOiJodHRwczovL2V4YW1wbGUu";
+      expect(isLikelyBase64(base64)).toBe(true);
+    });
+
+    it("returns true for URL-safe Base64", () => {
+      const urlSafe = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5_abc-def123456";
+      expect(isLikelyBase64(urlSafe)).toBe(true);
+    });
+  });
+
+  describe("getBase64Ratio", () => {
+    it("returns 0 for normal text", () => {
+      const ratio = getBase64Ratio("This is completely normal text.");
+      expect(ratio).toBe(0);
+    });
+
+    it("returns high ratio for mostly Base64 content", () => {
+      // Use actual high-entropy Base64, not repeated chars
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5eyJpc3M".repeat(3);
+      const content = "Token: " + base64;
+      const ratio = getBase64Ratio(content);
+      expect(ratio).toBeGreaterThan(0.5);
+    });
+
+    it("returns partial ratio for mixed content", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5";
+      const content = `Normal text here. ${base64} More normal text.`;
+      const ratio = getBase64Ratio(content);
+      expect(ratio).toBeGreaterThan(0);
+      expect(ratio).toBeLessThan(0.7);
+    });
+  });
+
+  describe("hasBinaryContent", () => {
+    it("returns false for normal text", () => {
+      expect(hasBinaryContent("Normal text")).toBe(false);
+    });
+
+    it("returns false for text with newlines and tabs", () => {
+      expect(hasBinaryContent("Line 1\nLine 2\tTabbed")).toBe(false);
+    });
+
+    it("returns true for null bytes", () => {
+      expect(hasBinaryContent("Text\x00with null")).toBe(true);
+    });
+
+    it("returns true for control characters", () => {
+      expect(hasBinaryContent("Text\x03with control")).toBe(true);
+    });
+  });
+
+  describe("removeBase64Blocks", () => {
+    it("removes Base64 blocks from content", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5eyJpc3M";
+      const content = `API Token: ${base64}\n\nNext section...`;
+      const result = removeBase64Blocks(content);
+
+      expect(result).not.toContain(base64);
+      expect(result).toContain("[ENCODED]");
+      expect(result).toContain("API Token:");
+      expect(result).toContain("Next section...");
+    });
+
+    it("preserves normal text", () => {
+      const content = "This is completely normal text without any encoding.";
+      expect(removeBase64Blocks(content)).toBe(content);
+    });
+  });
+
+  describe("redactSecrets", () => {
+    it("redacts JWT tokens", () => {
+      const jwt = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U";
+      const content = `Bearer ${jwt}`;
+      const { content: redacted, secretsFound } = redactSecrets(content);
+
+      expect(redacted).toContain("[JWT_REDACTED]");
+      expect(secretsFound).toContain("jwt");
+    });
+
+    it("redacts AWS access keys", () => {
+      const content = "AWS Key: AKIAIOSFODNN7EXAMPLE";
+      const { content: redacted, secretsFound } = redactSecrets(content);
+
+      expect(redacted).toContain("[AWSACCESSKEY_REDACTED]");
+      expect(secretsFound).toContain("awsAccessKey");
+    });
+
+    it("redacts GitHub tokens", () => {
+      const content = "Token: ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
+      const { content: redacted, secretsFound } = redactSecrets(content);
+
+      expect(redacted).toContain("[GITHUBTOKEN_REDACTED]");
+      expect(secretsFound).toContain("githubToken");
+    });
+
+    it("preserves normal text", () => {
+      const content = "This is normal text without secrets.";
+      const { content: redacted, secretsFound } = redactSecrets(content);
+
+      expect(redacted).toBe(content);
+      expect(secretsFound).toHaveLength(0);
+    });
+  });
+
+  describe("filterContent", () => {
+    it("returns 'index' for clean content", () => {
+      // Content must be at least 50 chars (minContentLength default)
+      const content = "This is clean, normal content for indexing. It contains enough text to pass the minimum length requirement.";
+      const result = filterContent(content);
+
+      expect(result.action).toBe("index");
+      expect(result.cleanedContent).toBe(content);
+      expect(result.reasons).toHaveLength(0);
+    });
+
+    it("returns 'skip' for binary content", () => {
+      const result = filterContent("Text\x00with null bytes");
+
+      expect(result.action).toBe("skip");
+      expect(result.reasons).toContain("Contains binary content");
+    });
+
+    it("returns 'skip' for mostly Base64 content", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5".repeat(10);
+      const result = filterContent(base64);
+
+      expect(result.action).toBe("skip");
+      expect(result.reasons[0]).toContain("Base64 encoded");
+    });
+
+    it("returns 'filter' for mixed content with Base64", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5";
+      const content = `This is important text. Token: ${base64}. More important content here that we want to index.`;
+      const result = filterContent(content);
+
+      expect(result.action).toBe("filter");
+      expect(result.cleanedContent).toContain("[ENCODED]");
+      expect(result.cleanedContent).toContain("This is important text");
+      expect(result.reasons.some(r => r.includes("Base64"))).toBe(true);
+    });
+
+    it("returns 'skip' if content too short after filtering", () => {
+      // Short text + Base64 that will be removed, leaving less than 50 chars
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5eyJpc3M";
+      const content = `Hi ${base64}`;
+      const result = filterContent(content);
+
+      expect(result.action).toBe("skip");
+      // After removing Base64, only "Hi [ENCODED]" remains which is too short
+    });
+
+    it("respects custom configuration", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5eyJpc3M";
+      // Need enough remaining content after potential filtering
+      const content = `This is some text before the token. Token: ${base64}. And this is some text after the token that should remain.`;
+
+      // With removeBase64 disabled, the Base64 should stay
+      const result = filterContent(content, { removeBase64: false });
+
+      expect(result.action).not.toBe("skip");
+      expect(result.cleanedContent).toContain(base64);
+    });
+  });
+
+  describe("shouldIndexContent", () => {
+    it("returns true for normal content", () => {
+      expect(shouldIndexContent("Normal text content")).toBe(true);
+    });
+
+    it("returns false for binary content", () => {
+      expect(shouldIndexContent("Binary\x00content")).toBe(false);
+    });
+
+    it("returns false for mostly Base64", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5".repeat(10);
+      expect(shouldIndexContent(base64)).toBe(false);
+    });
+  });
+});

package/src/utils/content-filter.ts

@@ -0,0 +1,275 @@
+/**
+ * Content quality filter for RAG indexing.
+ * Detects and filters Base64-encoded, binary, and secret content.
+ */
+
+import { createDebugLogger } from "./debug.js";
+
+const debug = createDebugLogger("CONTENT_FILTER");
+
+/**
+ * Result of content filtering.
+ */
+export interface FilterResult {
+  /** Whether to index this content */
+  action: "index" | "filter" | "skip";
+  /** Cleaned content (if action is "index" or "filter") */
+  cleanedContent?: string;
+  /** Reasons for filtering/skipping */
+  reasons: string[];
+}
+
+/**
+ * Calculate Shannon entropy of a string.
+ * Higher entropy = more random/encoded content.
+ *
+ * Typical values:
+ * - Normal text: 0.8 - 4.5
+ * - Base64: 5.0 - 6.0
+ * - Encrypted: 6.0+
+ *
+ * @param str - String to analyze
+ * @returns Entropy value (0-8)
+ */
+export function calculateEntropy(str: string): number {
+  if (!str || str.length === 0) return 0;
+
+  const freq = new Map<string, number>();
+  for (const char of str) {
+    freq.set(char, (freq.get(char) || 0) + 1);
+  }
+
+  let entropy = 0;
+  const len = str.length;
+  for (const count of freq.values()) {
+    const p = count / len;
+    entropy -= p * Math.log2(p);
+  }
+
+  return entropy;
+}
+
+/**
+ * Regex pattern for Base64 content (40+ chars).
+ */
+const BASE64_PATTERN = /[A-Za-z0-9+/]{40,}={0,2}/g;
+
+/**
+ * Regex pattern for URL-safe Base64.
+ */
+const BASE64_URL_SAFE_PATTERN = /[A-Za-z0-9_-]{40,}={0,2}/g;
+
+/**
+ * Patterns for common secrets/tokens.
+ */
+const SECRET_PATTERNS: Record<string, RegExp> = {
+  // Private Keys
+  privateKey: /-----BEGIN (?:RSA |DSA |EC |OPENSSH |PGP )?PRIVATE KEY(?: BLOCK)?-----/,
+
+  // JWT tokens
+  jwt: /eyJ[A-Za-z0-9-_=]+\.[A-Za-z0-9-_=]+\.?[A-Za-z0-9-_.+/=]*/g,
+
+  // AWS
+  awsAccessKey: /AKIA[0-9A-Z]{16}/g,
+
+  // GitHub
+  githubToken: /ghp_[a-zA-Z0-9]{36}/g,
+  githubFineGrained: /github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}/g,
+
+  // Slack
+  slackToken: /xox[baprs]-[0-9a-zA-Z]{10,48}/g,
+
+  // Stripe
+  stripeKey: /sk_live_[0-9a-zA-Z]{24}/g,
+
+  // Database URIs with credentials
+  dbUri: /(?:mongodb|postgres(?:ql)?|mysql|redis):\/\/[^\s'"]+:[^\s'"]+@[^\s'"]+/g,
+};
+
+/**
+ * Check if a string segment is likely Base64 encoded.
+ */
+export function isLikelyBase64(str: string): boolean {
+  // Minimum length check
+  if (str.length < 40) return false;
+
+  // Check if only Base64 characters
+  if (!/^[A-Za-z0-9+/=_-]+$/.test(str)) return false;
+
+  // Check entropy - Base64 typically has high entropy
+  const entropy = calculateEntropy(str);
+  return entropy > 4.5;
+}
+
+/**
+ * Calculate the ratio of Base64-like content in a string.
+ */
+export function getBase64Ratio(content: string): number {
+  const matches = content.match(BASE64_PATTERN) || [];
+  const urlSafeMatches = content.match(BASE64_URL_SAFE_PATTERN) || [];
+
+  // Combine and deduplicate
+  const allMatches = new Set([...matches, ...urlSafeMatches]);
+
+  let totalBase64Length = 0;
+  for (const match of allMatches) {
+    if (isLikelyBase64(match)) {
+      totalBase64Length += match.length;
+    }
+  }
+
+  return content.length > 0 ? totalBase64Length / content.length : 0;
+}
+
+/**
+ * Check if content contains binary/control characters.
+ */
+export function hasBinaryContent(content: string): boolean {
+  // Check for null bytes or control characters (except newlines/tabs)
+  return /[\x00-\x08\x0B\x0C\x0E-\x1F]/.test(content);
+}
+
+/**
+ * Remove Base64 blocks from content.
+ */
+export function removeBase64Blocks(content: string): string {
+  let result = content;
+
+  // Remove standard Base64
+  result = result.replace(BASE64_PATTERN, (match) => {
+    if (isLikelyBase64(match)) {
+      return "[ENCODED]";
+    }
+    return match;
+  });
+
+  // Remove URL-safe Base64
+  result = result.replace(BASE64_URL_SAFE_PATTERN, (match) => {
+    if (isLikelyBase64(match)) {
+      return "[ENCODED]";
+    }
+    return match;
+  });
+
+  return result;
+}
+
+/**
+ * Redact detected secrets in content.
+ */
+export function redactSecrets(content: string): { content: string; secretsFound: string[] } {
+  let result = content;
+  const secretsFound: string[] = [];
+
+  for (const [name, pattern] of Object.entries(SECRET_PATTERNS)) {
+    if (pattern.test(result)) {
+      // Reset lastIndex for global patterns
+      pattern.lastIndex = 0;
+      result = result.replace(pattern, `[${name.toUpperCase()}_REDACTED]`);
+      secretsFound.push(name);
+    }
+  }
+
+  return { content: result, secretsFound };
+}
+
+/**
+ * Configuration for content filtering.
+ */
+export interface FilterConfig {
+  /** Maximum Base64 ratio before skipping (default: 0.5) */
+  maxBase64Ratio?: number;
+  /** Minimum meaningful content length after filtering (default: 50) */
+  minContentLength?: number;
+  /** Whether to redact secrets (default: true) */
+  redactSecrets?: boolean;
+  /** Whether to remove Base64 blocks (default: true) */
+  removeBase64?: boolean;
+}
+
+const DEFAULT_CONFIG: Required<FilterConfig> = {
+  maxBase64Ratio: 0.5,
+  minContentLength: 50,
+  redactSecrets: true,
+  removeBase64: true,
+};
+
+/**
+ * Filter content for RAG indexing.
+ *
+ * @param content - Raw content to filter
+ * @param config - Filter configuration
+ * @returns Filter result with action and cleaned content
+ */
+export function filterContent(
+  content: string,
+  config: FilterConfig = {}
+): FilterResult {
+  const cfg = { ...DEFAULT_CONFIG, ...config };
+  const reasons: string[] = [];
+
+  // 1. Check for binary content - skip entirely
+  if (hasBinaryContent(content)) {
+    debug("Skipping content with binary characters");
+    return { action: "skip", reasons: ["Contains binary content"] };
+  }
+
+  // 2. Calculate Base64 ratio
+  const base64Ratio = getBase64Ratio(content);
+  debug(`Base64 ratio: ${(base64Ratio * 100).toFixed(1)}%`);
+
+  // Skip if too much encoded content
+  if (base64Ratio > cfg.maxBase64Ratio) {
+    debug(`Skipping content: ${(base64Ratio * 100).toFixed(1)}% Base64`);
+    return {
+      action: "skip",
+      reasons: [`${(base64Ratio * 100).toFixed(1)}% is Base64 encoded (threshold: ${(cfg.maxBase64Ratio * 100).toFixed(0)}%)`],
+    };
+  }
+
+  let cleanedContent = content;
+
+  // 3. Remove Base64 blocks if present and configured
+  if (cfg.removeBase64 && base64Ratio > 0.1) {
+    cleanedContent = removeBase64Blocks(cleanedContent);
+    reasons.push("Removed Base64 blocks");
+  }
+
+  // 4. Redact secrets if configured
+  if (cfg.redactSecrets) {
+    const { content: redacted, secretsFound } = redactSecrets(cleanedContent);
+    if (secretsFound.length > 0) {
+      cleanedContent = redacted;
+      reasons.push(`Redacted secrets: ${secretsFound.join(", ")}`);
+    }
+  }
+
+  // 5. Check if remaining content is meaningful
+  const meaningfulContent = cleanedContent
+    .replace(/\[.*?_REDACTED\]|\[ENCODED\]/g, "")
+    .trim();
+
+  if (meaningfulContent.length < cfg.minContentLength) {
+    debug(`Skipping: insufficient content after filtering (${meaningfulContent.length} chars)`);
+    return {
+      action: "skip",
+      reasons: ["Insufficient meaningful content after filtering"],
+    };
+  }
+
+  // Determine action
+  const action = reasons.length > 0 ? "filter" : "index";
+
+  return { action, cleanedContent, reasons };
+}
+
+/**
+ * Quick check if content should be indexed.
+ * Use this for fast pre-filtering before chunking.
+ */
+export function shouldIndexContent(content: string): boolean {
+  // Quick checks
+  if (hasBinaryContent(content)) return false;
+  if (getBase64Ratio(content) > 0.5) return false;
+  return true;
+}

package/src/utils/runtime.test.ts

@@ -0,0 +1,70 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+
+describe("runtime checks", () => {
+  beforeEach(() => {
+    vi.resetModules();
+  });
+
+  describe("isBunRuntime", () => {
+    it("should return false when Bun global is not defined", async () => {
+      // vitest runs in Node.js, so Bun is not defined
+      const { isBunRuntime } = await import("./runtime.js");
+      expect(isBunRuntime()).toBe(false);
+    });
+
+    it("should return true when Bun global is defined", async () => {
+      // Mock Bun global
+      (globalThis as Record<string, unknown>).Bun = {};
+      const { isBunRuntime } = await import("./runtime.js");
+      expect(isBunRuntime()).toBe(true);
+      delete (globalThis as Record<string, unknown>).Bun;
+    });
+  });
+
+  describe("checkBunRuntime", () => {
+    let mockExit: ReturnType<typeof vi.spyOn>;
+    let mockConsoleError: ReturnType<typeof vi.spyOn>;
+
+    beforeEach(() => {
+      mockExit = vi.spyOn(process, "exit").mockImplementation(() => {
+        throw new Error("process.exit called");
+      });
+      mockConsoleError = vi.spyOn(console, "error").mockImplementation(() => {});
+    });
+
+    afterEach(() => {
+      mockExit.mockRestore();
+      mockConsoleError.mockRestore();
+      delete (globalThis as Record<string, unknown>).Bun;
+    });
+
+    it("should exit with error message when Bun is not available", async () => {
+      const { checkBunRuntime } = await import("./runtime.js");
+      expect(() => checkBunRuntime()).toThrow("process.exit called");
+      expect(mockExit).toHaveBeenCalledWith(1);
+      expect(mockConsoleError).toHaveBeenCalled();
+    });
+
+    it("should not exit when Bun is available", async () => {
+      (globalThis as Record<string, unknown>).Bun = {};
+      const { checkBunRuntime } = await import("./runtime.js");
+      expect(() => checkBunRuntime()).not.toThrow();
+      expect(mockExit).not.toHaveBeenCalled();
+    });
+  });
+
+  describe("isTTY", () => {
+    it("should return boolean", async () => {
+      const { isTTY } = await import("./runtime.js");
+      expect(typeof isTTY()).toBe("boolean");
+    });
+
+    it("should return false when stdin or stdout is not a TTY", async () => {
+      // In CI/test environments, typically not a TTY
+      const { isTTY } = await import("./runtime.js");
+      // The result depends on the environment, but it should be a boolean
+      const result = isTTY();
+      expect(result === true || result === false).toBe(true);
+    });
+  });
+});

package/src/utils/runtime.ts

@@ -0,0 +1,40 @@
+/**
+ * Runtime environment checks.
+ */
+
+/**
+ * Check if running in Bun runtime.
+ */
+export function isBunRuntime(): boolean {
+  return typeof Bun !== "undefined";
+}
+
+/**
+ * Check Bun runtime and throw helpful error if not available.
+ */
+export function checkBunRuntime(): void {
+  if (!isBunRuntime()) {
+    console.error(`
+╭─────────────────────────────────────────────────────────────╮
+│  apple-notes-mcp requires Bun runtime                       │
+│                                                             │
+│  Install Bun:                                               │
+│    curl -fsSL https://bun.sh/install | bash                 │
+│                                                             │
+│  Or with Homebrew:                                          │
+│    brew install bun                                         │
+│                                                             │
+│  Then run again:                                            │
+│    apple-notes-mcp                                          │
+╰─────────────────────────────────────────────────────────────╯
+`);
+    process.exit(1);
+  }
+}
+
+/**
+ * Check if running in interactive terminal (TTY).
+ */
+export function isTTY(): boolean {
+  return process.stdin.isTTY === true && process.stdout.isTTY === true;
+}