@disco_trooper/apple-notes-mcp 1.2.0 → 1.3.0

package/src/utils/chunker.ts

@@ -0,0 +1,170 @@
+/**
+ * Text chunker with recursive character splitting that respects natural boundaries.
+ * Prioritizes splitting at: paragraphs > sentences > words > characters
+ */
+
+/**
+ * Separators in priority order - prefer splitting at larger boundaries first
+ */
+export const SEPARATORS = [
+  "\n\n", // Paragraph
+  "\n", // Line
+  ". ", // Sentence (period)
+  "! ", // Sentence (exclamation)
+  "? ", // Sentence (question)
+  "; ", // Clause
+  ", ", // Phrase
+  " ", // Word
+  "", // Character (fallback)
+] as const;
+
+export interface ChunkOptions {
+  /** Maximum size of each chunk in characters */
+  chunkSize: number;
+  /** Number of characters to overlap between chunks */
+  overlap: number;
+}
+
+export interface ChunkResult {
+  /** The text content of this chunk */
+  content: string;
+  /** Zero-based index of this chunk */
+  index: number;
+  /** Total number of chunks */
+  totalChunks: number;
+  /** Start position in original text */
+  startPos: number;
+  /** End position in original text (exclusive) */
+  endPos: number;
+}
+
+export const DEFAULT_CHUNK_OPTIONS: ChunkOptions = {
+  chunkSize: 500,
+  overlap: 100,
+};
+
+/**
+ * Find the best split point near the target position.
+ * Searches for separators in priority order within a reasonable range.
+ *
+ * @param text - The full text to search in
+ * @param target - The target position to split near
+ * @returns The best split position (after the separator)
+ */
+export function findSplitPoint(text: string, target: number): number {
+  // Search window: look backwards and forwards from target
+  const searchWindow = Math.min(50, Math.floor(target / 2));
+  const searchStart = Math.max(0, target - searchWindow);
+  const searchEnd = Math.min(text.length, target + searchWindow);
+  const searchText = text.slice(searchStart, searchEnd);
+
+  // Try each separator in priority order
+  for (const sep of SEPARATORS) {
+    if (sep === "") continue; // Skip empty string fallback for now
+
+    // Find all occurrences of separator in search window
+    let bestPos = -1;
+    let bestDistance = Infinity;
+
+    let idx = 0;
+    while ((idx = searchText.indexOf(sep, idx)) !== -1) {
+      const absolutePos = searchStart + idx + sep.length;
+      const distance = Math.abs(absolutePos - target);
+
+      if (distance < bestDistance) {
+        bestDistance = distance;
+        bestPos = absolutePos;
+      }
+      idx += 1;
+    }
+
+    if (bestPos !== -1) {
+      return bestPos;
+    }
+  }
+
+  // No separator found, return target as-is
+  return target;
+}
+
+/**
+ * Split text into overlapping chunks that respect natural boundaries.
+ *
+ * @param text - The text to chunk
+ * @param options - Chunk size and overlap options
+ * @returns Array of chunk results
+ */
+export function chunkText(
+  text: string,
+  options: ChunkOptions = DEFAULT_CHUNK_OPTIONS
+): ChunkResult[] {
+  const { chunkSize, overlap } = options;
+
+  // Handle empty or whitespace-only text
+  const trimmed = text.trim();
+  if (trimmed.length === 0) {
+    return [];
+  }
+
+  // If text fits in a single chunk, return it
+  if (text.length <= chunkSize) {
+    return [
+      {
+        content: text,
+        index: 0,
+        totalChunks: 1,
+        startPos: 0,
+        endPos: text.length,
+      },
+    ];
+  }
+
+  const chunks: ChunkResult[] = [];
+  let startPos = 0;
+  // Minimum step size to ensure progress and avoid tiny chunks
+  const minStep = Math.max(1, chunkSize - overlap);
+
+  while (startPos < text.length) {
+    // Calculate target end position
+    let endPos = Math.min(startPos + chunkSize, text.length);
+
+    // If not at the end, find a good split point
+    if (endPos < text.length) {
+      const splitPoint = findSplitPoint(text, endPos);
+      // Only use split point if it creates a reasonably sized chunk
+      if (
+        splitPoint > startPos + minStep / 2 &&
+        splitPoint - startPos <= chunkSize * 1.2
+      ) {
+        endPos = splitPoint;
+      }
+    }
+
+    // Extract chunk content
+    const content = text.slice(startPos, endPos);
+
+    chunks.push({
+      content,
+      index: chunks.length,
+      totalChunks: 0, // Will be set after all chunks are created
+      startPos,
+      endPos,
+    });
+
+    // If we've reached the end, stop
+    if (endPos >= text.length) {
+      break;
+    }
+
+    // Move to next chunk - ensure minimum step for progress
+    startPos = startPos + minStep;
+  }
+
+  // Set totalChunks on all chunks
+  const totalChunks = chunks.length;
+  for (const chunk of chunks) {
+    chunk.totalChunks = totalChunks;
+  }
+
+  return chunks;
+}

package/src/utils/content-filter.test.ts

@@ -0,0 +1,225 @@
+import { describe, it, expect } from "vitest";
+import {
+  calculateEntropy,
+  isLikelyBase64,
+  getBase64Ratio,
+  hasBinaryContent,
+  removeBase64Blocks,
+  redactSecrets,
+  filterContent,
+  shouldIndexContent,
+} from "./content-filter.js";
+
+describe("content-filter", () => {
+  describe("calculateEntropy", () => {
+    it("returns 0 for empty string", () => {
+      expect(calculateEntropy("")).toBe(0);
+    });
+
+    it("returns low entropy for repetitive text", () => {
+      const entropy = calculateEntropy("aaaaaaaaaa");
+      expect(entropy).toBe(0);
+    });
+
+    it("returns moderate entropy for normal text", () => {
+      const entropy = calculateEntropy("Hello, this is normal text.");
+      expect(entropy).toBeGreaterThan(2);
+      expect(entropy).toBeLessThan(5);
+    });
+
+    it("returns high entropy for Base64 content", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5";
+      const entropy = calculateEntropy(base64);
+      expect(entropy).toBeGreaterThan(4.5);
+    });
+  });
+
+  describe("isLikelyBase64", () => {
+    it("returns false for short strings", () => {
+      expect(isLikelyBase64("abc123")).toBe(false);
+    });
+
+    it("returns false for normal text", () => {
+      expect(isLikelyBase64("This is normal text with spaces and punctuation!")).toBe(false);
+    });
+
+    it("returns true for Base64 encoded content", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5eyJpc3MiOiJodHRwczovL2V4YW1wbGUu";
+      expect(isLikelyBase64(base64)).toBe(true);
+    });
+
+    it("returns true for URL-safe Base64", () => {
+      const urlSafe = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5_abc-def123456";
+      expect(isLikelyBase64(urlSafe)).toBe(true);
+    });
+  });
+
+  describe("getBase64Ratio", () => {
+    it("returns 0 for normal text", () => {
+      const ratio = getBase64Ratio("This is completely normal text.");
+      expect(ratio).toBe(0);
+    });
+
+    it("returns high ratio for mostly Base64 content", () => {
+      // Use actual high-entropy Base64, not repeated chars
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5eyJpc3M".repeat(3);
+      const content = "Token: " + base64;
+      const ratio = getBase64Ratio(content);
+      expect(ratio).toBeGreaterThan(0.5);
+    });
+
+    it("returns partial ratio for mixed content", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5";
+      const content = `Normal text here. ${base64} More normal text.`;
+      const ratio = getBase64Ratio(content);
+      expect(ratio).toBeGreaterThan(0);
+      expect(ratio).toBeLessThan(0.7);
+    });
+  });
+
+  describe("hasBinaryContent", () => {
+    it("returns false for normal text", () => {
+      expect(hasBinaryContent("Normal text")).toBe(false);
+    });
+
+    it("returns false for text with newlines and tabs", () => {
+      expect(hasBinaryContent("Line 1\nLine 2\tTabbed")).toBe(false);
+    });
+
+    it("returns true for null bytes", () => {
+      expect(hasBinaryContent("Text\x00with null")).toBe(true);
+    });
+
+    it("returns true for control characters", () => {
+      expect(hasBinaryContent("Text\x03with control")).toBe(true);
+    });
+  });
+
+  describe("removeBase64Blocks", () => {
+    it("removes Base64 blocks from content", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5eyJpc3M";
+      const content = `API Token: ${base64}\n\nNext section...`;
+      const result = removeBase64Blocks(content);
+
+      expect(result).not.toContain(base64);
+      expect(result).toContain("[ENCODED]");
+      expect(result).toContain("API Token:");
+      expect(result).toContain("Next section...");
+    });
+
+    it("preserves normal text", () => {
+      const content = "This is completely normal text without any encoding.";
+      expect(removeBase64Blocks(content)).toBe(content);
+    });
+  });
+
+  describe("redactSecrets", () => {
+    it("redacts JWT tokens", () => {
+      const jwt = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U";
+      const content = `Bearer ${jwt}`;
+      const { content: redacted, secretsFound } = redactSecrets(content);
+
+      expect(redacted).toContain("[JWT_REDACTED]");
+      expect(secretsFound).toContain("jwt");
+    });
+
+    it("redacts AWS access keys", () => {
+      const content = "AWS Key: AKIAIOSFODNN7EXAMPLE";
+      const { content: redacted, secretsFound } = redactSecrets(content);
+
+      expect(redacted).toContain("[AWSACCESSKEY_REDACTED]");
+      expect(secretsFound).toContain("awsAccessKey");
+    });
+
+    it("redacts GitHub tokens", () => {
+      const content = "Token: ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
+      const { content: redacted, secretsFound } = redactSecrets(content);
+
+      expect(redacted).toContain("[GITHUBTOKEN_REDACTED]");
+      expect(secretsFound).toContain("githubToken");
+    });
+
+    it("preserves normal text", () => {
+      const content = "This is normal text without secrets.";
+      const { content: redacted, secretsFound } = redactSecrets(content);
+
+      expect(redacted).toBe(content);
+      expect(secretsFound).toHaveLength(0);
+    });
+  });
+
+  describe("filterContent", () => {
+    it("returns 'index' for clean content", () => {
+      // Content must be at least 50 chars (minContentLength default)
+      const content = "This is clean, normal content for indexing. It contains enough text to pass the minimum length requirement.";
+      const result = filterContent(content);
+
+      expect(result.action).toBe("index");
+      expect(result.cleanedContent).toBe(content);
+      expect(result.reasons).toHaveLength(0);
+    });
+
+    it("returns 'skip' for binary content", () => {
+      const result = filterContent("Text\x00with null bytes");
+
+      expect(result.action).toBe("skip");
+      expect(result.reasons).toContain("Contains binary content");
+    });
+
+    it("returns 'skip' for mostly Base64 content", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5".repeat(10);
+      const result = filterContent(base64);
+
+      expect(result.action).toBe("skip");
+      expect(result.reasons[0]).toContain("Base64 encoded");
+    });
+
+    it("returns 'filter' for mixed content with Base64", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5";
+      const content = `This is important text. Token: ${base64}. More important content here that we want to index.`;
+      const result = filterContent(content);
+
+      expect(result.action).toBe("filter");
+      expect(result.cleanedContent).toContain("[ENCODED]");
+      expect(result.cleanedContent).toContain("This is important text");
+      expect(result.reasons.some(r => r.includes("Base64"))).toBe(true);
+    });
+
+    it("returns 'skip' if content too short after filtering", () => {
+      // Short text + Base64 that will be removed, leaving less than 50 chars
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5eyJpc3M";
+      const content = `Hi ${base64}`;
+      const result = filterContent(content);
+
+      expect(result.action).toBe("skip");
+      // After removing Base64, only "Hi [ENCODED]" remains which is too short
+    });
+
+    it("respects custom configuration", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5eyJpc3M";
+      // Need enough remaining content after potential filtering
+      const content = `This is some text before the token. Token: ${base64}. And this is some text after the token that should remain.`;
+
+      // With removeBase64 disabled, the Base64 should stay
+      const result = filterContent(content, { removeBase64: false });
+
+      expect(result.action).not.toBe("skip");
+      expect(result.cleanedContent).toContain(base64);
+    });
+  });
+
+  describe("shouldIndexContent", () => {
+    it("returns true for normal content", () => {
+      expect(shouldIndexContent("Normal text content")).toBe(true);
+    });
+
+    it("returns false for binary content", () => {
+      expect(shouldIndexContent("Binary\x00content")).toBe(false);
+    });
+
+    it("returns false for mostly Base64", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5".repeat(10);
+      expect(shouldIndexContent(base64)).toBe(false);
+    });
+  });
+});

package/src/utils/content-filter.ts

@@ -0,0 +1,275 @@
+/**
+ * Content quality filter for RAG indexing.
+ * Detects and filters Base64-encoded, binary, and secret content.
+ */
+
+import { createDebugLogger } from "./debug.js";
+
+const debug = createDebugLogger("CONTENT_FILTER");
+
+/**
+ * Result of content filtering.
+ */
+export interface FilterResult {
+  /** Whether to index this content */
+  action: "index" | "filter" | "skip";
+  /** Cleaned content (if action is "index" or "filter") */
+  cleanedContent?: string;
+  /** Reasons for filtering/skipping */
+  reasons: string[];
+}
+
+/**
+ * Calculate Shannon entropy of a string.
+ * Higher entropy = more random/encoded content.
+ *
+ * Typical values:
+ * - Normal text: 0.8 - 4.5
+ * - Base64: 5.0 - 6.0
+ * - Encrypted: 6.0+
+ *
+ * @param str - String to analyze
+ * @returns Entropy value (0-8)
+ */
+export function calculateEntropy(str: string): number {
+  if (!str || str.length === 0) return 0;
+
+  const freq = new Map<string, number>();
+  for (const char of str) {
+    freq.set(char, (freq.get(char) || 0) + 1);
+  }
+
+  let entropy = 0;
+  const len = str.length;
+  for (const count of freq.values()) {
+    const p = count / len;
+    entropy -= p * Math.log2(p);
+  }
+
+  return entropy;
+}
+
+/**
+ * Regex pattern for Base64 content (40+ chars).
+ */
+const BASE64_PATTERN = /[A-Za-z0-9+/]{40,}={0,2}/g;
+
+/**
+ * Regex pattern for URL-safe Base64.
+ */
+const BASE64_URL_SAFE_PATTERN = /[A-Za-z0-9_-]{40,}={0,2}/g;
+
+/**
+ * Patterns for common secrets/tokens.
+ */
+const SECRET_PATTERNS: Record<string, RegExp> = {
+  // Private Keys
+  privateKey: /-----BEGIN (?:RSA |DSA |EC |OPENSSH |PGP )?PRIVATE KEY(?: BLOCK)?-----/,
+
+  // JWT tokens
+  jwt: /eyJ[A-Za-z0-9-_=]+\.[A-Za-z0-9-_=]+\.?[A-Za-z0-9-_.+/=]*/g,
+
+  // AWS
+  awsAccessKey: /AKIA[0-9A-Z]{16}/g,
+
+  // GitHub
+  githubToken: /ghp_[a-zA-Z0-9]{36}/g,
+  githubFineGrained: /github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}/g,
+
+  // Slack
+  slackToken: /xox[baprs]-[0-9a-zA-Z]{10,48}/g,
+
+  // Stripe
+  stripeKey: /sk_live_[0-9a-zA-Z]{24}/g,
+
+  // Database URIs with credentials
+  dbUri: /(?:mongodb|postgres(?:ql)?|mysql|redis):\/\/[^\s'"]+:[^\s'"]+@[^\s'"]+/g,
+};
+
+/**
+ * Check if a string segment is likely Base64 encoded.
+ */
+export function isLikelyBase64(str: string): boolean {
+  // Minimum length check
+  if (str.length < 40) return false;
+
+  // Check if only Base64 characters
+  if (!/^[A-Za-z0-9+/=_-]+$/.test(str)) return false;
+
+  // Check entropy - Base64 typically has high entropy
+  const entropy = calculateEntropy(str);
+  return entropy > 4.5;
+}
+
+/**
+ * Calculate the ratio of Base64-like content in a string.
+ */
+export function getBase64Ratio(content: string): number {
+  const matches = content.match(BASE64_PATTERN) || [];
+  const urlSafeMatches = content.match(BASE64_URL_SAFE_PATTERN) || [];
+
+  // Combine and deduplicate
+  const allMatches = new Set([...matches, ...urlSafeMatches]);
+
+  let totalBase64Length = 0;
+  for (const match of allMatches) {
+    if (isLikelyBase64(match)) {
+      totalBase64Length += match.length;
+    }
+  }
+
+  return content.length > 0 ? totalBase64Length / content.length : 0;
+}
+
+/**
+ * Check if content contains binary/control characters.
+ */
+export function hasBinaryContent(content: string): boolean {
+  // Check for null bytes or control characters (except newlines/tabs)
+  return /[\x00-\x08\x0B\x0C\x0E-\x1F]/.test(content);
+}
+
+/**
+ * Remove Base64 blocks from content.
+ */
+export function removeBase64Blocks(content: string): string {
+  let result = content;
+
+  // Remove standard Base64
+  result = result.replace(BASE64_PATTERN, (match) => {
+    if (isLikelyBase64(match)) {
+      return "[ENCODED]";
+    }
+    return match;
+  });
+
+  // Remove URL-safe Base64
+  result = result.replace(BASE64_URL_SAFE_PATTERN, (match) => {
+    if (isLikelyBase64(match)) {
+      return "[ENCODED]";
+    }
+    return match;
+  });
+
+  return result;
+}
+
+/**
+ * Redact detected secrets in content.
+ */
+export function redactSecrets(content: string): { content: string; secretsFound: string[] } {
+  let result = content;
+  const secretsFound: string[] = [];
+
+  for (const [name, pattern] of Object.entries(SECRET_PATTERNS)) {
+    if (pattern.test(result)) {
+      // Reset lastIndex for global patterns
+      pattern.lastIndex = 0;
+      result = result.replace(pattern, `[${name.toUpperCase()}_REDACTED]`);
+      secretsFound.push(name);
+    }
+  }
+
+  return { content: result, secretsFound };
+}
+
+/**
+ * Configuration for content filtering.
+ */
+export interface FilterConfig {
+  /** Maximum Base64 ratio before skipping (default: 0.5) */
+  maxBase64Ratio?: number;
+  /** Minimum meaningful content length after filtering (default: 50) */
+  minContentLength?: number;
+  /** Whether to redact secrets (default: true) */
+  redactSecrets?: boolean;
+  /** Whether to remove Base64 blocks (default: true) */
+  removeBase64?: boolean;
+}
+
+const DEFAULT_CONFIG: Required<FilterConfig> = {
+  maxBase64Ratio: 0.5,
+  minContentLength: 50,
+  redactSecrets: true,
+  removeBase64: true,
+};
+
+/**
+ * Filter content for RAG indexing.
+ *
+ * @param content - Raw content to filter
+ * @param config - Filter configuration
+ * @returns Filter result with action and cleaned content
+ */
+export function filterContent(
+  content: string,
+  config: FilterConfig = {}
+): FilterResult {
+  const cfg = { ...DEFAULT_CONFIG, ...config };
+  const reasons: string[] = [];
+
+  // 1. Check for binary content - skip entirely
+  if (hasBinaryContent(content)) {
+    debug("Skipping content with binary characters");
+    return { action: "skip", reasons: ["Contains binary content"] };
+  }
+
+  // 2. Calculate Base64 ratio
+  const base64Ratio = getBase64Ratio(content);
+  debug(`Base64 ratio: ${(base64Ratio * 100).toFixed(1)}%`);
+
+  // Skip if too much encoded content
+  if (base64Ratio > cfg.maxBase64Ratio) {
+    debug(`Skipping content: ${(base64Ratio * 100).toFixed(1)}% Base64`);
+    return {
+      action: "skip",
+      reasons: [`${(base64Ratio * 100).toFixed(1)}% is Base64 encoded (threshold: ${(cfg.maxBase64Ratio * 100).toFixed(0)}%)`],
+    };
+  }
+
+  let cleanedContent = content;
+
+  // 3. Remove Base64 blocks if present and configured
+  if (cfg.removeBase64 && base64Ratio > 0.1) {
+    cleanedContent = removeBase64Blocks(cleanedContent);
+    reasons.push("Removed Base64 blocks");
+  }
+
+  // 4. Redact secrets if configured
+  if (cfg.redactSecrets) {
+    const { content: redacted, secretsFound } = redactSecrets(cleanedContent);
+    if (secretsFound.length > 0) {
+      cleanedContent = redacted;
+      reasons.push(`Redacted secrets: ${secretsFound.join(", ")}`);
+    }
+  }
+
+  // 5. Check if remaining content is meaningful
+  const meaningfulContent = cleanedContent
+    .replace(/\[.*?_REDACTED\]|\[ENCODED\]/g, "")
+    .trim();
+
+  if (meaningfulContent.length < cfg.minContentLength) {
+    debug(`Skipping: insufficient content after filtering (${meaningfulContent.length} chars)`);
+    return {
+      action: "skip",
+      reasons: ["Insufficient meaningful content after filtering"],
+    };
+  }
+
+  // Determine action
+  const action = reasons.length > 0 ? "filter" : "index";
+
+  return { action, cleanedContent, reasons };
+}
+
+/**
+ * Quick check if content should be indexed.
+ * Use this for fast pre-filtering before chunking.
+ */
+export function shouldIndexContent(content: string): boolean {
+  // Quick checks
+  if (hasBinaryContent(content)) return false;
+  if (getBase64Ratio(content) > 0.5) return false;
+  return true;
+}