@disco_trooper/apple-notes-mcp 1.1.0 → 1.3.0

package/src/utils/chunker.test.ts

@@ -0,0 +1,182 @@
+import { describe, expect, it } from "vitest";
+import {
+  chunkText,
+  type ChunkOptions,
+  DEFAULT_CHUNK_OPTIONS,
+  SEPARATORS,
+  findSplitPoint,
+} from "./chunker.js";
+
+describe("chunker", () => {
+  describe("exports", () => {
+    it("exports SEPARATORS array with correct order", () => {
+      expect(SEPARATORS).toEqual([
+        "\n\n",
+        "\n",
+        ". ",
+        "! ",
+        "? ",
+        "; ",
+        ", ",
+        " ",
+        "",
+      ]);
+    });
+
+    it("exports DEFAULT_CHUNK_OPTIONS with correct values", () => {
+      expect(DEFAULT_CHUNK_OPTIONS).toEqual({
+        chunkSize: 500,
+        overlap: 100,
+      });
+    });
+  });
+
+  describe("findSplitPoint", () => {
+    it("finds paragraph boundary near target", () => {
+      const text = "First paragraph.\n\nSecond paragraph.";
+      const target = 20;
+      const result = findSplitPoint(text, target);
+      // Should find the \n\n at position 16
+      expect(result).toBe(18); // After \n\n
+    });
+
+    it("falls back to sentence boundary", () => {
+      const text = "First sentence. Second sentence.";
+      const target = 18;
+      const result = findSplitPoint(text, target);
+      // Should find ". " at position 14-16
+      expect(result).toBe(16); // After ". "
+    });
+
+    it("falls back to word boundary", () => {
+      const text = "oneword anotherword";
+      const target = 10;
+      const result = findSplitPoint(text, target);
+      // Should find space at position 7
+      expect(result).toBe(8); // After " "
+    });
+
+    it("returns target when no separator found", () => {
+      const text = "noseparatorshere";
+      const target = 8;
+      const result = findSplitPoint(text, target);
+      expect(result).toBe(8);
+    });
+  });
+
+  describe("chunkText", () => {
+    it("returns single chunk for short text", () => {
+      const text = "Short text";
+      const options: ChunkOptions = { chunkSize: 100, overlap: 20 };
+
+      const result = chunkText(text, options);
+
+      expect(result).toHaveLength(1);
+      expect(result[0]).toEqual({
+        content: "Short text",
+        index: 0,
+        totalChunks: 1,
+        startPos: 0,
+        endPos: 10,
+      });
+    });
+
+    it("creates multiple chunks for long text", () => {
+      const text = "Word ".repeat(50).trim(); // 249 chars
+      const options: ChunkOptions = { chunkSize: 50, overlap: 10 };
+
+      const result = chunkText(text, options);
+
+      expect(result.length).toBeGreaterThan(1);
+      // Each chunk should have content
+      result.forEach((chunk) => {
+        expect(chunk.content.length).toBeGreaterThan(0);
+        expect(chunk.content.length).toBeLessThanOrEqual(options.chunkSize);
+      });
+    });
+
+    it("includes overlap between chunks", () => {
+      const text = "First part. Second part. Third part. Fourth part.";
+      const options: ChunkOptions = { chunkSize: 25, overlap: 10 };
+
+      const result = chunkText(text, options);
+
+      // Check that chunks overlap - endPos of chunk N should be > startPos of chunk N+1
+      for (let i = 0; i < result.length - 1; i++) {
+        const currentChunk = result[i];
+        const nextChunk = result[i + 1];
+        // Overlap means next chunk starts before current chunk ends
+        expect(nextChunk.startPos).toBeLessThan(currentChunk.endPos);
+      }
+    });
+
+    it("respects paragraph boundaries when splitting", () => {
+      const text = "First paragraph here.\n\nSecond paragraph here.\n\nThird paragraph.";
+      const options: ChunkOptions = { chunkSize: 30, overlap: 5 };
+
+      const result = chunkText(text, options);
+
+      // At least one chunk should end at a paragraph boundary
+      const hasParaBoundary = result.some((chunk) => {
+        const endContent = text.slice(chunk.startPos, chunk.endPos);
+        return endContent.endsWith("\n\n") || chunk.endPos === text.length;
+      });
+      expect(hasParaBoundary).toBe(true);
+    });
+
+    it("sets correct totalChunks on all chunks", () => {
+      const text = "A ".repeat(100).trim(); // Create text that will be chunked
+      const options: ChunkOptions = { chunkSize: 20, overlap: 5 };
+
+      const result = chunkText(text, options);
+
+      const expectedTotal = result.length;
+      result.forEach((chunk, idx) => {
+        expect(chunk.totalChunks).toBe(expectedTotal);
+        expect(chunk.index).toBe(idx);
+      });
+    });
+
+    it("handles empty text", () => {
+      const result = chunkText("", { chunkSize: 100, overlap: 20 });
+
+      expect(result).toHaveLength(0);
+    });
+
+    it("handles whitespace-only text", () => {
+      const result = chunkText("   \n\n   ", { chunkSize: 100, overlap: 20 });
+
+      expect(result).toHaveLength(0);
+    });
+
+    it("uses default options when not provided", () => {
+      const text = "Test";
+      const result = chunkText(text);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe("Test");
+    });
+
+    it("covers all original text with chunks", () => {
+      const text = "The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs.";
+      const options: ChunkOptions = { chunkSize: 30, overlap: 10 };
+
+      const result = chunkText(text, options);
+
+      // Verify chunks cover the entire text
+      expect(result[0].startPos).toBe(0);
+      expect(result[result.length - 1].endPos).toBe(text.length);
+
+      // Verify each chunk's content matches its position in original text
+      for (const chunk of result) {
+        expect(chunk.content).toBe(text.slice(chunk.startPos, chunk.endPos));
+      }
+
+      // Verify chunks are contiguous (no gaps)
+      for (let i = 0; i < result.length - 1; i++) {
+        // Next chunk should start before or at current chunk's end (overlap)
+        expect(result[i + 1].startPos).toBeLessThanOrEqual(result[i].endPos);
+      }
+    });
+  });
+});

package/src/utils/chunker.ts

@@ -0,0 +1,170 @@
+/**
+ * Text chunker with recursive character splitting that respects natural boundaries.
+ * Prioritizes splitting at: paragraphs > sentences > words > characters
+ */
+
+/**
+ * Separators in priority order - prefer splitting at larger boundaries first
+ */
+export const SEPARATORS = [
+  "\n\n", // Paragraph
+  "\n", // Line
+  ". ", // Sentence (period)
+  "! ", // Sentence (exclamation)
+  "? ", // Sentence (question)
+  "; ", // Clause
+  ", ", // Phrase
+  " ", // Word
+  "", // Character (fallback)
+] as const;
+
+export interface ChunkOptions {
+  /** Maximum size of each chunk in characters */
+  chunkSize: number;
+  /** Number of characters to overlap between chunks */
+  overlap: number;
+}
+
+export interface ChunkResult {
+  /** The text content of this chunk */
+  content: string;
+  /** Zero-based index of this chunk */
+  index: number;
+  /** Total number of chunks */
+  totalChunks: number;
+  /** Start position in original text */
+  startPos: number;
+  /** End position in original text (exclusive) */
+  endPos: number;
+}
+
+export const DEFAULT_CHUNK_OPTIONS: ChunkOptions = {
+  chunkSize: 500,
+  overlap: 100,
+};
+
+/**
+ * Find the best split point near the target position.
+ * Searches for separators in priority order within a reasonable range.
+ *
+ * @param text - The full text to search in
+ * @param target - The target position to split near
+ * @returns The best split position (after the separator)
+ */
+export function findSplitPoint(text: string, target: number): number {
+  // Search window: look backwards and forwards from target
+  const searchWindow = Math.min(50, Math.floor(target / 2));
+  const searchStart = Math.max(0, target - searchWindow);
+  const searchEnd = Math.min(text.length, target + searchWindow);
+  const searchText = text.slice(searchStart, searchEnd);
+
+  // Try each separator in priority order
+  for (const sep of SEPARATORS) {
+    if (sep === "") continue; // Skip empty string fallback for now
+
+    // Find all occurrences of separator in search window
+    let bestPos = -1;
+    let bestDistance = Infinity;
+
+    let idx = 0;
+    while ((idx = searchText.indexOf(sep, idx)) !== -1) {
+      const absolutePos = searchStart + idx + sep.length;
+      const distance = Math.abs(absolutePos - target);
+
+      if (distance < bestDistance) {
+        bestDistance = distance;
+        bestPos = absolutePos;
+      }
+      idx += 1;
+    }
+
+    if (bestPos !== -1) {
+      return bestPos;
+    }
+  }
+
+  // No separator found, return target as-is
+  return target;
+}
+
+/**
+ * Split text into overlapping chunks that respect natural boundaries.
+ *
+ * @param text - The text to chunk
+ * @param options - Chunk size and overlap options
+ * @returns Array of chunk results
+ */
+export function chunkText(
+  text: string,
+  options: ChunkOptions = DEFAULT_CHUNK_OPTIONS
+): ChunkResult[] {
+  const { chunkSize, overlap } = options;
+
+  // Handle empty or whitespace-only text
+  const trimmed = text.trim();
+  if (trimmed.length === 0) {
+    return [];
+  }
+
+  // If text fits in a single chunk, return it
+  if (text.length <= chunkSize) {
+    return [
+      {
+        content: text,
+        index: 0,
+        totalChunks: 1,
+        startPos: 0,
+        endPos: text.length,
+      },
+    ];
+  }
+
+  const chunks: ChunkResult[] = [];
+  let startPos = 0;
+  // Minimum step size to ensure progress and avoid tiny chunks
+  const minStep = Math.max(1, chunkSize - overlap);
+
+  while (startPos < text.length) {
+    // Calculate target end position
+    let endPos = Math.min(startPos + chunkSize, text.length);
+
+    // If not at the end, find a good split point
+    if (endPos < text.length) {
+      const splitPoint = findSplitPoint(text, endPos);
+      // Only use split point if it creates a reasonably sized chunk
+      if (
+        splitPoint > startPos + minStep / 2 &&
+        splitPoint - startPos <= chunkSize * 1.2
+      ) {
+        endPos = splitPoint;
+      }
+    }
+
+    // Extract chunk content
+    const content = text.slice(startPos, endPos);
+
+    chunks.push({
+      content,
+      index: chunks.length,
+      totalChunks: 0, // Will be set after all chunks are created
+      startPos,
+      endPos,
+    });
+
+    // If we've reached the end, stop
+    if (endPos >= text.length) {
+      break;
+    }
+
+    // Move to next chunk - ensure minimum step for progress
+    startPos = startPos + minStep;
+  }
+
+  // Set totalChunks on all chunks
+  const totalChunks = chunks.length;
+  for (const chunk of chunks) {
+    chunk.totalChunks = totalChunks;
+  }
+
+  return chunks;
+}

package/src/utils/content-filter.test.ts

@@ -0,0 +1,225 @@
+import { describe, it, expect } from "vitest";
+import {
+  calculateEntropy,
+  isLikelyBase64,
+  getBase64Ratio,
+  hasBinaryContent,
+  removeBase64Blocks,
+  redactSecrets,
+  filterContent,
+  shouldIndexContent,
+} from "./content-filter.js";
+
+describe("content-filter", () => {
+  describe("calculateEntropy", () => {
+    it("returns 0 for empty string", () => {
+      expect(calculateEntropy("")).toBe(0);
+    });
+
+    it("returns low entropy for repetitive text", () => {
+      const entropy = calculateEntropy("aaaaaaaaaa");
+      expect(entropy).toBe(0);
+    });
+
+    it("returns moderate entropy for normal text", () => {
+      const entropy = calculateEntropy("Hello, this is normal text.");
+      expect(entropy).toBeGreaterThan(2);
+      expect(entropy).toBeLessThan(5);
+    });
+
+    it("returns high entropy for Base64 content", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5";
+      const entropy = calculateEntropy(base64);
+      expect(entropy).toBeGreaterThan(4.5);
+    });
+  });
+
+  describe("isLikelyBase64", () => {
+    it("returns false for short strings", () => {
+      expect(isLikelyBase64("abc123")).toBe(false);
+    });
+
+    it("returns false for normal text", () => {
+      expect(isLikelyBase64("This is normal text with spaces and punctuation!")).toBe(false);
+    });
+
+    it("returns true for Base64 encoded content", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5eyJpc3MiOiJodHRwczovL2V4YW1wbGUu";
+      expect(isLikelyBase64(base64)).toBe(true);
+    });
+
+    it("returns true for URL-safe Base64", () => {
+      const urlSafe = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5_abc-def123456";
+      expect(isLikelyBase64(urlSafe)).toBe(true);
+    });
+  });
+
+  describe("getBase64Ratio", () => {
+    it("returns 0 for normal text", () => {
+      const ratio = getBase64Ratio("This is completely normal text.");
+      expect(ratio).toBe(0);
+    });
+
+    it("returns high ratio for mostly Base64 content", () => {
+      // Use actual high-entropy Base64, not repeated chars
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5eyJpc3M".repeat(3);
+      const content = "Token: " + base64;
+      const ratio = getBase64Ratio(content);
+      expect(ratio).toBeGreaterThan(0.5);
+    });
+
+    it("returns partial ratio for mixed content", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5";
+      const content = `Normal text here. ${base64} More normal text.`;
+      const ratio = getBase64Ratio(content);
+      expect(ratio).toBeGreaterThan(0);
+      expect(ratio).toBeLessThan(0.7);
+    });
+  });
+
+  describe("hasBinaryContent", () => {
+    it("returns false for normal text", () => {
+      expect(hasBinaryContent("Normal text")).toBe(false);
+    });
+
+    it("returns false for text with newlines and tabs", () => {
+      expect(hasBinaryContent("Line 1\nLine 2\tTabbed")).toBe(false);
+    });
+
+    it("returns true for null bytes", () => {
+      expect(hasBinaryContent("Text\x00with null")).toBe(true);
+    });
+
+    it("returns true for control characters", () => {
+      expect(hasBinaryContent("Text\x03with control")).toBe(true);
+    });
+  });
+
+  describe("removeBase64Blocks", () => {
+    it("removes Base64 blocks from content", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5eyJpc3M";
+      const content = `API Token: ${base64}\n\nNext section...`;
+      const result = removeBase64Blocks(content);
+
+      expect(result).not.toContain(base64);
+      expect(result).toContain("[ENCODED]");
+      expect(result).toContain("API Token:");
+      expect(result).toContain("Next section...");
+    });
+
+    it("preserves normal text", () => {
+      const content = "This is completely normal text without any encoding.";
+      expect(removeBase64Blocks(content)).toBe(content);
+    });
+  });
+
+  describe("redactSecrets", () => {
+    it("redacts JWT tokens", () => {
+      const jwt = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U";
+      const content = `Bearer ${jwt}`;
+      const { content: redacted, secretsFound } = redactSecrets(content);
+
+      expect(redacted).toContain("[JWT_REDACTED]");
+      expect(secretsFound).toContain("jwt");
+    });
+
+    it("redacts AWS access keys", () => {
+      const content = "AWS Key: AKIAIOSFODNN7EXAMPLE";
+      const { content: redacted, secretsFound } = redactSecrets(content);
+
+      expect(redacted).toContain("[AWSACCESSKEY_REDACTED]");
+      expect(secretsFound).toContain("awsAccessKey");
+    });
+
+    it("redacts GitHub tokens", () => {
+      const content = "Token: ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
+      const { content: redacted, secretsFound } = redactSecrets(content);
+
+      expect(redacted).toContain("[GITHUBTOKEN_REDACTED]");
+      expect(secretsFound).toContain("githubToken");
+    });
+
+    it("preserves normal text", () => {
+      const content = "This is normal text without secrets.";
+      const { content: redacted, secretsFound } = redactSecrets(content);
+
+      expect(redacted).toBe(content);
+      expect(secretsFound).toHaveLength(0);
+    });
+  });
+
+  describe("filterContent", () => {
+    it("returns 'index' for clean content", () => {
+      // Content must be at least 50 chars (minContentLength default)
+      const content = "This is clean, normal content for indexing. It contains enough text to pass the minimum length requirement.";
+      const result = filterContent(content);
+
+      expect(result.action).toBe("index");
+      expect(result.cleanedContent).toBe(content);
+      expect(result.reasons).toHaveLength(0);
+    });
+
+    it("returns 'skip' for binary content", () => {
+      const result = filterContent("Text\x00with null bytes");
+
+      expect(result.action).toBe("skip");
+      expect(result.reasons).toContain("Contains binary content");
+    });
+
+    it("returns 'skip' for mostly Base64 content", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5".repeat(10);
+      const result = filterContent(base64);
+
+      expect(result.action).toBe("skip");
+      expect(result.reasons[0]).toContain("Base64 encoded");
+    });
+
+    it("returns 'filter' for mixed content with Base64", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5";
+      const content = `This is important text. Token: ${base64}. More important content here that we want to index.`;
+      const result = filterContent(content);
+
+      expect(result.action).toBe("filter");
+      expect(result.cleanedContent).toContain("[ENCODED]");
+      expect(result.cleanedContent).toContain("This is important text");
+      expect(result.reasons.some(r => r.includes("Base64"))).toBe(true);
+    });
+
+    it("returns 'skip' if content too short after filtering", () => {
+      // Short text + Base64 that will be removed, leaving less than 50 chars
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5eyJpc3M";
+      const content = `Hi ${base64}`;
+      const result = filterContent(content);
+
+      expect(result.action).toBe("skip");
+      // After removing Base64, only "Hi [ENCODED]" remains which is too short
+    });
+
+    it("respects custom configuration", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5eyJpc3M";
+      // Need enough remaining content after potential filtering
+      const content = `This is some text before the token. Token: ${base64}. And this is some text after the token that should remain.`;
+
+      // With removeBase64 disabled, the Base64 should stay
+      const result = filterContent(content, { removeBase64: false });
+
+      expect(result.action).not.toBe("skip");
+      expect(result.cleanedContent).toContain(base64);
+    });
+  });
+
+  describe("shouldIndexContent", () => {
+    it("returns true for normal content", () => {
+      expect(shouldIndexContent("Normal text content")).toBe(true);
+    });
+
+    it("returns false for binary content", () => {
+      expect(shouldIndexContent("Binary\x00content")).toBe(false);
+    });
+
+    it("returns false for mostly Base64", () => {
+      const base64 = "ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKU1V6STFOaUo5".repeat(10);
+      expect(shouldIndexContent(base64)).toBe(false);
+    });
+  });
+});