@sarjallab09/figma-intelligence 1.0.0

package/scripts/convert-hub-pdfs.js

@@ -0,0 +1,425 @@
+#!/usr/bin/env node
+/**
+ * convert-hub-pdfs.js — One-time converter: PDF → chunked JSON
+ *
+ * Reads each PDF in knowledge-hub/, splits into chapter-sized chunks,
+ * and outputs .chunks.json files for instant loading by the bridge relay.
+ *
+ * Usage: node scripts/convert-hub-pdfs.js
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+const HUB_DIR = path.resolve(__dirname, "..", "figma-bridge-plugin", "knowledge-hub");
+
+// Lazy-load pdf-parse from the bridge plugin's node_modules
+let pdfParse;
+function getPdfParse() {
+  if (!pdfParse) {
+    const bridgeModules = path.resolve(__dirname, "..", "figma-bridge-plugin", "node_modules", "pdf-parse");
+    pdfParse = require(bridgeModules);
+  }
+  return pdfParse;
+}
+
+// ── Text Cleaning ──────────────────────────────────────────────────────────
+
+/**
+ * Clean OCR artifacts and PDF extraction noise from raw text.
+ */
+function cleanExtractedText(text) {
+  return text
+    // Remove page markers like "-- 1 of 320 --"
+    .replace(/\n-- \d+ of \d+ --\n/g, "\n\n")
+    // Normalize line endings
+    .replace(/\r\n/g, "\n")
+    // Remove standalone page numbers (line that's just a number)
+    .replace(/^\s*\d{1,4}\s*$/gm, "")
+    // Remove OCR garbage: sequences of random mixed-case letters/numbers that aren't real words
+    // e.g. "12ChapterC 20Ct.Ipnoddu" → removed
+    .replace(/\b\d+[A-Z][a-z]+[A-Z]\s+\d+[A-Z][a-z]+\.\w+\b/g, "")
+    // Remove repeated header/footer text (same line appearing every ~50 lines)
+    .replace(/^(.{10,80})\n(?:.*\n){20,80}\1$/gm, (match, header) => {
+      return match.replace(new RegExp("^" + header.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + "$", "gm"), "");
+    })
+    // Remove tab-heavy lines (likely table-of-contents remnants)
+    .replace(/^[^\n]*\t{2,}[^\n]*$/gm, "")
+    // Clean up excessive whitespace
+    .replace(/\n{4,}/g, "\n\n\n")
+    // Remove leading/trailing whitespace per line
+    .replace(/^[ \t]+|[ \t]+$/gm, (m) => m.length > 8 ? "" : m)
+    .trim();
+}
+
+/**
+ * Additional cleaning for individual chunk text.
+ */
+function cleanChunkText(text) {
+  return text
+    // Remove lines that are just numbers (page numbers caught inside chunks)
+    .replace(/^\s*\d{1,4}\s*$/gm, "")
+    // Remove very short lines that are likely headers/footers (< 5 chars, not part of content)
+    .replace(/^.{1,3}$/gm, "")
+    // Collapse multiple blank lines
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+// ── Heading Detection ──────────────────────────────────────────────────────
+
+/**
+ * Detect chapter/section headings in extracted PDF text.
+ * Returns an array of { heading, startIndex }.
+ */
+function detectHeadings(text) {
+  const lines = text.split("\n");
+  const headings = [];
+  let charPos = 0;
+
+  // First pass: detect TOC-like blocks and skip them
+  // TOC lines typically have dots or tabs between title and page number
+  const tocLinePattern = /^.{5,60}\s*\.{3,}\s*\d+\s*$/;
+  const tocRegions = new Set();
+  for (let i = 0; i < lines.length; i++) {
+    if (tocLinePattern.test(lines[i].trim())) {
+      // Mark a window around this line as TOC
+      for (let j = Math.max(0, i - 2); j < Math.min(lines.length, i + 3); j++) {
+        tocRegions.add(j);
+      }
+    }
+  }
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    const nextLine = (lines[i + 1] || "").trim();
+    const prevLine = i > 0 ? lines[i - 1].trim() : "";
+
+    // Skip TOC regions
+    if (tocRegions.has(i)) {
+      charPos += lines[i].length + 1;
+      continue;
+    }
+
+    // Skip lines with tabs (likely TOC entries) like "Chapter 07. \tAppendix \t142"
+    if (/\t/.test(line)) {
+      charPos += lines[i].length + 1;
+      continue;
+    }
+
+    // Skip very short or empty lines
+    if (line.length < 4) {
+      charPos += lines[i].length + 1;
+      continue;
+    }
+
+    // Pattern 1: "Chapter N" or "CHAPTER N" or "Part N" (without page numbers or tabs)
+    if (/^(chapter|part)\s+\d+[.:)]\s*/i.test(line) && line.length < 120 && !/\d+$/.test(line.replace(/^.*\d+[.:)]\s*/, ""))) {
+      headings.push({ heading: line, startIndex: charPos });
+    }
+    // Pattern 2: Numbered heading like "1. Introduction" or "1.1 Overview"
+    else if (/^\d+(\.\d+)?\s+[A-Z]/.test(line) && line.length < 120 && line.length > 5) {
+      // Exclude lines that end with just a page number (TOC entries)
+      if (!/\s+\d{1,3}\s*$/.test(line)) {
+        headings.push({ heading: line, startIndex: charPos });
+      }
+    }
+    // Pattern 3: ALL CAPS line (likely a section heading), short and followed by content
+    else if (
+      line.length > 3 &&
+      line.length < 80 &&
+      line === line.toUpperCase() &&
+      /[A-Z]{3,}/.test(line) &&
+      !/^\d+$/.test(line) && // Not just numbers
+      nextLine.length > 30
+    ) {
+      headings.push({ heading: line, startIndex: charPos });
+    }
+    // Pattern 4: Title-case line that's short, preceded by blank line, followed by content
+    else if (
+      line.length > 5 &&
+      line.length < 100 &&
+      /^[A-Z][a-z]/.test(line) &&
+      !line.endsWith(".") &&
+      !line.endsWith(",") &&
+      nextLine.length > 50 &&
+      prevLine === ""
+    ) {
+      headings.push({ heading: line, startIndex: charPos });
+    }
+
+    charPos += lines[i].length + 1;
+  }
+
+  return headings;
+}
+
+// ── Chunking ───────────────────────────────────────────────────────────────
+
+/**
+ * Split text into chunks based on detected headings.
+ * Falls back to paragraph-based splitting if no headings found.
+ */
+function chunkText(text, headings, targetChunkSize = 2000) {
+  const chunks = [];
+
+  if (headings.length > 0) {
+    // Handle text before first heading
+    if (headings[0].startIndex > 200) {
+      const preface = text.slice(0, headings[0].startIndex).trim();
+      if (preface.length > 100) {
+        chunks.push({
+          heading: "Introduction / Preface",
+          text: cleanChunkText(preface),
+        });
+      }
+    }
+
+    // Split by headings
+    for (let i = 0; i < headings.length; i++) {
+      const start = headings[i].startIndex;
+      const end = i + 1 < headings.length ? headings[i + 1].startIndex : text.length;
+      const raw = text.slice(start, end).trim();
+      const cleaned = cleanChunkText(raw);
+
+      if (cleaned.length < 80) continue;
+
+      if (cleaned.length > targetChunkSize * 2) {
+        const subChunks = splitByParagraphs(cleaned, targetChunkSize, headings[i].heading);
+        chunks.push(...subChunks);
+      } else {
+        chunks.push({ heading: headings[i].heading, text: cleaned });
+      }
+    }
+  } else {
+    const subChunks = splitByParagraphs(text, targetChunkSize, "Section");
+    chunks.push(...subChunks);
+  }
+
+  return chunks;
+}
+
+/**
+ * Split a block of text into chunks by paragraph boundaries.
+ */
+function splitByParagraphs(text, targetSize, parentHeading) {
+  const paragraphs = text.split(/\n{2,}/).filter((p) => p.trim().length > 30);
+  const chunks = [];
+  let currentText = "";
+  let chunkIndex = 1;
+
+  for (const para of paragraphs) {
+    if (currentText.length + para.length > targetSize && currentText.length > 200) {
+      chunks.push({
+        heading: `${parentHeading} (part ${chunkIndex})`,
+        text: cleanChunkText(currentText),
+      });
+      currentText = "";
+      chunkIndex++;
+    }
+    currentText += para + "\n\n";
+  }
+
+  if (currentText.trim().length > 80) {
+    chunks.push({
+      heading: chunks.length > 0 ? `${parentHeading} (part ${chunkIndex})` : parentHeading,
+      text: cleanChunkText(currentText),
+    });
+  }
+
+  return chunks;
+}
+
+// ── Keyword Extraction ─────────────────────────────────────────────────────
+
+const STOP_WORDS = new Set([
+  "a", "an", "the", "is", "are", "was", "were", "be", "been", "being",
+  "have", "has", "had", "do", "does", "did", "will", "would", "could",
+  "should", "may", "might", "shall", "can", "need", "must",
+  "i", "me", "my", "we", "our", "you", "your", "he", "she", "it",
+  "they", "them", "their", "this", "that", "these", "those",
+  "what", "which", "who", "whom", "how", "when", "where", "why",
+  "and", "or", "but", "not", "no", "nor", "so", "if", "then",
+  "of", "in", "on", "at", "to", "for", "with", "by", "from",
+  "about", "into", "through", "during", "before", "after",
+  "above", "below", "between", "under", "over",
+  "just", "also", "very", "too", "more", "most", "some", "any",
+  "all", "each", "every", "both", "few", "many", "much",
+  "such", "only", "other", "new", "like", "than", "even",
+  "one", "two", "three", "first", "second", "well", "way",
+  "use", "used", "using", "make", "made", "see", "get", "take",
+  "work", "part", "page", "chapter", "figure", "table",
+]);
+
+/**
+ * Extract top keywords from a chunk, including bigrams for compound concepts.
+ * Filters OCR garbage (non-dictionary-like words).
+ */
+function extractChunkKeywords(text, maxKeywords = 15) {
+  const words = text
+    .toLowerCase()
+    .replace(/[^\w\s-]/g, " ")
+    .split(/\s+/)
+    .filter((w) => {
+      if (w.length < 3) return false;
+      if (STOP_WORDS.has(w)) return false;
+      // Filter OCR garbage: words with unusual character patterns
+      // Real words are mostly lowercase letters; OCR produces mixed case/numbers
+      if (/\d/.test(w) && /[a-z]/i.test(w) && w.length > 6) return false; // "12chapterc"
+      if (/[A-Z].*[a-z].*[A-Z]/.test(w)) return false; // "ChapterC"
+      return true;
+    });
+
+  // Count word frequency
+  const freq = {};
+  for (const w of words) {
+    freq[w] = (freq[w] || 0) + 1;
+  }
+
+  // Extract bigrams for compound concepts (e.g., "design system", "user experience")
+  const bigramFreq = {};
+  for (let i = 0; i < words.length - 1; i++) {
+    const bigram = words[i] + " " + words[i + 1];
+    // Only keep bigrams where both words appear frequently
+    if ((freq[words[i]] || 0) >= 2 && (freq[words[i + 1]] || 0) >= 2) {
+      bigramFreq[bigram] = (bigramFreq[bigram] || 0) + 1;
+    }
+  }
+
+  // Combine: prefer bigrams (more specific), then unigrams
+  const allTerms = [
+    ...Object.entries(bigramFreq).filter(([, c]) => c >= 2).map(([term, count]) => ({ term, score: count * 3 })),
+    ...Object.entries(freq).map(([term, count]) => ({ term, score: count })),
+  ];
+
+  // Sort by score, deduplicate
+  allTerms.sort((a, b) => b.score - a.score);
+  const seen = new Set();
+  const keywords = [];
+  for (const { term } of allTerms) {
+    if (keywords.length >= maxKeywords) break;
+    if (seen.has(term)) continue;
+    seen.add(term);
+    // Don't add unigram if it's part of an already-added bigram
+    if (!term.includes(" ") && keywords.some((k) => k.includes(" ") && k.includes(term))) continue;
+    keywords.push(term);
+  }
+
+  return keywords;
+}
+
+// ── PDF Conversion ─────────────────────────────────────────────────────────
+
+/**
+ * Convert a single PDF to chunked JSON.
+ */
+async function convertPdf(filePath) {
+  const fileName = path.basename(filePath);
+  const { PDFParse } = getPdfParse();
+
+  console.log(`  Parsing ${fileName}...`);
+  const buffer = fs.readFileSync(filePath);
+  const uint8 = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+  const parser = new PDFParse(uint8);
+  const result = await parser.getText();
+
+  const rawText = result.text || result.pages.map((p) => p.text).join("\n\n");
+  const text = cleanExtractedText(rawText);
+
+  let pdfTitle = "";
+  try {
+    const info = await parser.getInfo();
+    pdfTitle = info?.Title || info?.title || "";
+  } catch {}
+
+  // Clean up title if it looks like garbage
+  if (pdfTitle && (/[^\x20-\x7E]/.test(pdfTitle) || pdfTitle.length > 150)) {
+    pdfTitle = "";
+  }
+
+  const numPages = result.total || result.pages?.length || 0;
+  console.log(`     ${text.length.toLocaleString()} chars, ${numPages} pages`);
+
+  const headings = detectHeadings(text);
+  console.log(`     ${headings.length} headings detected`);
+
+  const chunks = chunkText(text, headings);
+  console.log(`     ${chunks.length} chunks created`);
+
+  // Build output
+  const output = {
+    title: pdfTitle || fileName.replace(/\.pdf$/i, "").replace(/[-_]/g, " "),
+    source: fileName,
+    pages: numPages,
+    totalChars: text.length,
+    convertedAt: new Date().toISOString(),
+    chunks: chunks.map((chunk, i) => ({
+      id: `ch-${String(i + 1).padStart(3, "0")}`,
+      heading: chunk.heading,
+      text: chunk.text,
+      chars: chunk.text.length,
+      keywords: extractChunkKeywords(chunk.text),
+    })),
+  };
+
+  // Write output
+  const slug = fileName
+    .replace(/\.pdf$/i, "")
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-|-$/g, "");
+  const outPath = path.join(HUB_DIR, `${slug}.chunks.json`);
+  fs.writeFileSync(outPath, JSON.stringify(output, null, 2));
+
+  const outSize = fs.statSync(outPath).size;
+  console.log(
+    `     Wrote ${outPath.split("/").pop()} (${(outSize / 1024).toFixed(0)}KB, ${output.chunks.length} chunks)`
+  );
+
+  return { fileName, outPath, chunks: output.chunks.length };
+}
+
+// ── Main ───────────────────────────────────────────────────────────────────
+
+async function main() {
+  console.log("Converting Knowledge Hub PDFs to chunked JSON...\n");
+
+  if (!fs.existsSync(HUB_DIR)) {
+    console.error(`Knowledge hub directory not found: ${HUB_DIR}`);
+    process.exit(1);
+  }
+
+  const pdfFiles = fs
+    .readdirSync(HUB_DIR)
+    .filter((f) => f.toLowerCase().endsWith(".pdf"));
+
+  if (pdfFiles.length === 0) {
+    console.log("No PDF files found in knowledge-hub/. Nothing to convert.");
+    process.exit(0);
+  }
+
+  console.log(`Found ${pdfFiles.length} PDF file(s):\n`);
+
+  const results = [];
+  for (const pdf of pdfFiles) {
+    try {
+      const result = await convertPdf(path.join(HUB_DIR, pdf));
+      results.push(result);
+    } catch (err) {
+      console.error(`  Failed to convert ${pdf}: ${err.message}`);
+    }
+    console.log("");
+  }
+
+  console.log("-".repeat(50));
+  console.log(`Converted ${results.length}/${pdfFiles.length} PDFs\n`);
+
+  const totalChunks = results.reduce((s, r) => s + r.chunks, 0);
+  console.log(`   Total chunks: ${totalChunks}`);
+  console.log(`   Chunk files:  ${results.map((r) => path.basename(r.outPath)).join(", ")}`);
+}
+
+main().catch((err) => {
+  console.error("Fatal:", err.message);
+  process.exit(1);
+});