@twelvehart/supermemory-runtime 1.0.0-next.0

package/src/services/extractors/pdf.extractor.ts

@@ -0,0 +1,315 @@
+/**
+ * PDF extractor - extracts text content from PDF files
+ * Uses pdf-parse library for extraction
+ */
+
+import { ExtractionResult, ExtractorInterface, ContentType } from '../../types/document.types.js'
+import { DependencyError } from '../../utils/errors.js'
+
+// pdf-parse types (library doesn't have proper types)
+interface PdfData {
+  numpages: number
+  numrender: number
+  info: {
+    Title?: string
+    Author?: string
+    Subject?: string
+    Keywords?: string
+    Creator?: string
+    Producer?: string
+    CreationDate?: string
+    ModDate?: string
+  }
+  metadata: unknown
+  text: string
+  version: string
+}
+
+// Type for text content item from pdf.js
+interface TextItem {
+  str: string
+  dir?: string
+  width?: number
+  height?: number
+  transform?: number[]
+  fontName?: string
+}
+
+// Type for text content from getTextContent()
+interface TextContent {
+  items: TextItem[]
+  styles?: Record<string, unknown>
+}
+
+// Type for page data from pdf-parse pagerender callback
+interface PageData {
+  getTextContent(): Promise<TextContent>
+  pageNumber?: number
+}
+
+interface PdfParseOptions {
+  pagerender?: (pageData: unknown) => string
+  max?: number
+  version?: string
+}
+
+type PdfParseFunction = (buffer: Buffer, options?: PdfParseOptions) => Promise<PdfData>
+
+// Dynamic import for pdf-parse
+let pdfParse: PdfParseFunction | null = null
+
+async function loadPdfParse(): Promise<PdfParseFunction> {
+  if (!pdfParse) {
+    try {
+      const module = await import('pdf-parse')
+      pdfParse = module.default as PdfParseFunction
+    } catch {
+      throw new DependencyError('pdf-parse', 'npm install pdf-parse')
+    }
+  }
+  return pdfParse
+}
+
+export class PdfExtractor implements ExtractorInterface {
+  /**
+   * Check if content is a PDF buffer or base64 encoded PDF
+   */
+  canHandle(content: string | Buffer): boolean {
+    if (Buffer.isBuffer(content)) {
+      // Check PDF magic bytes: %PDF
+      return content.slice(0, 4).toString() === '%PDF'
+    }
+
+    if (typeof content === 'string') {
+      // Check if it's base64 encoded PDF
+      if (content.startsWith('data:application/pdf;base64,')) {
+        return true
+      }
+
+      // Check if it starts with PDF magic bytes
+      if (content.startsWith('%PDF')) {
+        return true
+      }
+
+      // Try to detect base64 PDF without data URI prefix
+      try {
+        const decoded = Buffer.from(content.slice(0, 100), 'base64').toString()
+        return decoded.startsWith('%PDF')
+      } catch {
+        return false
+      }
+    }
+
+    return false
+  }
+
+  /**
+   * Extract text content from PDF
+   */
+  async extract(content: string | Buffer, options?: Record<string, unknown>): Promise<ExtractionResult> {
+    const parse = await loadPdfParse()
+
+    const buffer = this.toBuffer(content)
+    const pdfData = await parse(buffer, {
+      max: options?.maxPages as number | undefined,
+    })
+
+    const cleanedText = this.cleanPdfText(pdfData.text)
+    const metadata = this.extractMetadata(pdfData, cleanedText)
+
+    return {
+      content: cleanedText,
+      contentType: 'pdf' as ContentType,
+      metadata,
+      rawContent: pdfData.text,
+    }
+  }
+
+  /**
+   * Convert input to Buffer
+   */
+  private toBuffer(content: string | Buffer): Buffer {
+    if (Buffer.isBuffer(content)) {
+      return content
+    }
+
+    // Handle data URI
+    if (content.startsWith('data:application/pdf;base64,')) {
+      const base64Data = content.replace('data:application/pdf;base64,', '')
+      return Buffer.from(base64Data, 'base64')
+    }
+
+    // Try base64 decode
+    try {
+      const buffer = Buffer.from(content, 'base64')
+      if (buffer.slice(0, 4).toString() === '%PDF') {
+        return buffer
+      }
+    } catch {
+      // Not base64
+    }
+
+    // Assume raw PDF string
+    return Buffer.from(content, 'binary')
+  }
+
+  /**
+   * Clean extracted PDF text
+   */
+  private cleanPdfText(text: string): string {
+    return (
+      text
+        // Fix common PDF extraction artifacts
+        .replace(/\f/g, '\n\n') // Form feeds to paragraph breaks
+        .replace(/\r\n/g, '\n')
+        .replace(/\r/g, '\n')
+        // Remove excessive whitespace
+        .replace(/[ \t]+/g, ' ')
+        .replace(/\n{3,}/g, '\n\n')
+        // Fix hyphenation at line breaks
+        .replace(/(\w)-\n(\w)/g, '$1$2')
+        // Remove page numbers (common patterns)
+        .replace(/^\s*\d+\s*$/gm, '')
+        .replace(/\n\s*Page\s+\d+\s*\n/gi, '\n')
+        // Trim lines
+        .split('\n')
+        .map((line) => line.trim())
+        .join('\n')
+        .trim()
+    )
+  }
+
+  /**
+   * Extract metadata from PDF data
+   */
+  private extractMetadata(pdfData: PdfData, cleanedText: string): ExtractionResult['metadata'] {
+    const words = cleanedText.split(/\s+/).filter((w) => w.length > 0)
+    const info = pdfData.info ?? {}
+
+    // Parse creation date if available
+    let createdAt: string | undefined
+    if (info.CreationDate) {
+      createdAt = this.parsePdfDate(info.CreationDate)
+    }
+
+    // Parse keywords into tags
+    let tags: string[] | undefined
+    if (info.Keywords) {
+      tags = info.Keywords.split(/[,;]/)
+        .map((k) => k.trim())
+        .filter((k) => k.length > 0)
+    }
+
+    return {
+      title: info.Title,
+      author: info.Author,
+      description: info.Subject,
+      tags,
+      source: 'pdf',
+      mimeType: 'application/pdf',
+      wordCount: words.length,
+      charCount: cleanedText.length,
+      pageCount: pdfData.numpages,
+      pdfVersion: pdfData.version,
+      creator: info.Creator,
+      producer: info.Producer,
+      createdAt,
+    }
+  }
+
+  /**
+   * Parse PDF date format (D:YYYYMMDDHHmmSS)
+   */
+  private parsePdfDate(dateStr: string): string | undefined {
+    try {
+      // Remove D: prefix if present
+      const clean = dateStr.replace(/^D:/, '')
+
+      // Extract date components
+      const year = clean.slice(0, 4)
+      const month = clean.slice(4, 6) || '01'
+      const day = clean.slice(6, 8) || '01'
+      const hour = clean.slice(8, 10) || '00'
+      const minute = clean.slice(10, 12) || '00'
+      const second = clean.slice(12, 14) || '00'
+
+      const date = new Date(`${year}-${month}-${day}T${hour}:${minute}:${second}Z`)
+
+      return date.toISOString()
+    } catch {
+      return undefined
+    }
+  }
+
+  /**
+   * Extract text from specific pages using pageData.getTextContent()
+   */
+  async extractPages(content: string | Buffer, startPage: number, endPage?: number): Promise<string[]> {
+    const parse = await loadPdfParse()
+
+    const buffer = this.toBuffer(content)
+    const pages: string[] = []
+    let currentPage = 0
+
+    // Custom page render function that extracts actual text content
+    const pageRender = async (pageData: PageData): Promise<string> => {
+      currentPage++
+
+      // Skip pages outside the requested range
+      if (currentPage < startPage || (endPage && currentPage > endPage)) {
+        return ''
+      }
+
+      try {
+        // Use getTextContent to extract actual text from the page
+        const textContent = await pageData.getTextContent()
+
+        if (!textContent || !textContent.items) {
+          pages.push('')
+          return ''
+        }
+
+        // Combine all text items into a single string
+        const pageText = textContent.items
+          .map((item: TextItem) => {
+            // Handle text items - they have a 'str' property
+            if ('str' in item && typeof item.str === 'string') {
+              return item.str
+            }
+            return ''
+          })
+          .join('')
+          .trim()
+
+        pages.push(this.cleanPdfText(pageText))
+        return pageText
+      } catch (error) {
+        // If getTextContent fails, add empty string for this page
+        console.warn(`Failed to extract text from page ${currentPage}:`, error)
+        pages.push('')
+        return ''
+      }
+    }
+
+    await parse(buffer, {
+      pagerender: pageRender as unknown as (pageData: unknown) => string,
+    })
+
+    return pages
+  }
+
+  /**
+   * Extract text from all pages with page boundaries preserved
+   */
+  async extractAllPages(content: string | Buffer): Promise<string[]> {
+    const parse = await loadPdfParse()
+    const buffer = this.toBuffer(content)
+
+    // First pass to get page count
+    const pdfData = await parse(buffer)
+    const totalPages = pdfData.numpages
+
+    // Extract each page
+    return this.extractPages(content, 1, totalPages)
+  }
+}

package/src/services/extractors/text.extractor.ts

@@ -0,0 +1,118 @@
+/**
+ * Plain text extractor - handles raw text content
+ */
+
+import { ExtractionResult, ExtractorInterface, ContentType } from '../../types/document.types.js'
+
+export class TextExtractor implements ExtractorInterface {
+  /**
+   * Check if this extractor can handle the content
+   */
+  canHandle(content: string): boolean {
+    // Text extractor is the fallback - it can handle anything
+    return typeof content === 'string' && content.length > 0
+  }
+
+  /**
+   * Extract text content with basic cleaning and metadata
+   */
+  async extract(content: string, options?: Record<string, unknown>): Promise<ExtractionResult> {
+    const cleanedContent = this.cleanText(content)
+    const metadata = this.extractMetadata(cleanedContent, options)
+
+    return {
+      content: cleanedContent,
+      contentType: 'text' as ContentType,
+      metadata,
+      rawContent: content,
+    }
+  }
+
+  /**
+   * Clean and normalize text content
+   */
+  private cleanText(text: string): string {
+    return (
+      text
+        // Normalize line endings
+        .replace(/\r\n/g, '\n')
+        .replace(/\r/g, '\n')
+        // Remove excessive whitespace while preserving paragraph breaks
+        .replace(/[ \t]+/g, ' ')
+        .replace(/\n{3,}/g, '\n\n')
+        // Trim each line
+        .split('\n')
+        .map((line) => line.trim())
+        .join('\n')
+        // Final trim
+        .trim()
+    )
+  }
+
+  /**
+   * Extract metadata from text content
+   */
+  private extractMetadata(content: string, options?: Record<string, unknown>): ExtractionResult['metadata'] {
+    const words = content.split(/\s+/).filter((w) => w.length > 0)
+    const lines = content.split('\n')
+
+    // Try to extract title from first line if it looks like a title
+    let title: string | undefined
+    if (lines.length > 0) {
+      const firstLine = lines[0]?.trim() ?? ''
+      // Use first line as title if it's non-empty and reasonably short
+      if (firstLine.length > 0 && firstLine.length < 200) {
+        title = firstLine
+      }
+    }
+
+    const metadataExtra = (options?.metadata as Record<string, unknown>) ?? {}
+
+    return {
+      title,
+      wordCount: words.length,
+      charCount: content.length,
+      lineCount: lines.length,
+      source: 'text',
+      ...metadataExtra,
+    }
+  }
+
+  /**
+   * Split text into sentences for more granular processing
+   */
+  splitIntoSentences(text: string): string[] {
+    // Simple sentence splitting - handles common cases
+    const sentenceEnders = /([.!?]+)\s+/g
+    const sentences: string[] = []
+    let lastIndex = 0
+    let match: RegExpExecArray | null
+
+    while ((match = sentenceEnders.exec(text)) !== null) {
+      const matchGroup = match[1] ?? ''
+      const sentence = text.slice(lastIndex, match.index + matchGroup.length).trim()
+      if (sentence.length > 0) {
+        sentences.push(sentence)
+      }
+      lastIndex = match.index + match[0].length
+    }
+
+    // Add remaining text as last sentence
+    const remaining = text.slice(lastIndex).trim()
+    if (remaining.length > 0) {
+      sentences.push(remaining)
+    }
+
+    return sentences
+  }
+
+  /**
+   * Split text into paragraphs
+   */
+  splitIntoParagraphs(text: string): string[] {
+    return text
+      .split(/\n\n+/)
+      .map((p) => p.trim())
+      .filter((p) => p.length > 0)
+  }
+}

package/src/services/extractors/url.extractor.ts

@@ -0,0 +1,243 @@
+/**
+ * URL extractor - fetches and cleans web page content
+ */
+
+import { ExtractionResult, ExtractorInterface, ContentType } from '../../types/document.types.js'
+import { ExternalServiceError } from '../../utils/errors.js'
+
+interface FetchOptions {
+  timeout?: number
+  userAgent?: string
+  followRedirects?: boolean
+}
+
+export class UrlExtractor implements ExtractorInterface {
+  private readonly defaultTimeout = 30000
+  private readonly defaultUserAgent = 'Mozilla/5.0 (compatible; SupermemoryBot/1.0)'
+
+  /**
+   * Check if content is a valid URL
+   */
+  canHandle(content: string): boolean {
+    try {
+      const trimmed = content.trim()
+      const url = new URL(trimmed)
+      return url.protocol === 'http:' || url.protocol === 'https:'
+    } catch {
+      return false
+    }
+  }
+
+  /**
+   * Fetch URL and extract clean content
+   */
+  async extract(url: string, options?: FetchOptions & Record<string, unknown>): Promise<ExtractionResult> {
+    const trimmedUrl = url.trim()
+    const html = await this.fetchUrl(trimmedUrl, options)
+    const { content, metadata } = this.parseHtml(html, trimmedUrl)
+
+    return {
+      content,
+      contentType: 'url' as ContentType,
+      metadata: {
+        ...metadata,
+        sourceUrl: trimmedUrl,
+        source: 'web',
+      },
+      rawContent: html,
+    }
+  }
+
+  /**
+   * Fetch URL content
+   */
+  private async fetchUrl(url: string, options?: FetchOptions): Promise<string> {
+    const timeout = options?.timeout ?? this.defaultTimeout
+    const controller = new AbortController()
+    const timeoutId = setTimeout(() => controller.abort(), timeout)
+
+    try {
+      const response = await fetch(url, {
+        headers: {
+          'User-Agent': options?.userAgent ?? this.defaultUserAgent,
+          Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+          'Accept-Language': 'en-US,en;q=0.5',
+        },
+        redirect: options?.followRedirects !== false ? 'follow' : 'manual',
+        signal: controller.signal,
+      })
+
+      if (!response.ok) {
+        throw new ExternalServiceError('HTTP', `HTTP ${response.status}: ${response.statusText}`, response.status, {
+          url,
+        })
+      }
+
+      return await response.text()
+    } finally {
+      clearTimeout(timeoutId)
+    }
+  }
+
+  /**
+   * Parse HTML and extract clean text content
+   */
+  private parseHtml(html: string, url: string): { content: string; metadata: ExtractionResult['metadata'] } {
+    // Extract title
+    const titleMatch = html.match(/<title[^>]*>([^<]*)<\/title>/i)
+    const title = titleMatch?.[1] ? this.decodeHtmlEntities(titleMatch[1].trim()) : undefined
+
+    // Extract meta description (handle both attribute orders)
+    const descMatch =
+      html.match(/<meta[^>]*name=["']description["'][^>]*content=["']([^"']*)["']/i) ??
+      html.match(/<meta[^>]*content=["']([^"']*)["'][^>]*name=["']description["']/i)
+    const description = descMatch?.[1] ? this.decodeHtmlEntities(descMatch[1].trim()) : undefined
+
+    // Extract author (handle both attribute orders)
+    const authorMatch =
+      html.match(/<meta[^>]*name=["']author["'][^>]*content=["']([^"']*)["']/i) ??
+      html.match(/<meta[^>]*content=["']([^"']*)["'][^>]*name=["']author["']/i)
+    const author = authorMatch?.[1] ? this.decodeHtmlEntities(authorMatch[1].trim()) : undefined
+
+    // Extract og:tags for additional metadata
+    const ogTags = this.extractOpenGraphTags(html)
+
+    // Clean HTML to get text content
+    const content = this.htmlToText(html)
+    const words = content.split(/\s+/).filter((w) => w.length > 0)
+
+    let domain: string | undefined
+    try {
+      domain = new URL(url).hostname
+    } catch {
+      // URL parsing failed, leave domain undefined
+    }
+
+    return {
+      content,
+      metadata: {
+        title: title ?? ogTags['title'],
+        description: description ?? ogTags['description'],
+        author,
+        wordCount: words.length,
+        charCount: content.length,
+        mimeType: 'text/html',
+        ogImage: ogTags['image'],
+        ogType: ogTags['type'],
+        domain,
+      },
+    }
+  }
+
+  /**
+   * Extract OpenGraph meta tags
+   */
+  private extractOpenGraphTags(html: string): Record<string, string | undefined> {
+    const tags: Record<string, string | undefined> = {}
+    const ogPattern = /<meta[^>]*property=["']og:([^"']*)["'][^>]*content=["']([^"']*)["']/gi
+    let match: RegExpExecArray | null
+
+    while ((match = ogPattern.exec(html)) !== null) {
+      const key = match[1]
+      const value = match[2]
+      if (key && value) {
+        tags[key] = this.decodeHtmlEntities(value)
+      }
+    }
+
+    return tags
+  }
+
+  /**
+   * Convert HTML to clean text
+   */
+  private htmlToText(html: string): string {
+    let text = html
+
+    // Remove script and style content
+    text = text.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+    text = text.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+    text = text.replace(/<noscript[^>]*>[\s\S]*?<\/noscript>/gi, '')
+
+    // Remove comments
+    text = text.replace(/<!--[\s\S]*?-->/g, '')
+
+    // Remove header, footer, nav, aside (common non-content areas)
+    text = text.replace(/<header[^>]*>[\s\S]*?<\/header>/gi, '')
+    text = text.replace(/<footer[^>]*>[\s\S]*?<\/footer>/gi, '')
+    text = text.replace(/<nav[^>]*>[\s\S]*?<\/nav>/gi, '')
+    text = text.replace(/<aside[^>]*>[\s\S]*?<\/aside>/gi, '')
+
+    // Convert block elements to newlines
+    text = text.replace(/<\/(p|div|h[1-6]|li|tr|br|hr)[^>]*>/gi, '\n')
+    text = text.replace(/<(br|hr)[^>]*\/?>/gi, '\n')
+
+    // Remove all remaining HTML tags
+    text = text.replace(/<[^>]+>/g, ' ')
+
+    // Decode HTML entities
+    text = this.decodeHtmlEntities(text)
+
+    // Clean up whitespace
+    text = text
+      .replace(/[ \t]+/g, ' ')
+      .replace(/\n[ \t]+/g, '\n')
+      .replace(/[ \t]+\n/g, '\n')
+      .replace(/\n{3,}/g, '\n\n')
+      .trim()
+
+    return text
+  }
+
+  /**
+   * Decode common HTML entities
+   */
+  private decodeHtmlEntities(text: string): string {
+    const entities: Record<string, string> = {
+      '&amp;': '&',
+      '&lt;': '<',
+      '&gt;': '>',
+      '&quot;': '"',
+      '&#39;': "'",
+      '&apos;': "'",
+      '&nbsp;': ' ',
+      '&mdash;': '--',
+      '&ndash;': '-',
+      '&hellip;': '...',
+      '&copy;': '(c)',
+      '&reg;': '(R)',
+      '&trade;': '(TM)',
+    }
+
+    let result = text
+    for (const [entity, char] of Object.entries(entities)) {
+      result = result.replace(new RegExp(entity, 'gi'), char)
+    }
+
+    // Handle numeric entities
+    result = result.replace(/&#(\d+);/g, (_, code: string) => String.fromCharCode(parseInt(code, 10)))
+    result = result.replace(/&#x([a-fA-F0-9]+);/g, (_, code: string) => String.fromCharCode(parseInt(code, 16)))
+
+    return result
+  }
+
+  /**
+   * Check if URL is accessible
+   */
+  async isAccessible(url: string): Promise<boolean> {
+    try {
+      const controller = new AbortController()
+      const timeoutId = setTimeout(() => controller.abort(), 5000)
+
+      const response = await fetch(url, {
+        method: 'HEAD',
+        signal: controller.signal,
+      })
+
+      clearTimeout(timeoutId)
+      return response.ok
+    } catch {
+      return false
+    }
+  }
+}