@operor/knowledge 0.1.0

package/src/index.ts

@@ -0,0 +1,41 @@
+// Types
+export type {
+  KBDocument,
+  KBChunk,
+  KBSearchResult,
+  KBSearchOptions,
+  KBStats,
+  KnowledgeStore,
+} from './types.js';
+
+// Core
+export { EmbeddingService } from './EmbeddingService.js';
+export type { EmbeddingServiceConfig } from './EmbeddingService.js';
+export { SQLiteKnowledgeStore } from './SQLiteKnowledgeStore.js';
+export { TextChunker } from './TextChunker.js';
+export type { ChunkOptions } from './TextChunker.js';
+
+// Pipelines
+export { IngestionPipeline } from './IngestionPipeline.js';
+export type { IngestInput, IngestFaqResult, IngestFaqOptions, RebuildResult, ContentReformatter } from './IngestionPipeline.js';
+export { RetrievalPipeline, splitCompoundQuery } from './RetrievalPipeline.js';
+export type { RetrievalResult, RetrievalPipelineOptions } from './RetrievalPipeline.js';
+
+// Query processing
+export { normalizeQuery } from './QueryNormalizer.js';
+export { reciprocalRankFusion, weightedScoreFusion } from './RankFusion.js';
+export { QueryRewriter } from './QueryRewriter.js';
+export type { QueryRewriterOptions, RewriteResult } from './QueryRewriter.js';
+
+// Ingestors
+export { UrlIngestor } from './ingestors/UrlIngestor.js';
+export type { CrawlOptions, UrlIngestorOptions } from './ingestors/UrlIngestor.js';
+export { FileIngestor } from './ingestors/FileIngestor.js';
+export { SiteCrawler } from './ingestors/SiteCrawler.js';
+export type { SiteCrawlOptions, SiteCrawlerOptions } from './ingestors/SiteCrawler.js';
+export { WatiFaqSync } from './ingestors/WatiFaqSync.js';
+export type { FaqPair, WatiFaqSyncOptions } from './ingestors/WatiFaqSync.js';
+
+// Content fetching
+export { fetchHtml, fetchContent, extractFromHtml, extractLinks, resetCrawl4aiHealthCache } from './ingestors/content-fetcher.js';
+export type { FetchContentOptions, FetchContentResult } from './ingestors/content-fetcher.js';

package/src/ingestors/FileIngestor.ts

@@ -0,0 +1,85 @@
+import { readFile } from 'node:fs/promises';
+import { extname } from 'node:path';
+import type { IngestionPipeline } from '../IngestionPipeline.js';
+import type { KBDocument } from '../types.js';
+
+export class FileIngestor {
+  private pipeline: IngestionPipeline;
+
+  constructor(pipeline: IngestionPipeline) {
+    this.pipeline = pipeline;
+  }
+
+  async ingestFile(filePath: string, title?: string, options?: { priority?: number }): Promise<KBDocument> {
+    const ext = extname(filePath).toLowerCase();
+    const content = await this.extractContent(filePath, ext);
+    const fileName = filePath.split('/').pop() || filePath;
+
+    return this.pipeline.ingest({
+      sourceType: 'file',
+      fileName,
+      title: title || fileName,
+      content,
+      priority: options?.priority,
+    });
+  }
+
+  private async extractContent(filePath: string, ext: string): Promise<string> {
+    switch (ext) {
+      case '.pdf':
+        return this.extractPdf(filePath);
+      case '.docx':
+        return this.extractDocx(filePath);
+      case '.xlsx':
+      case '.xls':
+        return this.extractXlsx(filePath);
+      case '.csv':
+      case '.txt':
+      case '.md':
+        return readFile(filePath, 'utf-8');
+      case '.html':
+      case '.htm':
+        return this.extractHtml(filePath);
+      default:
+        return readFile(filePath, 'utf-8');
+    }
+  }
+
+  private async extractPdf(filePath: string): Promise<string> {
+    const { getDocumentProxy, extractText } = await import('unpdf');
+    const buffer = await readFile(filePath);
+    const pdf = await getDocumentProxy(new Uint8Array(buffer));
+    const { text } = await extractText(pdf, { mergePages: true });
+    return text;
+  }
+
+  private async extractDocx(filePath: string): Promise<string> {
+    const mammoth = await import('mammoth');
+    const buffer = await readFile(filePath);
+    const result = await mammoth.extractRawText({ buffer });
+    return result.value;
+  }
+
+  private async extractXlsx(filePath: string): Promise<string> {
+    const XLSX = await import('xlsx');
+    const buffer = await readFile(filePath);
+    const workbook = XLSX.read(buffer, { type: 'buffer' });
+    const lines: string[] = [];
+    for (const sheetName of workbook.SheetNames) {
+      const sheet = workbook.Sheets[sheetName];
+      const csv = XLSX.utils.sheet_to_csv(sheet);
+      lines.push(`## ${sheetName}\n${csv}`);
+    }
+    return lines.join('\n\n');
+  }
+
+  private async extractHtml(filePath: string): Promise<string> {
+    const { parseHTML } = await import('linkedom');
+    const { Readability } = await import('@mozilla/readability');
+    const html = await readFile(filePath, 'utf-8');
+    const { document } = parseHTML(html);
+    const reader = new Readability(document as any);
+    const article = reader.parse();
+    return article?.textContent?.trim() || html;
+  }
+}

package/src/ingestors/SiteCrawler.ts

@@ -0,0 +1,153 @@
+import { fetchHtml, fetchContent, extractLinks } from './content-fetcher.js';
+import type { FetchContentOptions } from './content-fetcher.js';
+import type { IngestionPipeline } from '../IngestionPipeline.js';
+import type { KBDocument } from '../types.js';
+
+export interface SiteCrawlOptions {
+  maxDepth?: number;
+  maxPages?: number;
+  useSitemap?: boolean;
+  delayMs?: number;
+  onProgress?: (crawled: number, discovered: number, url: string) => void;
+}
+
+export interface SiteCrawlerOptions {
+  crawl4aiUrl?: string;
+}
+
+export class SiteCrawler {
+  private pipeline: IngestionPipeline;
+  private fetchOptions: FetchContentOptions;
+
+  constructor(pipeline: IngestionPipeline, options?: SiteCrawlerOptions) {
+    this.pipeline = pipeline;
+    this.fetchOptions = { crawl4aiUrl: options?.crawl4aiUrl };
+  }
+
+  async crawlSite(startUrl: string, options: SiteCrawlOptions = {}): Promise<KBDocument[]> {
+    const {
+      maxDepth = 2,
+      maxPages = 50,
+      useSitemap = true,
+      delayMs = 500,
+      onProgress,
+    } = options;
+
+    const docs: KBDocument[] = [];
+    const visited = new Set<string>();
+
+    if (useSitemap) {
+      const sitemapUrls = await this.tryFetchSitemap(startUrl);
+      if (sitemapUrls.length > 0) {
+        const urlsToIngest = sitemapUrls.slice(0, maxPages);
+        for (const url of urlsToIngest) {
+          if (visited.has(url)) continue;
+          visited.add(url);
+          try {
+            onProgress?.(docs.length + 1, urlsToIngest.length, url);
+            const doc = await this.ingestPage(url);
+            docs.push(doc);
+            if (delayMs > 0) await this.delay(delayMs);
+          } catch {
+            // Skip failed URLs
+          }
+          if (docs.length >= maxPages) break;
+        }
+        return docs;
+      }
+    }
+
+    const queue: { url: string; depth: number }[] = [{ url: startUrl, depth: 0 }];
+
+    while (queue.length > 0 && docs.length < maxPages) {
+      const item = queue.shift()!;
+      if (visited.has(item.url) || item.depth > maxDepth) continue;
+      visited.add(item.url);
+
+      try {
+        onProgress?.(docs.length + 1, docs.length + queue.length + 1, item.url);
+        const html = await fetchHtml(item.url);
+        const { title, content, isMarkdown } = await fetchContent(item.url, this.fetchOptions);
+
+        const doc = await this.pipeline.ingest({
+          sourceType: 'url',
+          sourceUrl: item.url,
+          title,
+          content,
+          isMarkdown,
+        });
+        docs.push(doc);
+
+        if (delayMs > 0) await this.delay(delayMs);
+
+        if (item.depth < maxDepth) {
+          for (const link of extractLinks(html, item.url)) {
+            if (!visited.has(link)) {
+              queue.push({ url: link, depth: item.depth + 1 });
+            }
+          }
+        }
+      } catch {
+        // Skip failed URLs
+      }
+    }
+
+    return docs;
+  }
+
+  private async tryFetchSitemap(baseUrl: string): Promise<string[]> {
+    try {
+      const url = new URL(baseUrl);
+      const sitemapUrl = `${url.protocol}//${url.hostname}/sitemap.xml`;
+      const xml = await fetchHtml(sitemapUrl);
+      return await this.parseSitemapUrls(xml);
+    } catch {
+      return [];
+    }
+  }
+
+  private async parseSitemapUrls(xml: string): Promise<string[]> {
+    const urls: string[] = [];
+    const locRegex = /<loc>(.*?)<\/loc>/g;
+    let match;
+    while ((match = locRegex.exec(xml)) !== null) {
+      urls.push(match[1]);
+    }
+
+    if (xml.includes('<sitemapindex') || urls.every(u => u.endsWith('.xml'))) {
+      const pageUrls: string[] = [];
+      for (const childSitemapUrl of urls) {
+        try {
+          const childXml = await fetchHtml(childSitemapUrl);
+          const childUrls: string[] = [];
+          const childRegex = /<loc>(.*?)<\/loc>/g;
+          let childMatch;
+          while ((childMatch = childRegex.exec(childXml)) !== null) {
+            childUrls.push(childMatch[1]);
+          }
+          pageUrls.push(...childUrls.filter(u => !u.endsWith('.xml')));
+        } catch {
+          // Skip unreachable child sitemaps
+        }
+      }
+      return pageUrls;
+    }
+
+    return urls.filter(u => !u.endsWith('.xml'));
+  }
+
+  private async ingestPage(url: string): Promise<KBDocument> {
+    const { title, content, isMarkdown } = await fetchContent(url, this.fetchOptions);
+    return this.pipeline.ingest({
+      sourceType: 'url',
+      sourceUrl: url,
+      title,
+      content,
+      isMarkdown,
+    });
+  }
+
+  private delay(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+}

package/src/ingestors/UrlIngestor.ts

@@ -0,0 +1,106 @@
+import { fetchHtml, fetchContent, extractLinks } from './content-fetcher.js';
+import type { FetchContentOptions } from './content-fetcher.js';
+import type { IngestionPipeline } from '../IngestionPipeline.js';
+import type { KBDocument } from '../types.js';
+
+export interface CrawlOptions {
+  maxPages?: number;
+  maxDepth?: number;
+}
+
+export interface UrlIngestorOptions {
+  crawl4aiUrl?: string;
+}
+
+export class UrlIngestor {
+  private pipeline: IngestionPipeline;
+  private fetchOptions: FetchContentOptions;
+
+  constructor(pipeline: IngestionPipeline, options?: UrlIngestorOptions) {
+    this.pipeline = pipeline;
+    this.fetchOptions = { crawl4aiUrl: options?.crawl4aiUrl };
+  }
+
+  async ingestUrl(url: string, options?: { priority?: number; extractQA?: boolean }): Promise<KBDocument> {
+    const { title, content, isMarkdown } = await fetchContent(url, this.fetchOptions);
+
+    return this.pipeline.ingest({
+      sourceType: 'url',
+      sourceUrl: url,
+      title,
+      content,
+      isMarkdown,
+      priority: options?.priority,
+      extractQA: options?.extractQA,
+    });
+  }
+
+  async ingestSitemap(sitemapUrl: string, options?: CrawlOptions): Promise<KBDocument[]> {
+    const maxPages = options?.maxPages || 50;
+    const xml = await fetchHtml(sitemapUrl);
+    const urls = this.parseSitemapUrls(xml).slice(0, maxPages);
+
+    const docs: KBDocument[] = [];
+    for (const url of urls) {
+      try {
+        const doc = await this.ingestUrl(url);
+        docs.push(doc);
+      } catch {
+        // Skip failed URLs
+      }
+    }
+    return docs;
+  }
+
+  async crawl(startUrl: string, options?: CrawlOptions): Promise<KBDocument[]> {
+    const maxPages = options?.maxPages || 20;
+    const maxDepth = options?.maxDepth || 2;
+    const visited = new Set<string>();
+    const docs: KBDocument[] = [];
+
+    const queue: { url: string; depth: number }[] = [{ url: startUrl, depth: 0 }];
+
+    while (queue.length > 0 && docs.length < maxPages) {
+      const item = queue.shift()!;
+      if (visited.has(item.url) || item.depth > maxDepth) continue;
+      visited.add(item.url);
+
+      try {
+        const html = await fetchHtml(item.url);
+        const { title, content, isMarkdown } = await fetchContent(item.url, this.fetchOptions);
+
+        const doc = await this.pipeline.ingest({
+          sourceType: 'url',
+          sourceUrl: item.url,
+          title,
+          content,
+          isMarkdown,
+        });
+        docs.push(doc);
+
+        if (item.depth < maxDepth) {
+          const links = extractLinks(html, item.url);
+          for (const link of links) {
+            if (!visited.has(link)) {
+              queue.push({ url: link, depth: item.depth + 1 });
+            }
+          }
+        }
+      } catch {
+        // Skip failed URLs
+      }
+    }
+
+    return docs;
+  }
+
+  private parseSitemapUrls(xml: string): string[] {
+    const urls: string[] = [];
+    const locRegex = /<loc>(.*?)<\/loc>/g;
+    let match;
+    while ((match = locRegex.exec(xml)) !== null) {
+      urls.push(match[1]);
+    }
+    return urls.filter(u => !u.endsWith('.xml'));
+  }
+}

package/src/ingestors/WatiFaqSync.ts

@@ -0,0 +1,75 @@
+import type { IngestionPipeline } from '../IngestionPipeline.js';
+import type { KBDocument } from '../types.js';
+
+export interface FaqPair {
+  question: string;
+  answer: string;
+}
+
+export interface WatiFaqSyncOptions {
+  minAnswerLength?: number;
+  maxPairs?: number;
+}
+
+export class WatiFaqSync {
+  private pipeline: IngestionPipeline;
+  private llmExtract?: (conversation: string) => Promise<FaqPair[]>;
+
+  constructor(
+    pipeline: IngestionPipeline,
+    llmExtract?: (conversation: string) => Promise<FaqPair[]>,
+  ) {
+    this.pipeline = pipeline;
+    this.llmExtract = llmExtract;
+  }
+
+  async syncFromConversations(
+    conversations: string[],
+    options?: WatiFaqSyncOptions,
+  ): Promise<KBDocument[]> {
+    const minLen = options?.minAnswerLength || 20;
+    const maxPairs = options?.maxPairs || 100;
+
+    if (!this.llmExtract) {
+      throw new Error('LLM extract function required for FAQ extraction');
+    }
+
+    const allPairs: FaqPair[] = [];
+    for (const convo of conversations) {
+      const pairs = await this.llmExtract(convo);
+      allPairs.push(...pairs);
+      if (allPairs.length >= maxPairs) break;
+    }
+
+    // Quality filtering
+    const filtered = allPairs
+      .filter((p) => p.answer.length >= minLen && p.question.trim().length > 0)
+      .slice(0, maxPairs);
+
+    const docs: KBDocument[] = [];
+    for (const pair of filtered) {
+      const doc = await this.pipeline.ingestFaq(pair.question, pair.answer, {
+        source: 'wati-sync',
+      });
+      docs.push(doc);
+    }
+
+    return docs;
+  }
+
+  async syncFromPairs(pairs: FaqPair[], options?: WatiFaqSyncOptions): Promise<KBDocument[]> {
+    const minLen = options?.minAnswerLength || 20;
+    const filtered = pairs.filter(
+      (p) => p.answer.length >= minLen && p.question.trim().length > 0,
+    );
+
+    const docs: KBDocument[] = [];
+    for (const pair of filtered) {
+      const doc = await this.pipeline.ingestFaq(pair.question, pair.answer, {
+        source: 'wati-sync',
+      });
+      docs.push(doc);
+    }
+    return docs;
+  }
+}

package/src/ingestors/content-fetcher.ts

@@ -0,0 +1,142 @@
+import { Readability } from '@mozilla/readability';
+import { parseHTML } from 'linkedom';
+
+export interface FetchContentOptions {
+  crawl4aiUrl?: string;
+}
+
+export interface FetchContentResult {
+  title: string;
+  content: string;
+  isMarkdown: boolean;
+}
+
+// Cache Crawl4AI health check result for 5 minutes
+let crawl4aiHealthy: boolean | null = null;
+let crawl4aiHealthCheckedAt = 0;
+const HEALTH_CACHE_MS = 5 * 60 * 1000;
+
+/** Reset the Crawl4AI health check cache. Used in tests. */
+export function resetCrawl4aiHealthCache(): void {
+  crawl4aiHealthy = null;
+  crawl4aiHealthCheckedAt = 0;
+}
+
+/**
+ * Fetch raw HTML from a URL. Used for link extraction, sitemaps, etc.
+ */
+export async function fetchHtml(url: string): Promise<string> {
+  const response = await fetch(url, {
+    headers: { 'User-Agent': 'Operor-KB/1.0' },
+  });
+  if (!response.ok) throw new Error(`Failed to fetch ${url}: ${response.status}`);
+  return response.text();
+}
+
+/**
+ * Smart content fetch: tries Crawl4AI first (if configured), falls back to Readability.
+ */
+export async function fetchContent(url: string, options?: FetchContentOptions): Promise<FetchContentResult> {
+  if (options?.crawl4aiUrl) {
+    try {
+      if (await isCrawl4aiHealthy(options.crawl4aiUrl)) {
+        return await fetchViaCrawl4AI(url, options.crawl4aiUrl);
+      }
+    } catch {
+      // Fall through to Readability
+    }
+  }
+
+  const html = await fetchHtml(url);
+  const { title, content } = extractFromHtml(html, url);
+  return { title, content, isMarkdown: false };
+}
+
+/**
+ * Extract readable content from pre-fetched HTML using @mozilla/readability.
+ */
+export function extractFromHtml(html: string, url: string): { title: string; content: string } {
+  const { document } = parseHTML(html);
+  const reader = new Readability(document as any, { url });
+  const article = reader.parse();
+  return {
+    title: article?.title || '',
+    content: article?.textContent?.trim() || '',
+  };
+}
+
+/**
+ * Extract same-domain links from HTML.
+ */
+export function extractLinks(html: string, baseUrl: string): string[] {
+  const { document } = parseHTML(html);
+  const links: string[] = [];
+  const base = new URL(baseUrl);
+
+  for (const a of document.querySelectorAll('a[href]')) {
+    try {
+      const href = (a as any).getAttribute('href');
+      if (!href) continue;
+      const resolved = new URL(href, baseUrl);
+      if (resolved.hostname === base.hostname && resolved.protocol.startsWith('http')) {
+        links.push(resolved.href.split('#')[0]);
+      }
+    } catch {
+      // Skip invalid URLs
+    }
+  }
+
+  return [...new Set(links)];
+}
+
+async function isCrawl4aiHealthy(baseUrl: string): Promise<boolean> {
+  if (crawl4aiHealthy !== null && Date.now() - crawl4aiHealthCheckedAt < HEALTH_CACHE_MS) {
+    return crawl4aiHealthy;
+  }
+  try {
+    const res = await fetch(`${baseUrl}/health`, { signal: AbortSignal.timeout(2000) });
+    crawl4aiHealthy = res.ok;
+  } catch {
+    crawl4aiHealthy = false;
+  }
+  crawl4aiHealthCheckedAt = Date.now();
+  return crawl4aiHealthy;
+}
+
+async function fetchViaCrawl4AI(url: string, baseUrl: string): Promise<FetchContentResult> {
+  const res = await fetch(`${baseUrl}/crawl`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      urls: [url],
+      browser_config: { type: 'BrowserConfig', params: { headless: true } },
+      crawler_config: {
+        type: 'CrawlerRunConfig',
+        params: {
+          cache_mode: 'bypass',
+          markdown_generator: {
+            type: 'DefaultMarkdownGenerator',
+            params: {
+              content_filter: { type: 'PruningContentFilter', params: { threshold: 0.48 } },
+            },
+          },
+        },
+      },
+    }),
+    signal: AbortSignal.timeout(30000),
+  });
+
+  if (!res.ok) throw new Error(`Crawl4AI error: ${res.status}`);
+  const data = await res.json() as any;
+  const result = data.results?.[0];
+  if (!result) throw new Error('Crawl4AI returned no results');
+
+  const markdown = result.markdown?.fit_markdown || result.markdown?.raw_markdown || '';
+  if (!markdown) throw new Error('Crawl4AI returned empty markdown');
+
+  // Extract title from first heading
+  const titleMatch = markdown.match(/^#\s+(.+)$/m);
+  const title = titleMatch?.[1] || '';
+
+  return { title, content: markdown, isMarkdown: true };
+}

package/src/types.ts

@@ -0,0 +1,62 @@
+export interface KBDocument {
+  id: string;
+  sourceType: 'url' | 'file' | 'faq' | 'annotation';
+  sourceUrl?: string;
+  fileName?: string;
+  title?: string;
+  content: string;
+  metadata?: Record<string, any>;
+  createdAt: number;
+  updatedAt: number;
+  priority?: number; // 1=official, 2=supplementary, 3=archived (default: 2)
+  contentHash?: string; // SHA-256 for deduplication
+}
+
+export interface KBChunk {
+  id: string;
+  documentId: string;
+  content: string;
+  chunkIndex: number;
+  embedding?: number[];
+  metadata?: Record<string, any>;
+}
+
+export interface KBSearchResult {
+  chunk: KBChunk;
+  document: KBDocument;
+  score: number;
+  distance: number;
+}
+
+export interface KBSearchOptions {
+  limit?: number;
+  scoreThreshold?: number;
+  sourceTypes?: KBDocument['sourceType'][];
+  metadata?: Record<string, any>;
+}
+
+export interface KBStats {
+  documentCount: number;
+  chunkCount: number;
+  embeddingDimensions: number;
+  dbSizeBytes: number;
+}
+
+export interface KnowledgeStore {
+  initialize(): Promise<void>;
+  close(): Promise<void>;
+
+  addDocument(doc: KBDocument): Promise<void>;
+  getDocument(id: string): Promise<KBDocument | null>;
+  listDocuments(): Promise<KBDocument[]>;
+  deleteDocument(id: string): Promise<void>;
+
+  addChunks(chunks: KBChunk[]): Promise<void>;
+  search(query: string, embedding: number[], options?: KBSearchOptions): Promise<KBSearchResult[]>;
+  searchByEmbedding(embedding: number[], options?: KBSearchOptions): Promise<KBSearchResult[]>;
+  searchByKeyword?(query: string, options?: KBSearchOptions): Promise<KBSearchResult[]>;
+  getStats(): Promise<KBStats>;
+
+  /** Returns the number of chunks stored for a given document. Optional — only SQLiteKnowledgeStore implements this. */
+  getChunkCount?(documentId: string): number;
+}

package/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

package/tsdown.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from 'tsdown';
+
+export default defineConfig({
+  entry: ['src/index.ts'],
+  format: ['esm'],
+  dts: true,
+  clean: true,
+  sourcemap: true,
+  outExtensions: () => ({ js: '.js', dts: '.d.ts' }),
+});