@staticn0va/wigolo 0.1.0 → 0.1.1

package/dist/embedding/embed.js

@@ -0,0 +1,233 @@
+import {
+  getVectorStore
+} from "../providers/vector-store.js";
+import {
+  updateCacheEmbedding,
+  getAllEmbeddings,
+  normalizeUrl
+} from "../cache/store.js";
+import { FastembedEmbedProvider } from "./fastembed-provider.js";
+import { createLogger } from "../logger.js";
+const log = createLogger("embedding");
+class EmbeddingService {
+  provider;
+  store = null;
+  knownUrls = /* @__PURE__ */ new Set();
+  available = false;
+  providerVerified = false;
+  constructor(provider) {
+    this.provider = provider ?? new FastembedEmbedProvider();
+  }
+  async init() {
+    try {
+      this.store = await getVectorStore();
+      try {
+        const existingSize = await this.store.size();
+        if (existingSize === 0) {
+          await this.migrateLegacyEmbeddings();
+        } else {
+        }
+      } catch (err) {
+        log.warn("embedding migration check failed", {
+          error: err instanceof Error ? err.message : String(err)
+        });
+      }
+      try {
+        await this.provider.embed(["embedding service probe"]);
+        this.providerVerified = true;
+        log.info("embedding provider verified", {
+          modelId: this.provider.modelId,
+          dim: this.provider.dim
+        });
+      } catch (err) {
+        log.warn("embedding provider probe failed \u2014 embeddings disabled", {
+          error: err instanceof Error ? err.message : String(err)
+        });
+        this.providerVerified = false;
+      }
+      this.available = true;
+    } catch (err) {
+      log.error("EmbeddingService init failed", { error: String(err) });
+      this.available = false;
+    }
+  }
+  isAvailable() {
+    return this.available;
+  }
+  setAvailable(value) {
+    this.available = value;
+  }
+  /** Backwards-compat alias preserved for callers that gated on subprocess readiness. */
+  isSubprocessReady() {
+    return this.providerVerified;
+  }
+  /**
+   * Lightweight index view. Returns `size` from the backing VectorStore and
+   * `has` from a local URL-cache populated by embedAndStore. Callers that
+   * need richer access should consume the VectorStore directly via
+   * `getVectorStore()`.
+   */
+  getIndex() {
+    const knownUrls = this.knownUrls;
+    const store = this.store;
+    return {
+      size: () => store ? this.cachedSize : knownUrls.size,
+      has: (url) => knownUrls.has(url)
+    };
+  }
+  /**
+   * Cached size from the store, refreshed after upserts. Reads from a
+   * VectorStore would be async; getIndex().size() callers expect a
+   * synchronous return so we maintain this counter.
+   */
+  cachedSize = 0;
+  async embedAndStore(url, markdown) {
+    if (!this.available) {
+      log.debug("embedding skipped: service not available", { url });
+      return;
+    }
+    try {
+      const [vector] = await this.provider.embed([markdown]);
+      if (!vector || vector.length === 0) {
+        log.warn("embedding returned empty vector", { url });
+        return;
+      }
+      const buffer = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
+      const model = this.provider.modelId;
+      const dims = vector.length;
+      let normalizedUrl;
+      try {
+        normalizedUrl = normalizeUrl(url);
+      } catch {
+        normalizedUrl = url;
+      }
+      updateCacheEmbedding(normalizedUrl, buffer, model, dims);
+      if (this.store) {
+        const record = {
+          id: normalizedUrl,
+          vector,
+          metadata: { url: normalizedUrl, contentHash: "", modelId: model }
+        };
+        await this.store.upsert([record]);
+        if (!this.knownUrls.has(normalizedUrl)) {
+          this.knownUrls.add(normalizedUrl);
+          this.cachedSize += 1;
+        }
+      }
+      log.debug("embedded and stored", { url: normalizedUrl, dims });
+    } catch (err) {
+      log.warn("embedAndStore failed", { url, error: String(err) });
+    }
+  }
+  embedAsync(url, markdown) {
+    if (!this.available) return;
+    this.embedAndStore(url, markdown).catch((err) => {
+      log.warn("async embedding failed", { url, error: String(err) });
+    });
+  }
+  async findSimilar(queryText, topK, excludeUrls) {
+    if (!this.available || !this.store) {
+      return [];
+    }
+    if (this.cachedSize === 0) {
+      try {
+        this.cachedSize = await this.store.size();
+      } catch {
+        this.cachedSize = 0;
+      }
+      if (this.cachedSize === 0) return [];
+    }
+    try {
+      const [queryVector] = await this.provider.embed([queryText]);
+      if (!queryVector || queryVector.length === 0) {
+        log.warn("query embedding failed: empty vector");
+        return [];
+      }
+      const overscan = excludeUrls && excludeUrls.size > 0 ? Math.max(topK + excludeUrls.size, topK * 2) : topK;
+      const hits = await this.store.search(queryVector, overscan);
+      const results = [];
+      for (const hit of hits) {
+        if (excludeUrls?.has(hit.id)) continue;
+        results.push({ url: hit.id, score: hit.score });
+        if (results.length >= topK) break;
+      }
+      return results;
+    } catch (err) {
+      log.warn("findSimilar failed", { error: String(err) });
+      return [];
+    }
+  }
+  shutdown() {
+    try {
+      this.knownUrls.clear();
+      this.cachedSize = 0;
+      this.store = null;
+      this.available = false;
+      this.providerVerified = false;
+      log.info("EmbeddingService shut down");
+    } catch (err) {
+      log.error("EmbeddingService shutdown error", { error: String(err) });
+    }
+  }
+  async migrateLegacyEmbeddings() {
+    if (!this.store) return;
+    const legacy = getAllEmbeddings(this.provider.modelId);
+    if (legacy.length === 0) {
+      this.cachedSize = 0;
+      return;
+    }
+    const records = [];
+    for (const row of legacy) {
+      if (!row.embedding || row.dims <= 0) continue;
+      try {
+        const vector = new Float32Array(
+          row.embedding.buffer.slice(
+            row.embedding.byteOffset,
+            row.embedding.byteOffset + row.dims * Float32Array.BYTES_PER_ELEMENT
+          )
+        );
+        records.push({
+          id: row.normalizedUrl,
+          vector,
+          metadata: {
+            url: row.normalizedUrl,
+            contentHash: "",
+            modelId: row.model
+          }
+        });
+        this.knownUrls.add(row.normalizedUrl);
+      } catch (err) {
+        log.warn("legacy embedding migration: failed to decode vector", {
+          url: row.normalizedUrl,
+          error: err instanceof Error ? err.message : String(err)
+        });
+      }
+    }
+    if (records.length === 0) {
+      this.cachedSize = 0;
+      return;
+    }
+    log.info("migrating embeddings into sqlite-vec store", { count: records.length });
+    await this.store.upsert(records);
+    this.cachedSize = await this.store.size();
+  }
+}
+let globalInstance = null;
+function getEmbeddingService() {
+  if (!globalInstance) {
+    globalInstance = new EmbeddingService();
+  }
+  return globalInstance;
+}
+function resetEmbeddingService() {
+  if (globalInstance) {
+    globalInstance.shutdown();
+    globalInstance = null;
+  }
+}
+export {
+  EmbeddingService,
+  getEmbeddingService,
+  resetEmbeddingService
+};
+//# sourceMappingURL=embed.js.map

package/dist/embedding/embed.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/embedding/embed.ts"],"sourcesContent":["import type { EmbedProvider } from '../providers/embed-provider.js';\nimport {\n  getVectorStore,\n  type VectorStore,\n  type VectorRecord,\n} from '../providers/vector-store.js';\nimport {\n  updateCacheEmbedding,\n  getAllEmbeddings,\n  normalizeUrl,\n} from '../cache/store.js';\nimport { FastembedEmbedProvider } from './fastembed-provider.js';\nimport { createLogger } from '../logger.js';\n\nconst log = createLogger('embedding');\n\nexport interface SimilarResult {\n  url: string;\n  score: number;\n}\n\n/**\n * Index shim exposed by `getIndex()` for callers that still need\n * lightweight size/membership checks. Kept narrow so future stores can\n * implement it without dragging in extra surface area.\n */\nexport interface IndexView {\n  size(): number;\n  has(url: string): boolean;\n}\n\n/**\n * Embedding service backed by the native fastembed (ONNX) provider and\n * the sqlite-vec VectorStore.\n *\n * Phase 5 replaced the in-memory VectorIndex with the sqlite-vec backed\n * store accessed via getVectorStore(). The public surface (init /\n * embedAndStore / embedAsync / findSimilar / getIndex / isAvailable /\n * shutdown) is unchanged so callers in server.ts, tools/fetch.ts,\n * research/pipeline.ts, search/find-similar.ts, and the legacy SearXNG\n * orchestrator continue to work without modification.\n */\nexport class EmbeddingService {\n  private provider: EmbedProvider;\n  private store: VectorStore | null = null;\n  private knownUrls = new Set<string>();\n  private available = false;\n  private providerVerified = false;\n\n  constructor(provider?: EmbedProvider) {\n    this.provider = provider ?? new FastembedEmbedProvider();\n  }\n\n  async init(): Promise<void> {\n    try {\n      this.store = await getVectorStore();\n\n      // Migrate any embeddings persisted in url_cache (pre-Phase-5 layout)\n      // into the sqlite-vec backed store on first use. Skips on hit so\n      // re-init is cheap.\n      try {\n        const existingSize = await this.store.size();\n        if (existingSize === 0) {\n          await this.migrateLegacyEmbeddings();\n        } else {\n          // Seed knownUrls from the store so embedAndStore can avoid\n          // unnecessary re-upserts when content has not changed.\n          // The current store has no list API, so we leave knownUrls empty\n          // and rely on upsert idempotency.\n        }\n      } catch (err) {\n        log.warn('embedding migration check failed', {\n          error: err instanceof Error ? err.message : String(err),\n        });\n      }\n\n      // Probe the provider so we know up front whether ONNX init works.\n      try {\n        await this.provider.embed(['embedding service probe']);\n        this.providerVerified = true;\n        log.info('embedding provider verified', {\n          modelId: this.provider.modelId,\n          dim: this.provider.dim,\n        });\n      } catch (err) {\n        log.warn('embedding provider probe failed — embeddings disabled', {\n          error: err instanceof Error ? err.message : String(err),\n        });\n        this.providerVerified = false;\n      }\n\n      this.available = true;\n    } catch (err) {\n      log.error('EmbeddingService init failed', { error: String(err) });\n      this.available = false;\n    }\n  }\n\n  isAvailable(): boolean {\n    return this.available;\n  }\n\n  setAvailable(value: boolean): void {\n    this.available = value;\n  }\n\n  /** Backwards-compat alias preserved for callers that gated on subprocess readiness. */\n  isSubprocessReady(): boolean {\n    return this.providerVerified;\n  }\n\n  /**\n   * Lightweight index view. Returns `size` from the backing VectorStore and\n   * `has` from a local URL-cache populated by embedAndStore. Callers that\n   * need richer access should consume the VectorStore directly via\n   * `getVectorStore()`.\n   */\n  getIndex(): IndexView {\n    const knownUrls = this.knownUrls;\n    const store = this.store;\n    return {\n      size: () => (store ? this.cachedSize : knownUrls.size),\n      has: (url: string) => knownUrls.has(url),\n    };\n  }\n\n  /**\n   * Cached size from the store, refreshed after upserts. Reads from a\n   * VectorStore would be async; getIndex().size() callers expect a\n   * synchronous return so we maintain this counter.\n   */\n  private cachedSize = 0;\n\n  async embedAndStore(url: string, markdown: string): Promise<void> {\n    if (!this.available) {\n      log.debug('embedding skipped: service not available', { url });\n      return;\n    }\n\n    try {\n      const [vector] = await this.provider.embed([markdown]);\n      if (!vector || vector.length === 0) {\n        log.warn('embedding returned empty vector', { url });\n        return;\n      }\n\n      const buffer = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);\n      const model = this.provider.modelId;\n      const dims = vector.length;\n\n      let normalizedUrl: string;\n      try {\n        normalizedUrl = normalizeUrl(url);\n      } catch {\n        normalizedUrl = url;\n      }\n\n      updateCacheEmbedding(normalizedUrl, buffer, model, dims);\n\n      if (this.store) {\n        const record: VectorRecord = {\n          id: normalizedUrl,\n          vector,\n          metadata: { url: normalizedUrl, contentHash: '', modelId: model },\n        };\n        await this.store.upsert([record]);\n        if (!this.knownUrls.has(normalizedUrl)) {\n          this.knownUrls.add(normalizedUrl);\n          this.cachedSize += 1;\n        }\n      }\n\n      log.debug('embedded and stored', { url: normalizedUrl, dims });\n    } catch (err) {\n      log.warn('embedAndStore failed', { url, error: String(err) });\n    }\n  }\n\n  embedAsync(url: string, markdown: string): void {\n    if (!this.available) return;\n\n    this.embedAndStore(url, markdown).catch(err => {\n      log.warn('async embedding failed', { url, error: String(err) });\n    });\n  }\n\n  async findSimilar(\n    queryText: string,\n    topK: number,\n    excludeUrls?: Set<string>,\n  ): Promise<SimilarResult[]> {\n    if (!this.available || !this.store) {\n      return [];\n    }\n    if (this.cachedSize === 0) {\n      // Refresh once before returning empty so newly-populated stores\n      // (e.g. legacy migration just finished) are visible to callers.\n      try {\n        this.cachedSize = await this.store.size();\n      } catch {\n        this.cachedSize = 0;\n      }\n      if (this.cachedSize === 0) return [];\n    }\n\n    try {\n      const [queryVector] = await this.provider.embed([queryText]);\n      if (!queryVector || queryVector.length === 0) {\n        log.warn('query embedding failed: empty vector');\n        return [];\n      }\n\n      const overscan = excludeUrls && excludeUrls.size > 0\n        ? Math.max(topK + excludeUrls.size, topK * 2)\n        : topK;\n      const hits = await this.store.search(queryVector, overscan);\n\n      const results: SimilarResult[] = [];\n      for (const hit of hits) {\n        if (excludeUrls?.has(hit.id)) continue;\n        results.push({ url: hit.id, score: hit.score });\n        if (results.length >= topK) break;\n      }\n      return results;\n    } catch (err) {\n      log.warn('findSimilar failed', { error: String(err) });\n      return [];\n    }\n  }\n\n  shutdown(): void {\n    try {\n      this.knownUrls.clear();\n      this.cachedSize = 0;\n      this.store = null;\n      this.available = false;\n      this.providerVerified = false;\n      log.info('EmbeddingService shut down');\n    } catch (err) {\n      log.error('EmbeddingService shutdown error', { error: String(err) });\n    }\n  }\n\n  private async migrateLegacyEmbeddings(): Promise<void> {\n    if (!this.store) return;\n    const legacy = getAllEmbeddings(this.provider.modelId);\n    if (legacy.length === 0) {\n      this.cachedSize = 0;\n      return;\n    }\n\n    const records: VectorRecord[] = [];\n    for (const row of legacy) {\n      if (!row.embedding || row.dims <= 0) continue;\n      try {\n        const vector = new Float32Array(\n          row.embedding.buffer.slice(\n            row.embedding.byteOffset,\n            row.embedding.byteOffset + row.dims * Float32Array.BYTES_PER_ELEMENT,\n          ),\n        );\n        records.push({\n          id: row.normalizedUrl,\n          vector,\n          metadata: {\n            url: row.normalizedUrl,\n            contentHash: '',\n            modelId: row.model,\n          },\n        });\n        this.knownUrls.add(row.normalizedUrl);\n      } catch (err) {\n        log.warn('legacy embedding migration: failed to decode vector', {\n          url: row.normalizedUrl,\n          error: err instanceof Error ? err.message : String(err),\n        });\n      }\n    }\n\n    if (records.length === 0) {\n      this.cachedSize = 0;\n      return;\n    }\n\n    log.info('migrating embeddings into sqlite-vec store', { count: records.length });\n    await this.store.upsert(records);\n    this.cachedSize = await this.store.size();\n  }\n}\n\nlet globalInstance: EmbeddingService | null = null;\n\nexport function getEmbeddingService(): EmbeddingService {\n  if (!globalInstance) {\n    globalInstance = new EmbeddingService();\n  }\n  return globalInstance;\n}\n\nexport function resetEmbeddingService(): void {\n  if (globalInstance) {\n    globalInstance.shutdown();\n    globalInstance = null;\n  }\n}\n"],"mappings":"AACA;AAAA,EACE;AAAA,OAGK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,8BAA8B;AACvC,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,WAAW;AA4B7B,MAAM,iBAAiB;AAAA,EACpB;AAAA,EACA,QAA4B;AAAA,EAC5B,YAAY,oBAAI,IAAY;AAAA,EAC5B,YAAY;AAAA,EACZ,mBAAmB;AAAA,EAE3B,YAAY,UAA0B;AACpC,SAAK,WAAW,YAAY,IAAI,uBAAuB;AAAA,EACzD;AAAA,EAEA,MAAM,OAAsB;AAC1B,QAAI;AACF,WAAK,QAAQ,MAAM,eAAe;AAKlC,UAAI;AACF,cAAM,eAAe,MAAM,KAAK,MAAM,KAAK;AAC3C,YAAI,iBAAiB,GAAG;AACtB,gBAAM,KAAK,wBAAwB;AAAA,QACrC,OAAO;AAAA,QAKP;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,KAAK,oCAAoC;AAAA,UAC3C,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,QACxD,CAAC;AAAA,MACH;AAGA,UAAI;AACF,cAAM,KAAK,SAAS,MAAM,CAAC,yBAAyB,CAAC;AACrD,aAAK,mBAAmB;AACxB,YAAI,KAAK,+BAA+B;AAAA,UACtC,SAAS,KAAK,SAAS;AAAA,UACvB,KAAK,KAAK,SAAS;AAAA,QACrB,CAAC;AAAA,MACH,SAAS,KAAK;AACZ,YAAI,KAAK,8DAAyD;AAAA,UAChE,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,QACxD,CAAC;AACD,aAAK,mBAAmB;AAAA,MAC1B;AAEA,WAAK,YAAY;AAAA,IACnB,SAAS,KAAK;AACZ,UAAI,MAAM,gCAAgC,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAChE,WAAK,YAAY;AAAA,IACnB;AAAA,EACF;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,aAAa,OAAsB;AACjC,SAAK,YAAY;AAAA,EACnB;AAAA;AAAA,EAGA,oBAA6B;AAC3B,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,WAAsB;AACpB,UAAM,YAAY,KAAK;AACvB,UAAM,QAAQ,KAAK;AACnB,WAAO;AAAA,MACL,MAAM,MAAO,QAAQ,KAAK,aAAa,UAAU;AAAA,MACjD,KAAK,CAAC,QAAgB,UAAU,IAAI,GAAG;AAAA,IACzC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOQ,aAAa;AAAA,EAErB,MAAM,cAAc,KAAa,UAAiC;AAChE,QAAI,CAAC,KAAK,WAAW;AACnB,UAAI,MAAM,4CAA4C,EAAE,IAAI,CAAC;AAC7D;AAAA,IACF;AAEA,QAAI;AACF,YAAM,CAAC,MAAM,IAAI,MAAM,KAAK,SAAS,MAAM,CAAC,QAAQ,CAAC;AACrD,UAAI,CAAC,UAAU,OAAO,WAAW,GAAG;AAClC,YAAI,KAAK,mCAAmC,EAAE,IAAI,CAAC;AACnD;AAAA,MACF;AAEA,YAAM,SAAS,OAAO,KAAK,OAAO,QAAQ,OAAO,YAAY,OAAO,UAAU;AAC9E,YAAM,QAAQ,KAAK,SAAS;AAC5B,YAAM,OAAO,OAAO;AAEpB,UAAI;AACJ,UAAI;AACF,wBAAgB,aAAa,GAAG;AAAA,MAClC,QAAQ;AACN,wBAAgB;AAAA,MAClB;AAEA,2BAAqB,eAAe,QAAQ,OAAO,IAAI;AAEvD,UAAI,KAAK,OAAO;AACd,cAAM,SAAuB;AAAA,UAC3B,IAAI;AAAA,UACJ;AAAA,UACA,UAAU,EAAE,KAAK,eAAe,aAAa,IAAI,SAAS,MAAM;AAAA,QAClE;AACA,cAAM,KAAK,MAAM,OAAO,CAAC,MAAM,CAAC;AAChC,YAAI,CAAC,KAAK,UAAU,IAAI,aAAa,GAAG;AACtC,eAAK,UAAU,IAAI,aAAa;AAChC,eAAK,cAAc;AAAA,QACrB;AAAA,MACF;AAEA,UAAI,MAAM,uBAAuB,EAAE,KAAK,eAAe,KAAK,CAAC;AAAA,IAC/D,SAAS,KAAK;AACZ,UAAI,KAAK,wBAAwB,EAAE,KAAK,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,IAC9D;AAAA,EACF;AAAA,EAEA,WAAW,KAAa,UAAwB;AAC9C,QAAI,CAAC,KAAK,UAAW;AAErB,SAAK,cAAc,KAAK,QAAQ,EAAE,MAAM,SAAO;AAC7C,UAAI,KAAK,0BAA0B,EAAE,KAAK,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,IAChE,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,YACJ,WACA,MACA,aAC0B;AAC1B,QAAI,CAAC,KAAK,aAAa,CAAC,KAAK,OAAO;AAClC,aAAO,CAAC;AAAA,IACV;AACA,QAAI,KAAK,eAAe,GAAG;AAGzB,UAAI;AACF,aAAK,aAAa,MAAM,KAAK,MAAM,KAAK;AAAA,MAC1C,QAAQ;AACN,aAAK,aAAa;AAAA,MACpB;AACA,UAAI,KAAK,eAAe,EAAG,QAAO,CAAC;AAAA,IACrC;AAEA,QAAI;AACF,YAAM,CAAC,WAAW,IAAI,MAAM,KAAK,SAAS,MAAM,CAAC,SAAS,CAAC;AAC3D,UAAI,CAAC,eAAe,YAAY,WAAW,GAAG;AAC5C,YAAI,KAAK,sCAAsC;AAC/C,eAAO,CAAC;AAAA,MACV;AAEA,YAAM,WAAW,eAAe,YAAY,OAAO,IAC/C,KAAK,IAAI,OAAO,YAAY,MAAM,OAAO,CAAC,IAC1C;AACJ,YAAM,OAAO,MAAM,KAAK,MAAM,OAAO,aAAa,QAAQ;AAE1D,YAAM,UAA2B,CAAC;AAClC,iBAAW,OAAO,MAAM;AACtB,YAAI,aAAa,IAAI,IAAI,EAAE,EAAG;AAC9B,gBAAQ,KAAK,EAAE,KAAK,IAAI,IAAI,OAAO,IAAI,MAAM,CAAC;AAC9C,YAAI,QAAQ,UAAU,KAAM;AAAA,MAC9B;AACA,aAAO;AAAA,IACT,SAAS,KAAK;AACZ,UAAI,KAAK,sBAAsB,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AACrD,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAAA,EAEA,WAAiB;AACf,QAAI;AACF,WAAK,UAAU,MAAM;AACrB,WAAK,aAAa;AAClB,WAAK,QAAQ;AACb,WAAK,YAAY;AACjB,WAAK,mBAAmB;AACxB,UAAI,KAAK,4BAA4B;AAAA,IACvC,SAAS,KAAK;AACZ,UAAI,MAAM,mCAAmC,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,IACrE;AAAA,EACF;AAAA,EAEA,MAAc,0BAAyC;AACrD,QAAI,CAAC,KAAK,MAAO;AACjB,UAAM,SAAS,iBAAiB,KAAK,SAAS,OAAO;AACrD,QAAI,OAAO,WAAW,GAAG;AACvB,WAAK,aAAa;AAClB;AAAA,IACF;AAEA,UAAM,UAA0B,CAAC;AACjC,eAAW,OAAO,QAAQ;AACxB,UAAI,CAAC,IAAI,aAAa,IAAI,QAAQ,EAAG;AACrC,UAAI;AACF,cAAM,SAAS,IAAI;AAAA,UACjB,IAAI,UAAU,OAAO;AAAA,YACnB,IAAI,UAAU;AAAA,YACd,IAAI,UAAU,aAAa,IAAI,OAAO,aAAa;AAAA,UACrD;AAAA,QACF;AACA,gBAAQ,KAAK;AAAA,UACX,IAAI,IAAI;AAAA,UACR;AAAA,UACA,UAAU;AAAA,YACR,KAAK,IAAI;AAAA,YACT,aAAa;AAAA,YACb,SAAS,IAAI;AAAA,UACf;AAAA,QACF,CAAC;AACD,aAAK,UAAU,IAAI,IAAI,aAAa;AAAA,MACtC,SAAS,KAAK;AACZ,YAAI,KAAK,uDAAuD;AAAA,UAC9D,KAAK,IAAI;AAAA,UACT,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,QACxD,CAAC;AAAA,MACH;AAAA,IACF;AAEA,QAAI,QAAQ,WAAW,GAAG;AACxB,WAAK,aAAa;AAClB;AAAA,IACF;AAEA,QAAI,KAAK,8CAA8C,EAAE,OAAO,QAAQ,OAAO,CAAC;AAChF,UAAM,KAAK,MAAM,OAAO,OAAO;AAC/B,SAAK,aAAa,MAAM,KAAK,MAAM,KAAK;AAAA,EAC1C;AACF;AAEA,IAAI,iBAA0C;AAEvC,SAAS,sBAAwC;AACtD,MAAI,CAAC,gBAAgB;AACnB,qBAAiB,IAAI,iBAAiB;AAAA,EACxC;AACA,SAAO;AACT;AAEO,SAAS,wBAA8B;AAC5C,MAAI,gBAAgB;AAClB,mBAAe,SAAS;AACxB,qBAAiB;AAAA,EACnB;AACF;","names":[]}

package/dist/embedding/fastembed-provider.d.ts

@@ -0,0 +1,19 @@
+import type { EmbedProvider } from '../providers/embed-provider.js';
+/**
+ * Native ONNX embedding provider using fastembed-rs Node bindings.
+ *
+ * Model: BGE-small-en-v1.5 (384-dim). First call to `warmup()` downloads
+ * the ONNX model to `${dataDir}/fastembed`. Subsequent runs reuse the cache.
+ * Replaces the legacy sentence-transformers Python subprocess.
+ */
+export declare class FastembedEmbedProvider implements EmbedProvider {
+    private model;
+    private modelPromise;
+    readonly modelId: string;
+    readonly dim: number;
+    constructor();
+    warmup(): Promise<void>;
+    private getModel;
+    embed(texts: string[]): Promise<Float32Array[]>;
+}
+//# sourceMappingURL=fastembed-provider.d.ts.map

package/dist/embedding/fastembed-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fastembed-provider.d.ts","sourceRoot":"","sources":["../../src/embedding/fastembed-provider.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAMpE;;;;;;GAMG;AACH,qBAAa,sBAAuB,YAAW,aAAa;IAC1D,OAAO,CAAC,KAAK,CAA8B;IAC3C,OAAO,CAAC,YAAY,CAAuC;IAC3D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;;IAOf,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAI7B,OAAO,CAAC,QAAQ;IAmBV,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAWtD"}

package/dist/embedding/fastembed-provider.js

@@ -0,0 +1,51 @@
+import { join } from "node:path";
+import { FlagEmbedding, EmbeddingModel } from "fastembed";
+import { getConfig } from "../config.js";
+import { createLogger } from "../logger.js";
+const log = createLogger("embedding");
+class FastembedEmbedProvider {
+  model = null;
+  modelPromise = null;
+  modelId;
+  dim;
+  constructor() {
+    this.modelId = "BGE-small-en-v1.5";
+    this.dim = 384;
+  }
+  async warmup() {
+    await this.getModel();
+  }
+  getModel() {
+    if (this.model) return Promise.resolve(this.model);
+    if (this.modelPromise) return this.modelPromise;
+    log.info("Loading embedding model", { modelId: this.modelId });
+    const cacheDir = join(getConfig().dataDir, "fastembed");
+    this.modelPromise = FlagEmbedding.init({
+      model: EmbeddingModel.BGESmallENV15,
+      cacheDir
+    }).then((m) => {
+      this.model = m;
+      log.info("Embedding model ready", { modelId: this.modelId, dim: this.dim });
+      return m;
+    }).catch((err) => {
+      this.modelPromise = null;
+      throw err;
+    });
+    return this.modelPromise;
+  }
+  async embed(texts) {
+    if (texts.length === 0) return [];
+    const model = await this.getModel();
+    const out = [];
+    for await (const batch of model.embed(texts, texts.length)) {
+      for (const vec of batch) {
+        out.push(Float32Array.from(vec));
+      }
+    }
+    return out;
+  }
+}
+export {
+  FastembedEmbedProvider
+};
+//# sourceMappingURL=fastembed-provider.js.map

package/dist/embedding/fastembed-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/embedding/fastembed-provider.ts"],"sourcesContent":["import { join } from 'node:path';\nimport { FlagEmbedding, EmbeddingModel } from 'fastembed';\nimport type { EmbedProvider } from '../providers/embed-provider.js';\nimport { getConfig } from '../config.js';\nimport { createLogger } from '../logger.js';\n\nconst log = createLogger('embedding');\n\n/**\n * Native ONNX embedding provider using fastembed-rs Node bindings.\n *\n * Model: BGE-small-en-v1.5 (384-dim). First call to `warmup()` downloads\n * the ONNX model to `${dataDir}/fastembed`. Subsequent runs reuse the cache.\n * Replaces the legacy sentence-transformers Python subprocess.\n */\nexport class FastembedEmbedProvider implements EmbedProvider {\n  private model: FlagEmbedding | null = null;\n  private modelPromise: Promise<FlagEmbedding> | null = null;\n  readonly modelId: string;\n  readonly dim: number;\n\n  constructor() {\n    this.modelId = 'BGE-small-en-v1.5';\n    this.dim = 384;\n  }\n\n  async warmup(): Promise<void> {\n    await this.getModel();\n  }\n\n  private getModel(): Promise<FlagEmbedding> {\n    if (this.model) return Promise.resolve(this.model);\n    if (this.modelPromise) return this.modelPromise;\n    log.info('Loading embedding model', { modelId: this.modelId });\n    const cacheDir = join(getConfig().dataDir, 'fastembed');\n    this.modelPromise = FlagEmbedding.init({\n      model: EmbeddingModel.BGESmallENV15,\n      cacheDir,\n    }).then(m => {\n      this.model = m;\n      log.info('Embedding model ready', { modelId: this.modelId, dim: this.dim });\n      return m;\n    }).catch(err => {\n      this.modelPromise = null;\n      throw err;\n    });\n    return this.modelPromise;\n  }\n\n  async embed(texts: string[]): Promise<Float32Array[]> {\n    if (texts.length === 0) return [];\n    const model = await this.getModel();\n    const out: Float32Array[] = [];\n    for await (const batch of model.embed(texts, texts.length)) {\n      for (const vec of batch) {\n        out.push(Float32Array.from(vec));\n      }\n    }\n    return out;\n  }\n}\n"],"mappings":"AAAA,SAAS,YAAY;AACrB,SAAS,eAAe,sBAAsB;AAE9C,SAAS,iBAAiB;AAC1B,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,WAAW;AAS7B,MAAM,uBAAgD;AAAA,EACnD,QAA8B;AAAA,EAC9B,eAA8C;AAAA,EAC7C;AAAA,EACA;AAAA,EAET,cAAc;AACZ,SAAK,UAAU;AACf,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,MAAM,SAAwB;AAC5B,UAAM,KAAK,SAAS;AAAA,EACtB;AAAA,EAEQ,WAAmC;AACzC,QAAI,KAAK,MAAO,QAAO,QAAQ,QAAQ,KAAK,KAAK;AACjD,QAAI,KAAK,aAAc,QAAO,KAAK;AACnC,QAAI,KAAK,2BAA2B,EAAE,SAAS,KAAK,QAAQ,CAAC;AAC7D,UAAM,WAAW,KAAK,UAAU,EAAE,SAAS,WAAW;AACtD,SAAK,eAAe,cAAc,KAAK;AAAA,MACrC,OAAO,eAAe;AAAA,MACtB;AAAA,IACF,CAAC,EAAE,KAAK,OAAK;AACX,WAAK,QAAQ;AACb,UAAI,KAAK,yBAAyB,EAAE,SAAS,KAAK,SAAS,KAAK,KAAK,IAAI,CAAC;AAC1E,aAAO;AAAA,IACT,CAAC,EAAE,MAAM,SAAO;AACd,WAAK,eAAe;AACpB,YAAM;AAAA,IACR,CAAC;AACD,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,MAAM,OAA0C;AACpD,QAAI,MAAM,WAAW,EAAG,QAAO,CAAC;AAChC,UAAM,QAAQ,MAAM,KAAK,SAAS;AAClC,UAAM,MAAsB,CAAC;AAC7B,qBAAiB,SAAS,MAAM,MAAM,OAAO,MAAM,MAAM,GAAG;AAC1D,iBAAW,OAAO,OAAO;AACvB,YAAI,KAAK,aAAa,KAAK,GAAG,CAAC;AAAA,MACjC;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/embedding/key-terms.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Extract key terms from page content and title for FTS5 query building.
+ * Prioritizes: title words > headings > bold text > first paragraph.
+ * Returns up to 20 deduplicated, lowercased, stopword-free terms.
+ */
+export declare function extractKeyTerms(content: string, title: string): string[];
+export declare function removeStopwords(words: string[]): string[];
+export declare function extractHeadings(content: string): string[];
+export declare function extractBoldText(content: string): string[];
+export declare function extractFirstParagraph(content: string): string;
+export declare function buildFTS5Query(terms: string[]): string;
+//# sourceMappingURL=key-terms.d.ts.map

package/dist/embedding/key-terms.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"key-terms.d.ts","sourceRoot":"","sources":["../../src/embedding/key-terms.ts"],"names":[],"mappings":"AAwBA;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAkDxE;AAiBD,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAEzD;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAazD;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAazD;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAqB7D;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAKtD"}

package/dist/embedding/key-terms.js

@@ -0,0 +1,234 @@
+import { createLogger } from "../logger.js";
+const log = createLogger("embedding");
+const STOPWORDS = /* @__PURE__ */ new Set([
+  "a",
+  "an",
+  "the",
+  "and",
+  "or",
+  "but",
+  "in",
+  "on",
+  "at",
+  "to",
+  "for",
+  "of",
+  "with",
+  "by",
+  "from",
+  "is",
+  "it",
+  "as",
+  "be",
+  "was",
+  "were",
+  "been",
+  "are",
+  "am",
+  "has",
+  "have",
+  "had",
+  "do",
+  "does",
+  "did",
+  "will",
+  "would",
+  "could",
+  "should",
+  "may",
+  "might",
+  "can",
+  "shall",
+  "not",
+  "no",
+  "nor",
+  "so",
+  "yet",
+  "both",
+  "each",
+  "few",
+  "more",
+  "most",
+  "other",
+  "some",
+  "such",
+  "than",
+  "too",
+  "very",
+  "just",
+  "about",
+  "above",
+  "after",
+  "again",
+  "all",
+  "also",
+  "any",
+  "because",
+  "before",
+  "below",
+  "between",
+  "during",
+  "further",
+  "here",
+  "how",
+  "into",
+  "its",
+  "itself",
+  "me",
+  "my",
+  "myself",
+  "once",
+  "only",
+  "our",
+  "ours",
+  "ourselves",
+  "out",
+  "over",
+  "own",
+  "same",
+  "she",
+  "he",
+  "her",
+  "him",
+  "his",
+  "hers",
+  "that",
+  "their",
+  "theirs",
+  "them",
+  "themselves",
+  "then",
+  "there",
+  "these",
+  "they",
+  "this",
+  "those",
+  "through",
+  "under",
+  "until",
+  "up",
+  "we",
+  "what",
+  "when",
+  "where",
+  "which",
+  "while",
+  "who",
+  "whom",
+  "why",
+  "you",
+  "your",
+  "yours",
+  "yourself",
+  "i",
+  "if"
+]);
+const MAX_TERMS = 20;
+const FIRST_PARAGRAPH_MAX_CHARS = 200;
+function extractKeyTerms(content, title) {
+  try {
+    if (!content.trim() && !title.trim()) {
+      return [];
+    }
+    const allTerms = [];
+    const titleWords = tokenize(title);
+    allTerms.push(...titleWords);
+    const headings = extractHeadings(content);
+    for (const heading of headings) {
+      allTerms.push(...tokenize(heading));
+    }
+    const boldPhrases = extractBoldText(content);
+    for (const phrase of boldPhrases) {
+      allTerms.push(...tokenize(phrase));
+    }
+    const firstPara = extractFirstParagraph(content);
+    if (firstPara) {
+      allTerms.push(...tokenize(firstPara));
+    }
+    const cleaned = removeStopwords(allTerms);
+    const seen = /* @__PURE__ */ new Set();
+    const unique = [];
+    for (const term of cleaned) {
+      if (!seen.has(term)) {
+        seen.add(term);
+        unique.push(term);
+      }
+      if (unique.length >= MAX_TERMS) break;
+    }
+    log.debug("extracted key terms", {
+      titleTerms: titleWords.length,
+      headingTerms: headings.length,
+      boldTerms: boldPhrases.length,
+      uniqueTerms: unique.length
+    });
+    return unique;
+  } catch (err) {
+    log.error("key term extraction failed", { error: String(err) });
+    return [];
+  }
+}
+function tokenize(text) {
+  if (!text) return [];
+  let cleaned = text.replace(/https?:\/\/[^\s)]+/g, "");
+  cleaned = cleaned.replace(/```[\s\S]*?```/g, "");
+  cleaned = cleaned.replace(/`[^`]+`/g, "");
+  cleaned = cleaned.replace(/^#{1,6}\s+/gm, "");
+  cleaned = cleaned.replace(/\*{1,3}|_{1,3}/g, "");
+  return cleaned.toLowerCase().split(/[^a-z0-9-]+/).filter((w) => w.length > 1 && !/^\d+$/.test(w));
+}
+function removeStopwords(words) {
+  return words.filter((w) => w.length > 1 && !STOPWORDS.has(w.toLowerCase()));
+}
+function extractHeadings(content) {
+  if (!content) return [];
+  const headings = [];
+  const regex = /^#{1,3}\s+(.+)$/gm;
+  let match;
+  while ((match = regex.exec(content)) !== null) {
+    const text = match[1].trim();
+    if (text) headings.push(text);
+  }
+  return headings;
+}
+function extractBoldText(content) {
+  if (!content) return [];
+  const bold = [];
+  const regex = /\*\*(.+?)\*\*|__(.+?)__/g;
+  let match;
+  while ((match = regex.exec(content)) !== null) {
+    const text = (match[1] || match[2]).trim();
+    if (text) bold.push(text);
+  }
+  return bold;
+}
+function extractFirstParagraph(content) {
+  if (!content) return "";
+  const lines = content.split("\n");
+  const paragraphLines = [];
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) continue;
+    if (trimmed.startsWith("```")) continue;
+    paragraphLines.push(trimmed);
+    const joined = paragraphLines.join(" ");
+    if (joined.length >= FIRST_PARAGRAPH_MAX_CHARS) {
+      return joined.slice(0, FIRST_PARAGRAPH_MAX_CHARS);
+    }
+    break;
+  }
+  return paragraphLines.join(" ").slice(0, FIRST_PARAGRAPH_MAX_CHARS);
+}
+function buildFTS5Query(terms) {
+  if (terms.length === 0) return "";
+  const escaped = terms.map((t) => t.replace(/['"()]/g, ""));
+  return escaped.filter(Boolean).join(" OR ");
+}
+export {
+  buildFTS5Query,
+  extractBoldText,
+  extractFirstParagraph,
+  extractHeadings,
+  extractKeyTerms,
+  removeStopwords
+};
+//# sourceMappingURL=key-terms.js.map

package/dist/embedding/key-terms.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/embedding/key-terms.ts"],"sourcesContent":["import { createLogger } from '../logger.js';\n\nconst log = createLogger('embedding');\n\nconst STOPWORDS = new Set([\n  'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',\n  'of', 'with', 'by', 'from', 'is', 'it', 'as', 'be', 'was', 'were',\n  'been', 'are', 'am', 'has', 'have', 'had', 'do', 'does', 'did', 'will',\n  'would', 'could', 'should', 'may', 'might', 'can', 'shall', 'not', 'no',\n  'nor', 'so', 'yet', 'both', 'each', 'few', 'more', 'most', 'other',\n  'some', 'such', 'than', 'too', 'very', 'just', 'about', 'above', 'after',\n  'again', 'all', 'also', 'any', 'because', 'before', 'below', 'between',\n  'during', 'further', 'here', 'how', 'into', 'its', 'itself', 'me', 'my',\n  'myself', 'once', 'only', 'our', 'ours', 'ourselves', 'out', 'over',\n  'own', 'same', 'she', 'he', 'her', 'him', 'his', 'hers', 'that', 'their',\n  'theirs', 'them', 'themselves', 'then', 'there', 'these', 'they', 'this',\n  'those', 'through', 'under', 'until', 'up', 'we', 'what', 'when', 'where',\n  'which', 'while', 'who', 'whom', 'why', 'you', 'your', 'yours', 'yourself',\n  'i', 'if',\n]);\n\nconst MAX_TERMS = 20;\nconst FIRST_PARAGRAPH_MAX_CHARS = 200;\n\n/**\n * Extract key terms from page content and title for FTS5 query building.\n * Prioritizes: title words > headings > bold text > first paragraph.\n * Returns up to 20 deduplicated, lowercased, stopword-free terms.\n */\nexport function extractKeyTerms(content: string, title: string): string[] {\n  try {\n    if (!content.trim() && !title.trim()) {\n      return [];\n    }\n\n    const allTerms: string[] = [];\n\n    const titleWords = tokenize(title);\n    allTerms.push(...titleWords);\n\n    const headings = extractHeadings(content);\n    for (const heading of headings) {\n      allTerms.push(...tokenize(heading));\n    }\n\n    const boldPhrases = extractBoldText(content);\n    for (const phrase of boldPhrases) {\n      allTerms.push(...tokenize(phrase));\n    }\n\n    const firstPara = extractFirstParagraph(content);\n    if (firstPara) {\n      allTerms.push(...tokenize(firstPara));\n    }\n\n    const cleaned = removeStopwords(allTerms);\n    const seen = new Set<string>();\n    const unique: string[] = [];\n\n    for (const term of cleaned) {\n      if (!seen.has(term)) {\n        seen.add(term);\n        unique.push(term);\n      }\n      if (unique.length >= MAX_TERMS) break;\n    }\n\n    log.debug('extracted key terms', {\n      titleTerms: titleWords.length,\n      headingTerms: headings.length,\n      boldTerms: boldPhrases.length,\n      uniqueTerms: unique.length,\n    });\n\n    return unique;\n  } catch (err) {\n    log.error('key term extraction failed', { error: String(err) });\n    return [];\n  }\n}\n\nfunction tokenize(text: string): string[] {\n  if (!text) return [];\n\n  let cleaned = text.replace(/https?:\\/\\/[^\\s)]+/g, '');\n  cleaned = cleaned.replace(/```[\\s\\S]*?```/g, '');\n  cleaned = cleaned.replace(/`[^`]+`/g, '');\n  cleaned = cleaned.replace(/^#{1,6}\\s+/gm, '');\n  cleaned = cleaned.replace(/\\*{1,3}|_{1,3}/g, '');\n\n  return cleaned\n    .toLowerCase()\n    .split(/[^a-z0-9-]+/)\n    .filter(w => w.length > 1 && !/^\\d+$/.test(w));\n}\n\nexport function removeStopwords(words: string[]): string[] {\n  return words.filter(w => w.length > 1 && !STOPWORDS.has(w.toLowerCase()));\n}\n\nexport function extractHeadings(content: string): string[] {\n  if (!content) return [];\n\n  const headings: string[] = [];\n  const regex = /^#{1,3}\\s+(.+)$/gm;\n  let match: RegExpExecArray | null;\n\n  while ((match = regex.exec(content)) !== null) {\n    const text = match[1].trim();\n    if (text) headings.push(text);\n  }\n\n  return headings;\n}\n\nexport function extractBoldText(content: string): string[] {\n  if (!content) return [];\n\n  const bold: string[] = [];\n  const regex = /\\*\\*(.+?)\\*\\*|__(.+?)__/g;\n  let match: RegExpExecArray | null;\n\n  while ((match = regex.exec(content)) !== null) {\n    const text = (match[1] || match[2]).trim();\n    if (text) bold.push(text);\n  }\n\n  return bold;\n}\n\nexport function extractFirstParagraph(content: string): string {\n  if (!content) return '';\n\n  const lines = content.split('\\n');\n  const paragraphLines: string[] = [];\n\n  for (const line of lines) {\n    const trimmed = line.trim();\n    if (!trimmed || trimmed.startsWith('#')) continue;\n    if (trimmed.startsWith('```')) continue;\n\n    paragraphLines.push(trimmed);\n    const joined = paragraphLines.join(' ');\n    if (joined.length >= FIRST_PARAGRAPH_MAX_CHARS) {\n      return joined.slice(0, FIRST_PARAGRAPH_MAX_CHARS);\n    }\n\n    break;\n  }\n\n  return paragraphLines.join(' ').slice(0, FIRST_PARAGRAPH_MAX_CHARS);\n}\n\nexport function buildFTS5Query(terms: string[]): string {\n  if (terms.length === 0) return '';\n\n  const escaped = terms.map(t => t.replace(/['\"()]/g, ''));\n  return escaped.filter(Boolean).join(' OR ');\n}\n"],"mappings":"AAAA,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,WAAW;AAEpC,MAAM,YAAY,oBAAI,IAAI;AAAA,EACxB;AAAA,EAAK;AAAA,EAAM;AAAA,EAAO;AAAA,EAAO;AAAA,EAAM;AAAA,EAAO;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAC9D;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAO;AAAA,EAC3D;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAM;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAO;AAAA,EAChE;AAAA,EAAS;AAAA,EAAS;AAAA,EAAU;AAAA,EAAO;AAAA,EAAS;AAAA,EAAO;AAAA,EAAS;AAAA,EAAO;AAAA,EACnE;AAAA,EAAO;AAAA,EAAM;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAQ;AAAA,EAC3D;AAAA,EAAQ;AAAA,EAAQ;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAQ;AAAA,EAAS;AAAA,EAAS;AAAA,EACjE;AAAA,EAAS;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAW;AAAA,EAAU;AAAA,EAAS;AAAA,EAC7D;AAAA,EAAU;AAAA,EAAW;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAU;AAAA,EAAM;AAAA,EACnE;AAAA,EAAU;AAAA,EAAQ;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAa;AAAA,EAAO;AAAA,EAC7D;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAM;AAAA,EAAO;AAAA,EAAO;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAQ;AAAA,EACjE;AAAA,EAAU;AAAA,EAAQ;AAAA,EAAc;AAAA,EAAQ;AAAA,EAAS;AAAA,EAAS;AAAA,EAAQ;AAAA,EAClE;AAAA,EAAS;AAAA,EAAW;AAAA,EAAS;AAAA,EAAS;AAAA,EAAM;AAAA,EAAM;AAAA,EAAQ;AAAA,EAAQ;AAAA,EAClE;AAAA,EAAS;AAAA,EAAS;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAO;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAS;AAAA,EAChE;AAAA,EAAK;AACP,CAAC;AAED,MAAM,YAAY;AAClB,MAAM,4BAA4B;AAO3B,SAAS,gBAAgB,SAAiB,OAAyB;AACxE,MAAI;AACF,QAAI,CAAC,QAAQ,KAAK,KAAK,CAAC,MAAM,KAAK,GAAG;AACpC,aAAO,CAAC;AAAA,IACV;AAEA,UAAM,WAAqB,CAAC;AAE5B,UAAM,aAAa,SAAS,KAAK;AACjC,aAAS,KAAK,GAAG,UAAU;AAE3B,UAAM,WAAW,gBAAgB,OAAO;AACxC,eAAW,WAAW,UAAU;AAC9B,eAAS,KAAK,GAAG,SAAS,OAAO,CAAC;AAAA,IACpC;AAEA,UAAM,cAAc,gBAAgB,OAAO;AAC3C,eAAW,UAAU,aAAa;AAChC,eAAS,KAAK,GAAG,SAAS,MAAM,CAAC;AAAA,IACnC;AAEA,UAAM,YAAY,sBAAsB,OAAO;AAC/C,QAAI,WAAW;AACb,eAAS,KAAK,GAAG,SAAS,SAAS,CAAC;AAAA,IACtC;AAEA,UAAM,UAAU,gBAAgB,QAAQ;AACxC,UAAM,OAAO,oBAAI,IAAY;AAC7B,UAAM,SAAmB,CAAC;AAE1B,eAAW,QAAQ,SAAS;AAC1B,UAAI,CAAC,KAAK,IAAI,IAAI,GAAG;AACnB,aAAK,IAAI,IAAI;AACb,eAAO,KAAK,IAAI;AAAA,MAClB;AACA,UAAI,OAAO,UAAU,UAAW;AAAA,IAClC;AAEA,QAAI,MAAM,uBAAuB;AAAA,MAC/B,YAAY,WAAW;AAAA,MACvB,cAAc,SAAS;AAAA,MACvB,WAAW,YAAY;AAAA,MACvB,aAAa,OAAO;AAAA,IACtB,CAAC;AAED,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,QAAI,MAAM,8BAA8B,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAC9D,WAAO,CAAC;AAAA,EACV;AACF;AAEA,SAAS,SAAS,MAAwB;AACxC,MAAI,CAAC,KAAM,QAAO,CAAC;AAEnB,MAAI,UAAU,KAAK,QAAQ,uBAAuB,EAAE;AACpD,YAAU,QAAQ,QAAQ,mBAAmB,EAAE;AAC/C,YAAU,QAAQ,QAAQ,YAAY,EAAE;AACxC,YAAU,QAAQ,QAAQ,gBAAgB,EAAE;AAC5C,YAAU,QAAQ,QAAQ,mBAAmB,EAAE;AAE/C,SAAO,QACJ,YAAY,EACZ,MAAM,aAAa,EACnB,OAAO,OAAK,EAAE,SAAS,KAAK,CAAC,QAAQ,KAAK,CAAC,CAAC;AACjD;AAEO,SAAS,gBAAgB,OAA2B;AACzD,SAAO,MAAM,OAAO,OAAK,EAAE,SAAS,KAAK,CAAC,UAAU,IAAI,EAAE,YAAY,CAAC,CAAC;AAC1E;AAEO,SAAS,gBAAgB,SAA2B;AACzD,MAAI,CAAC,QAAS,QAAO,CAAC;AAEtB,QAAM,WAAqB,CAAC;AAC5B,QAAM,QAAQ;AACd,MAAI;AAEJ,UAAQ,QAAQ,MAAM,KAAK,OAAO,OAAO,MAAM;AAC7C,UAAM,OAAO,MAAM,CAAC,EAAE,KAAK;AAC3B,QAAI,KAAM,UAAS,KAAK,IAAI;AAAA,EAC9B;AAEA,SAAO;AACT;AAEO,SAAS,gBAAgB,SAA2B;AACzD,MAAI,CAAC,QAAS,QAAO,CAAC;AAEtB,QAAM,OAAiB,CAAC;AACxB,QAAM,QAAQ;AACd,MAAI;AAEJ,UAAQ,QAAQ,MAAM,KAAK,OAAO,OAAO,MAAM;AAC7C,UAAM,QAAQ,MAAM,CAAC,KAAK,MAAM,CAAC,GAAG,KAAK;AACzC,QAAI,KAAM,MAAK,KAAK,IAAI;AAAA,EAC1B;AAEA,SAAO;AACT;AAEO,SAAS,sBAAsB,SAAyB;AAC7D,MAAI,CAAC,QAAS,QAAO;AAErB,QAAM,QAAQ,QAAQ,MAAM,IAAI;AAChC,QAAM,iBAA2B,CAAC;AAElC,aAAW,QAAQ,OAAO;AACxB,UAAM,UAAU,KAAK,KAAK;AAC1B,QAAI,CAAC,WAAW,QAAQ,WAAW,GAAG,EAAG;AACzC,QAAI,QAAQ,WAAW,KAAK,EAAG;AAE/B,mBAAe,KAAK,OAAO;AAC3B,UAAM,SAAS,eAAe,KAAK,GAAG;AACtC,QAAI,OAAO,UAAU,2BAA2B;AAC9C,aAAO,OAAO,MAAM,GAAG,yBAAyB;AAAA,IAClD;AAEA;AAAA,EACF;AAEA,SAAO,eAAe,KAAK,GAAG,EAAE,MAAM,GAAG,yBAAyB;AACpE;AAEO,SAAS,eAAe,OAAyB;AACtD,MAAI,MAAM,WAAW,EAAG,QAAO;AAE/B,QAAM,UAAU,MAAM,IAAI,OAAK,EAAE,QAAQ,WAAW,EAAE,CAAC;AACvD,SAAO,QAAQ,OAAO,OAAO,EAAE,KAAK,MAAM;AAC5C;","names":[]}

package/dist/extraction/boilerplate.d.ts

@@ -0,0 +1,15 @@
+export declare const BOILERPLATE_TEXT_EQUALITY: ReadonlyArray<string>;
+export declare const BOILERPLATE_TEXT_PATTERNS: ReadonlyArray<RegExp>;
+export declare const BOILERPLATE_SELECTORS: ReadonlyArray<string>;
+export interface BoilerplateDocument {
+    querySelectorAll(selector: string): ArrayLike<BoilerplateElement>;
+}
+interface BoilerplateElement {
+    parentNode: {
+        removeChild(child: BoilerplateElement): void;
+    } | null;
+}
+export declare function stripBoilerplateMarkdown(md: string): string;
+export declare function stripBoilerplateDom(document: BoilerplateDocument): void;
+export {};
+//# sourceMappingURL=boilerplate.d.ts.map

package/dist/extraction/boilerplate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"boilerplate.d.ts","sourceRoot":"","sources":["../../src/extraction/boilerplate.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,yBAAyB,EAAE,aAAa,CAAC,MAAM,CAQ3D,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,aAAa,CAAC,MAAM,CAE3D,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,aAAa,CAAC,MAAM,CAUvD,CAAC;AAEF,MAAM,WAAW,mBAAmB;IAClC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;CACnE;AAED,UAAU,kBAAkB;IAC1B,UAAU,EAAE;QAAE,WAAW,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;CACrE;AAED,wBAAgB,wBAAwB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAU3D;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,GAAG,IAAI,CASvE"}