extract-webpage 1.2.119 → 1.2.121

package/dist/html-to-cite/extract-source.d.ts

@@ -1,3 +1,7 @@
+/**
+ * @fileoverview Utility for extracting the publication/source name from a
+ * document's `og:site_name` meta tag or class-based markers.
+ */
 /**
  * Extract source from document using common class names
  *

package/dist/html-to-cite/metadata-to-cite.d.ts

@@ -1,3 +1,7 @@
+/**
+ * @fileoverview Extracts citation metadata (author, date, title, source) from
+ * a document's `<meta>` tags using a list of commonly used property/name attributes.
+ */
 export interface CiteMetadata {
     author?: string;
     date?: string;

package/dist/suggest-next-words/autocomplete-ai.d.ts

@@ -0,0 +1,4 @@
+/**
+ * @fileoverview Local small-model next-word prediction utility (DistilGPT2 via
+ * HuggingFace Transformers). Currently disabled/commented out; kept for reference.
+ */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "extract-webpage",
-  "version": "1.2.119",
+  "version": "1.2.121",
   "module": "./dist/extract-webpage.es.js",
   "description": "Search, extract, cite, and outline the web for a topic with AI Research Agent.",
   "author": "vtempest <grokthiscontact@gmail.com>",
@@ -41,6 +41,7 @@
     "test": "vitest",
     "ship": "npm run build && npx standard-version --release-as patch; rm CHANGELOG.md; npm publish",
     "test-ui": "vitest --ui --watch",
+    "test:coverage": "vitest run --coverage",
     "make": "rm -rf dist/*; NODE_OPTIONS=--max-old-space-size=15192 BUN_JSC_forceRAMSize=15192  vite build "
   },
   "peerDependencies": {
@@ -54,6 +55,7 @@
   "devDependencies": {
     "@tsconfig/svelte": "^5.0.8",
     "@types/node": "^22.0.0",
+    "@vitest/coverage-v8": "^4.0.18",
     "@vitest/ui": "^4.0.18",
     "axios": "^1.13.6",
     "clsx": "^2.1.1",
@@ -72,10 +74,10 @@
   "dependencies": {
     "@huggingface/transformers": "^3.8.1",
     "ai": "^5.0.0",
-    "chat-agent-toolkit": "^1.2.119",
+    "chat-agent-toolkit": "^1.2.121",
     "chrono-node": "^2.9.0",
     "drizzle-orm": "^0.45.1",
-    "extract-pdf": "^0.1.106",
+    "extract-pdf": "^0.1.108",
     "extract-youtube": "^1.0.103",
     "html-entities": "^2.6.0",
     "js-yaml": "^4.1.1",

package/src/config/serverRegistry.ts

@@ -1,3 +1,7 @@
+/**
+ * @fileoverview Convenience accessors for reading server-side config values
+ * (model providers, search backends, scrape limits) from the shared ConfigManager.
+ */
 import configManager from "./index";
 import type { ConfigModelProvider } from "./types";
 

package/src/fs-mock.js

@@ -1,3 +1,7 @@
+/**
+ * @fileoverview No-op `fs` module mock used to stub out Node's filesystem API
+ * in bundled/browser builds where real filesystem access is unavailable.
+ */
 export const promises = {
   mkdir: async () => {},
   readFile: async () => "",

package/src/html-to-cite/extract-source.ts

@@ -1,4 +1,9 @@
 
+/**
+ * @fileoverview Utility for extracting the publication/source name from a
+ * document's `og:site_name` meta tag or class-based markers.
+ */
+
 /**
  * Extract source from document using common class names
  *

package/src/html-to-cite/metadata-to-cite.ts

@@ -1,3 +1,7 @@
+/**
+ * @fileoverview Extracts citation metadata (author, date, title, source) from
+ * a document's `<meta>` tags using a list of commonly used property/name attributes.
+ */
 
 export interface CiteMetadata {
   author?: string;

package/src/seektopic/vector-search.ts

@@ -1,3 +1,8 @@
+/**
+ * @fileoverview Text embedding and semantic similarity utilities using
+ * HuggingFace Transformers (local MiniLM model) or the remote Inference API.
+ * Provides cosine-similarity-based document reranking against one or more queries.
+ */
 import grab from "../utils/grab";
 
 /**

package/src/suggest-next-words/autocomplete-ai.ts

@@ -1,3 +1,7 @@
+/**
+ * @fileoverview Local small-model next-word prediction utility (DistilGPT2 via
+ * HuggingFace Transformers). Currently disabled/commented out; kept for reference.
+ */
 // import { pipeline, type TextGenerationSingle } from "@huggingface/transformers";
 
 // /**

package/src/url-to-content/url-to-html.ts

@@ -1,4 +1,8 @@
 // @ts-nocheck
+/**
+ * @fileoverview Web crawler utility ("Tardigrade") that fetches a URL's raw HTML
+ * with bot-detection handling and Cloudflare/JINA fallbacks, plus robots.txt checking.
+ */
 import { convertHTMLToBasicHTML } from "../html-to-content/html-to-basic-html";
 import { convertMarkdownToFormattedHTML } from "../html-to-content/html-utils";
 import grab from "../utils/grab";

package/src/utils/documents.ts

@@ -1,3 +1,7 @@
+/**
+ * @fileoverview Fetches web pages from a list of links, strips them to plain
+ * text, and splits them into `Document` chunks for downstream use.
+ */
 import axios from 'axios';
 import { splitTextIntoChunks, type Document } from 'chat-agent-toolkit';