extract-webpage 1.2.5

package/src/config/index.ts

@@ -0,0 +1,233 @@
+/**
+ * @fileoverview Simplified ConfigManager for the research-agent package.
+ * Manages model providers and search config in memory.
+ */
+import type { ConfigModelProvider, MCPServerConfig, Config, UIConfigSections } from "./types";
+import { getEnv } from "./env";
+
+const hashObj = (obj: { [key: string]: any }) => {
+  const str = JSON.stringify(obj, Object.keys(obj).sort());
+  let hash = 0;
+  for (let i = 0; i < str.length; i++) {
+    hash = (hash << 5) - hash + str.charCodeAt(i);
+    hash |= 0;
+  }
+  return String(Math.abs(hash).toString(36));
+};
+
+class ConfigManager {
+  configVersion = 1;
+  currentConfig: Config = {
+    version: this.configVersion,
+    setupComplete: getEnv("SETUP_COMPLETE") === "true" || false,
+    preferences: {},
+    personalization: {},
+    modelProviders: [],
+    mcpServers: [],
+    search: {
+      searxngURL: "",
+      tavilyApiKey: "",
+      sourceScrapeCount: 3,
+      sourceScrapeTimeout: 5,
+    },
+  };
+  uiConfigSections: UIConfigSections = {
+    preferences: [],
+    personalization: [],
+    modelProviders: [],
+    mcpServers: [],
+    search: [
+      {
+        name: "SearXNG URL",
+        key: "searxngURL",
+        type: "string",
+        required: false,
+        description: "The URL of your SearXNG instance",
+        placeholder: "http://localhost:4000",
+        default: "",
+        scope: "server",
+        env: "SEARXNG_API_URL",
+      },
+      {
+        name: "Tavily API Key",
+        key: "tavilyApiKey",
+        type: "password",
+        required: false,
+        description:
+          "Optional. Enter your own Tavily API key to override the site default.",
+        placeholder: "tvly-...",
+        default: "",
+        scope: "server",
+        env: "TAVILY_API_KEY",
+      },
+      {
+        name: "Source pages to scrape",
+        key: "sourceScrapeCount",
+        type: "select",
+        options: [
+          { name: "Disabled (snippet only)", value: "0" },
+          { name: "1 page", value: "1" },
+          { name: "2 pages", value: "2" },
+          { name: "3 pages (default)", value: "3" },
+          { name: "5 pages", value: "5" },
+        ],
+        required: false,
+        description: "Number of top search result URLs to fully scrape.",
+        default: "3",
+        scope: "server",
+      },
+      {
+        name: "Scrape timeout (seconds)",
+        key: "sourceScrapeTimeout",
+        type: "select",
+        options: [
+          { name: "3 seconds", value: "3" },
+          { name: "5 seconds (default)", value: "5" },
+          { name: "10 seconds", value: "10" },
+          { name: "15 seconds", value: "15" },
+          { name: "20 seconds", value: "20" },
+        ],
+        required: false,
+        description: "Maximum time to wait when scraping each source URL.",
+        default: "5",
+        scope: "server",
+      },
+    ],
+  };
+
+  private initialized = false;
+
+  constructor() {
+    // Don't initialize in constructor to avoid circular dependency
+    // Initialize lazily when config is first accessed
+  }
+
+  private ensureInitialized() {
+    if (this.initialized) return;
+    this.initialize();
+    this.initialized = true;
+  }
+
+  private initialize() {
+    // Search config from env
+    this.uiConfigSections.search.forEach((f) => {
+      if (f.env && !this.currentConfig.search[f.key]) {
+        this.currentConfig.search[f.key] =
+          getEnv(f.env) ?? f.default ?? "";
+      }
+    });
+  }
+
+  public getConfig(key: string, defaultValue?: any): any {
+    this.ensureInitialized();
+    const nested = key.split(".");
+    let obj: any = this.currentConfig;
+
+    for (let i = 0; i < nested.length; i++) {
+      const part = nested[i];
+      if (obj == null) return defaultValue;
+      obj = obj[part];
+    }
+
+    return obj === undefined ? defaultValue : obj;
+  }
+
+  public updateConfig(key: string, val: any) {
+    const parts = key.split(".");
+    if (parts.length === 0) return;
+
+    let target: any = this.currentConfig;
+    for (let i = 0; i < parts.length - 1; i++) {
+      const part = parts[i];
+      if (target[part] === null || typeof target[part] !== "object") {
+        target[part] = {};
+      }
+      target = target[part];
+    }
+
+    const finalKey = parts[parts.length - 1];
+    target[finalKey] = val;
+  }
+
+  public addModelProvider(type: string, config: any) {
+    this.ensureInitialized();
+    const hash = hashObj(config);
+    const section = this.uiConfigSections.modelProviders.find(s => s.key === type);
+    const name = section?.name || type;
+
+    const newModelProvider: ConfigModelProvider = {
+      id: hash,
+      name,
+      type,
+      config,
+      chatModels: [],
+      hash: hash,
+    };
+
+    this.currentConfig.modelProviders.push(newModelProvider);
+    return newModelProvider;
+  }
+
+  public removeModelProvider(id: string) {
+    this.currentConfig.modelProviders =
+      this.currentConfig.modelProviders.filter((p) => p.id !== id);
+  }
+
+  public async updateModelProvider(id: string, config: any) {
+    const provider = this.currentConfig.modelProviders.find(
+      (p) => p.id === id,
+    );
+    if (!provider) throw new Error("Provider not found");
+
+    provider.config = config;
+    return provider;
+  }
+
+  public addProviderModel(providerId: string, type: "chat", model: any) {
+    const provider = this.currentConfig.modelProviders.find(
+      (p) => p.id === providerId,
+    );
+    if (!provider) throw new Error("Invalid provider id");
+
+    delete model.type;
+    provider.chatModels.push(model);
+    return model;
+  }
+
+  public removeProviderModel(
+    providerId: string,
+    type: "chat",
+    modelKey: string,
+  ) {
+    const provider = this.currentConfig.modelProviders.find(
+      (p) => p.id === providerId,
+    );
+    if (!provider) throw new Error("Invalid provider id");
+
+    provider.chatModels = provider.chatModels.filter((m) => m.key !== modelKey);
+  }
+
+  public isSetupComplete() {
+    return this.currentConfig.setupComplete;
+  }
+
+  public markSetupComplete() {
+    if (!this.currentConfig.setupComplete) {
+      this.currentConfig.setupComplete = true;
+    }
+  }
+
+  public getUIConfigSections(): UIConfigSections {
+    this.ensureInitialized();
+    return this.uiConfigSections;
+  }
+
+  public getCurrentConfig(): Config {
+    this.ensureInitialized();
+    return JSON.parse(JSON.stringify(this.currentConfig));
+  }
+}
+
+const configManager = new ConfigManager();
+
+export default configManager;

package/src/config/serverRegistry.ts

@@ -0,0 +1,24 @@
+import configManager from "./index";
+import type { ConfigModelProvider } from "./types";
+
+export const getConfiguredModelProviders = (): ConfigModelProvider[] => {
+  return configManager.getConfig("modelProviders", []);
+};
+
+export const getConfiguredModelProviderById = (
+  id: string,
+): ConfigModelProvider | undefined => {
+  return getConfiguredModelProviders().find((p) => p.id === id) ?? undefined;
+};
+
+export const getSearxngURL = () =>
+  configManager.getConfig("search.searxngURL", "");
+
+export const getTavilyApiKey = () =>
+  configManager.getConfig("search.tavilyApiKey", "");
+
+export const getSourceScrapeCount = (): number =>
+  parseInt(configManager.getConfig("search.sourceScrapeCount", "3"), 10) || 3;
+
+export const getSourceScrapeTimeout = (): number =>
+  parseInt(configManager.getConfig("search.sourceScrapeTimeout", "5"), 10) || 5;

package/src/config/types.ts

@@ -0,0 +1,17 @@
+/**
+ * Re-export config types from the package's central type definitions.
+ */
+export type {
+  UIConfigField,
+  Config,
+  EnvMap,
+  UIConfigSections,
+  SelectUIConfigField,
+  StringUIConfigField,
+  ModelProviderUISection,
+  MCPServerUISection,
+  ConfigModelProvider,
+  MCPServerConfig,
+  TextareaUIConfigField,
+  SwitchUIConfigField,
+} from "../types";

package/src/fs-mock.js

@@ -0,0 +1,22 @@
+export const promises = {
+  mkdir: async () => {},
+  readFile: async () => "",
+  writeFile: async () => {},
+  appendFile: async () => {},
+  stat: async () => ({ isDirectory: () => false }),
+};
+
+export const mkdir = promises.mkdir;
+export const readFile = promises.readFile;
+export const writeFile = promises.writeFile;
+export const appendFile = promises.appendFile;
+export const stat = promises.stat;
+
+export default {
+  promises,
+  mkdir,
+  readFile,
+  writeFile,
+  appendFile,
+  stat,
+};

package/src/global.d.ts

@@ -0,0 +1,8 @@
+declare module "@huggingface/transformers" {
+  export function pipeline(task: string, model: string, options?: any): Promise<any>;
+}
+
+declare module "*.csv?raw" {
+  const content: string;
+  export default content;
+}

package/src/html-to-cite/extract-author.ts

@@ -0,0 +1,125 @@
+/**
+ * @fileoverview Utility for identifying and extracting author names from HTML metadata and content.
+ * Handles individual authors, multiple authors, and organizational authors.
+ */
+import { extractHumanName } from "./human-names-recognize";
+
+// https://www.scribbr.com/citation/generator/folders/2rx21jyIjZIKRrcwLk2oXE/lists/4OeJQ4euxzyTk9BiPTRjwn/
+const AUTHOR_META_TAGS = [
+  "byl",
+  "clmst",
+  "dc.author",
+  "dcsext.author",
+  "dc.creator",
+  "rbauthors",
+  "authors",
+  "sailthru.author",
+  "article:author",
+  "parsely-author",
+];
+
+const AUTHOR_SELECTORS = [
+  ".entry .entry-author",
+  ".author.vcard .fn",
+  ".author .vcard .fn",
+  ".byline.vcard .fn",
+  ".byline .vcard .fn",
+  ".byline .by .author",
+  ".byline .by",
+  ".byline .author",
+  ".post-author.vcard",
+  ".post-author .vcard",
+  "a[rel=author]",
+  "#by_author",
+  ".by_author",
+  "#entryAuthor",
+  ".entryAuthor",
+  ".byline a[href*=author]",
+  "#author .authorname",
+  ".author .authorname",
+  "#author",
+  ".author",
+  ".articleauthor",
+  ".ArticleAuthor",
+  ".byline",
+];
+
+const AUTHOR_MAX_LENGTH = 300;
+const BYLINE_REGEX = /^[\n\s]*By\s*:?\s*/i;
+const CLEAN_AUTHOR_RE = /^\s*(posted |written )?by\s*:?\s*(.*)/i;
+
+/**
+ * Extracts the author from the document and validates it as a human name
+ *
+ * @param {Document} document
+ * @returns {object|null} author_cite, author_short, author_type - or null if no valid author found
+*/
+export function extractAuthor(document) {
+
+
+  // 1. Check meta tags
+  for (const tag of AUTHOR_META_TAGS) {
+    const metaElement = document.querySelector(
+      `meta[name="${tag}"], meta[property="${tag}"]`
+    );
+    if (metaElement) {
+      const author = extractAndValidateHumanName(
+        metaElement.getAttribute("content")
+      );
+      if (author) return author;
+    }
+  }
+
+  // 2. Check selectors
+  for (const selector of AUTHOR_SELECTORS) {
+    const element = document.querySelector(selector);
+    if (element) {
+      const author = extractAndValidateHumanName(element.textContent);
+      if (author) return author;
+    }
+  }
+
+  // 4. Broader search in divs and spans
+  const elements = [
+    ...document.getElementsByTagName("div"),
+    ...document.getElementsByTagName("span"),
+  ];
+  for (const element of elements) {
+    if (
+      element.id.match(/author|byline/i) ||
+      element?.className?.match(/author|byline/i)
+    ) {
+      const author = extractAndValidateHumanName(element.innerText);
+      if (author) return author;
+    }
+  }
+
+
+  return null;
+}
+
+// Helper function to clean and validate author string
+const validateAuthor = (author) => {
+  if (!author) return null;
+
+  if (author.startsWith("@")) author = author.slice(1);
+  if (author.startsWith("http")) return null;
+
+  author = author?.replace(CLEAN_AUTHOR_RE, "$2").trim();
+  return author.length > 0 && author.length < AUTHOR_MAX_LENGTH
+    ? author
+    : null;
+};
+
+// Function to extract and validate human name
+const extractAndValidateHumanName = (author) => {
+  const validatedAuthor = validateAuthor(author);
+  if (validatedAuthor) {
+    return extractHumanName(validatedAuthor, {
+      formatCiteShortenAuthor: false
+    });
+  }
+  return null;
+};
+
+

package/src/html-to-cite/extract-cite.ts

@@ -0,0 +1,97 @@
+/**
+ * @fileoverview Orchestrator for extracting high-quality citations (author, date, title, source) from HTML.
+ * Validates names and sources against known patterns and benchmarks.
+ */
+import { parseHTML } from "linkedom";
+import { extractAuthor } from "./extract-author";
+import { extractDateQuick } from "./extract-date/extract-date-quick";
+import { extractDate } from "./extract-date/extract-date";
+import { extractSource } from "./extract-source";
+import { extractTitle } from "./extract-title";
+import { extractCiteFromMetadata } from "./metadata-to-cite";
+import { extractHumanName } from "./human-names-recognize";
+import { parseDate } from "chrono-node";
+
+export interface ExtractCiteResult {
+  author?: string;
+  author_cite?: string;
+  date?: string;
+  title?: string;
+  source?: string;
+}
+
+export interface ExtractCiteOptions {
+  url?: string;
+}
+
+/**
+ * ### \u1f4da\u1f48e Extract Expert Excerpt
+ * <img width="350px" src="https://i.imgur.com/4GOOM9s.jpeg" />
+ *
+ * Extract author, date, source, and title from HTML using meta tags
+ * and common class names. Validates human name from author string to check
+ * against common list of 90k first names, last names,and organizations to infer
+ * if it should be reversed starting by author last name (accounting for affixes/titles),
+ * since organizations are not reversed.
+ * [Article Extraction Benchmark](https://github.com/scrapinghub/article-extraction-benchmark?tab=readme-ov-file#results)
+ * @param {Document | string} document dom object or html string with article content
+ * @param {ExtractCiteOptions} [options={}]
+ * @returns {ExtractCiteResult | null} An object containing extracted citation information.
+ * @category Extract
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ */
+export function extractCite(
+  document: Document | string,
+  options: ExtractCiteOptions = {}
+) {
+  const { url = "" } = options;
+
+  if (!document) return null;
+
+  //if passing in html string, convert to dom object
+  if (typeof document === "string") document = parseHTML(document)?.document;
+
+  var { author, date, title, source } = extractCiteFromMetadata(document);
+
+  if (author?.length < 3 || author?.length > 100) author = null;
+
+  var { author_cite, author_short, author_type } =
+    extractAuthor(document) || extractHumanName(author);
+
+  date = extractDateQuick(document, url) || date;
+
+  //extract from ways of writing dates in natural language into standard format
+  date = parseDate(date)?.toISOString().split("T")[0] || null;
+
+  if (!date)
+    try {
+      date = extractDate(
+        document,
+        true,
+        true,
+        "%Y-%m-%d",
+        url,
+        false,
+        new Date("2002-01-01")
+      );
+    } catch (e) {
+      console.log(e);
+    }
+
+  date = parseDate(date)?.toISOString().split("T")[0] || null;
+
+  title = extractTitle(document) || title;
+  source = extractSource(document) || source;
+
+  // URL TO SOURCE
+  if (!source && url?.length > 20)
+    source = url
+      .split("//")[1]
+      .split("/")[0]
+      ?.replace("www.", "")
+      .split(" ")
+      .map((word) => word[0].toUpperCase() + word.slice(1))
+      .join(" ");
+
+  return { author, author_cite, date, title, source };
+}