npm - edgecrawl - Versions diffs - 0.3.0 - Mend

edgecrawl 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (13) hide show

package/LICENSE +21 -0
package/README.md +260 -0
package/index.mjs +22 -0
package/package.json +63 -0
package/schemas/article.json +15 -0
package/schemas/news.json +19 -0
package/schemas/product.json +20 -0
package/src/cli.mjs +226 -0
package/src/html2md.mjs +268 -0
package/src/llm.mjs +281 -0
package/src/pipeline.mjs +165 -0
package/src/scraper.mjs +218 -0
package/src/structured-extract.mjs +226 -0

package/src/html2md.mjs ADDED Viewed

@@ -0,0 +1,268 @@
+// src/html2md.mjs
+// HTML -> clean Markdown conversion
+// CSS selector-based HTML cleaning + node-html-markdown
+import { JSDOM } from "jsdom";
+import { NodeHtmlMarkdown } from "node-html-markdown";
+// ---- CSS selector-based HTML cleaning ----
+// Elements to remove: nav, footer, sidebar, ads, modals, social, etc.
+const REMOVE_SELECTORS = [
+  // Nav, header, footer
+  "header", "footer", "nav",
+  ".header", ".top", ".navbar", "#header",
+  ".footer", ".bottom", "#footer",
+  // Sidebar
+  "aside",
+  ".sidebar", ".side", ".aside", "#sidebar",
+  // Ads
+  ".ad", ".ads", ".advert", "#ad",
+  "[class*='ad-']", "[class*='ads-']", "[id*='ad-']",
+  // Modals, popups
+  ".modal", ".popup", "#modal", ".overlay",
+  // Social, share
+  ".social", ".social-media", ".social-links", "#social",
+  ".share", "#share",
+  // Nav, menu, breadcrumbs
+  ".menu", ".navigation", "#nav",
+  ".breadcrumbs", "#breadcrumbs", ".breadcrumb",
+  // Widgets, cookie banners
+  ".widget", "#widget",
+  ".cookie", "#cookie", ".cookie-banner",
+  // Language selectors
+  ".lang-selector", ".language", "#language-selector",
+];
+// Protected elements: do not remove if they contain or are contained by these
+const PROTECT_SELECTORS = [
+  "#main", "#content", "main", "article",
+  "[role='main']", "[role='article']",
+];
+/**
+ * Remove unwanted noise elements from HTML using CSS selectors
+ * @param {Document} document - JSDOM document
+ */
+function removeUnwantedElements(document) {
+  // 0. Remove <head> — title and meta description are extracted beforehand
+  const head = document.querySelector("head");
+  if (head) head.remove();
+  // 1. Remove script, style, noscript, iframe, svg, canvas, template
+  for (const tag of ["script", "style", "noscript", "iframe", "svg", "canvas", "template"]) {
+    for (const el of document.querySelectorAll(tag)) {
+      el.remove();
+    }
+  }
+  // 2. Strip style attributes, event handlers, and data-* attributes
+  for (const el of document.querySelectorAll("*")) {
+    const attrs = Array.from(el.attributes || []);
+    for (const attr of attrs) {
+      if (
+        attr.name === "style" ||
+        attr.name.startsWith("on") ||
+        attr.name.startsWith("data-")
+      ) {
+        el.removeAttribute(attr.name);
+      }
+    }
+  }
+  // 3. Remove hidden elements
+  for (const el of document.querySelectorAll("[hidden], [aria-hidden='true']")) {
+    el.remove();
+  }
+  // 4. Collect protected elements
+  const protectedElements = new Set();
+  for (const sel of PROTECT_SELECTORS) {
+    try {
+      for (const el of document.querySelectorAll(sel)) {
+        protectedElements.add(el);
+      }
+    } catch { /* invalid selector */ }
+  }
+  function isProtected(el) {
+    for (const p of protectedElements) {
+      if (p.contains(el) || el.contains(p)) return true;
+    }
+    return false;
+  }
+  // 5. Remove noise elements by CSS selector
+  for (const sel of REMOVE_SELECTORS) {
+    try {
+      for (const el of document.querySelectorAll(sel)) {
+        if (!isProtected(el)) el.remove();
+      }
+    } catch { /* invalid selector */ }
+  }
+  // 6. Remove empty elements
+  for (const tag of ["div", "span", "p", "section"]) {
+    for (const el of document.querySelectorAll(tag)) {
+      if (!el.textContent.trim() && !el.querySelector("img, video")) {
+        el.remove();
+      }
+    }
+  }
+}
+/**
+ * Convert relative URLs to absolute URLs
+ * @param {Document} document
+ * @param {string} baseUrl
+ */
+function absolutifyURLs(document, baseUrl) {
+  try {
+    const base = new URL(baseUrl);
+    for (const img of document.querySelectorAll("img[src]")) {
+      try {
+        img.setAttribute("src", new URL(img.getAttribute("src"), base).href);
+      } catch { /* invalid URL */ }
+    }
+    for (const a of document.querySelectorAll("a[href]")) {
+      try {
+        const href = a.getAttribute("href");
+        if (href && !href.startsWith("#") && !href.startsWith("javascript:")) {
+          a.setAttribute("href", new URL(href, base).href);
+        }
+      } catch { /* invalid URL */ }
+    }
+  } catch { /* invalid baseUrl */ }
+}
+/**
+ * Resolve img srcset to the largest image URL
+ * @param {Document} document
+ * @param {string} baseUrl
+ */
+function resolveSrcset(document, baseUrl) {
+  for (const img of document.querySelectorAll("img[srcset]")) {
+    try {
+      const srcset = img.getAttribute("srcset");
+      const candidates = srcset.split(",").map((s) => {
+        const parts = s.trim().split(/\s+/);
+        const url = parts[0];
+        const descriptor = parts[1] || "1x";
+        const size = parseFloat(descriptor) || 1;
+        return { url, size };
+      });
+      candidates.sort((a, b) => b.size - a.size);
+      if (candidates.length > 0) {
+        const resolved = new URL(candidates[0].url, baseUrl).href;
+        img.setAttribute("src", resolved);
+      }
+      img.removeAttribute("srcset");
+    } catch { /* parse error */ }
+  }
+}
+/**
+ * Main HTML-to-Markdown conversion
+ * CSS selector-based HTML cleaning -> node-html-markdown
+ * @param {string} html - Raw HTML
+ * @param {string} url  - Source URL
+ * @param {object} options
+ * @param {string} options.selector - CSS selector to narrow target content
+ * @returns {{ markdown: string, title: string, excerpt: string }}
+ */
+export function htmlToMarkdown(html, url = "https://example.com", options = {}) {
+  const { selector } = options;
+  const dom = new JSDOM(html, { url });
+  const document = dom.window.document;
+  // Extract title and meta description first
+  const title = document.title || "";
+  const metaDesc = document.querySelector('meta[name="description"]');
+  const excerpt = metaDesc ? metaDesc.getAttribute("content") || "" : "";
+  // If selector specified, replace body with target element
+  if (selector) {
+    const target = document.querySelector(selector);
+    if (target) {
+      document.body.innerHTML = target.outerHTML;
+    }
+  }
+  // HTML cleaning
+  removeUnwantedElements(document);
+  absolutifyURLs(document, url);
+  resolveSrcset(document, url);
+  // Convert cleaned HTML to Markdown via node-html-markdown
+  const bodyHtml = document.body?.innerHTML || "";
+  if (!bodyHtml.trim()) {
+    return { markdown: "", title, excerpt };
+  }
+  const markdown = NodeHtmlMarkdown.translate(bodyHtml, {
+    maxConsecutiveNewlines: 2,
+  }).trim();
+  return { markdown, title, excerpt };
+}
+/**
+ * Markdown post-processing: generic noise reduction for LLM input
+ * Algorithm-based, no site-specific rules
+ * @param {string} md - Raw Markdown
+ * @param {object} options
+ * @param {boolean} options.removeImages - Remove image references (default: false)
+ * @returns {string} Cleaned Markdown
+ */
+export function cleanMarkdown(md, options = {}) {
+  const { removeImages = false } = options;
+  let result = md;
+  // 1. Remove lines containing javascript: links
+  result = result.replace(/^.*\[.*\]\(javascript:.*\).*$/gm, "");
+  // 2. Remove empty links and alt-less images
+  result = result
+    .replace(/\[\s*\]\([^)]*\)/g, "")          // [](url) or [ ](url)
+    .replace(/!\[\]\([^)]*\)/g, "");            // ![](url) alt-less image
+  // 3. Hash-only links -> keep text only
+  result = result.replace(/\[([^\]]+)\]\(#[^)]*\)/g, "$1");
+  // 4. Overly long URL links -> keep text only
+  result = result.replace(/\[([^\]]+)\]\([^)]{150,}\)/g, "$1");
+  // 5. Normalize extra whitespace in link text
+  result = result.replace(/\[([^\]]*)\]/g, (_, t) => `[${t.replace(/\s+/g, " ").trim()}]`);
+  // 6. Remove lines containing template variables (${var}, %var%)
+  result = result.replace(/^.*\$\{[^}]+\}.*$/gm, "");
+  result = result.replace(/^\s*%[a-zA-Z]\w*%\s*$/gm, "");
+  // 7. Remove JSON-like lines (escaped included: {" or {\\" patterns)
+  result = result.replace(/^\s*\{\\?".*[\w_]+\\?"\s*:.*\}\s*$/gm, "");
+  // 8. Remove images (optional)
+  if (removeImages) {
+    result = result.replace(/!\[[^\]]*\]\([^)]*\)/g, "");
+  }
+  // 9. Collapse consecutive blank lines
+  return result
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+/**
+ * Truncate Markdown to fit within LLM token limits
+ * Rough estimate: 1 token ~ 3 characters
+ */
+export function truncateForLLM(markdown, maxTokens = 2048) {
+  const approxMaxChars = maxTokens * 3;
+  if (markdown.length <= approxMaxChars) return markdown;
+  return markdown.slice(0, approxMaxChars) + "\n\n[... truncated]";
+}

package/src/llm.mjs ADDED Viewed

@@ -0,0 +1,281 @@
+// src/llm.mjs
+// ONNX LLM wrapper using Transformers.js v4
+// Runs on Node.js with WebGPU or WASM
+import { pipeline, env } from "@huggingface/transformers";
+const MODEL_PRESETS = {
+  // NuExtract: 500M - specialized for structured extraction
+  granite: {
+    model: "onnx-community/granite-4.0-350m-ONNX-web",
+    dtype: "fp16",
+    maxNewTokens: 32768,
+    type: "granite",
+  },
+  nuextract: {
+    model: "onnx-community/NuExtract-1.5-tiny-ONNX",
+    dtype: "q4",
+    maxNewTokens: 32768,
+    type: "nuextract",
+  },
+  qwen25: {
+    model: "onnx-community/Qwen2.5-0.5B-Instruct-ONNX-MHA",
+    dtype: "q4f16",
+    maxNewTokens: 32768,
+    type: "qwen25",
+  },
+  // Light: 0.6B - Qwen3
+  light: {
+    model: "onnx-community/Qwen3-0.6B-ONNX",
+    dtype: "q4f16",
+    maxNewTokens: 32768,
+    type: "qwen3",
+  },
+  // Balanced: 1.7B - Qwen3
+  balanced: {
+    model: "onnx-community/Qwen3-1.7B-ONNX",
+    dtype: "q4f16",
+    maxNewTokens: 32768,
+    type: "qwen3",
+  },
+  // Quality: 4B - Qwen3
+  quality: {
+    model: "onnx-community/Qwen3-4B-ONNX",
+    dtype: "q4f16",
+    maxNewTokens: 32768,
+    type: "qwen3",
+  },
+};
+let generator = null;
+let currentPreset = null;
+/**
+ * Initialize the LLM
+ */
+export async function initLLM(preset = "balanced", device = "webgpu") {
+  if (generator && currentPreset === preset) return;
+  const config = MODEL_PRESETS[preset];
+  if (!config) throw new Error(`Unknown preset: ${preset}. Available: ${Object.keys(MODEL_PRESETS).join(", ")}`);
+  console.log(`Loading model: ${config.model} (${config.dtype}, ${device})...`);
+  env.cacheDir = "./.model-cache";
+  generator = await pipeline("text-generation", config.model, {
+    dtype: config.dtype,
+    device,
+  });
+  currentPreset = preset;
+  console.log("Model loaded.");
+}
+/**
+ * Generate a template object from a JSON Schema
+ * NuExtract expects an empty output template, not a JSON Schema
+ */
+function schemaToTemplate(schema) {
+  if (!schema.properties) return schema;
+  const template = {};
+  for (const [key, def] of Object.entries(schema.properties)) {
+    if (def.type === "array") {
+      if (def.items?.properties) {
+        const item = {};
+        for (const [k, v] of Object.entries(def.items.properties)) {
+          item[k] = "";
+        }
+        template[key] = [item];
+      } else {
+        template[key] = [];
+      }
+    } else if (def.type === "object" && def.properties) {
+      template[key] = schemaToTemplate(def);
+    } else {
+      template[key] = "";
+    }
+  }
+  return template;
+}
+/**
+ * Extract structured data (JSON) from Markdown content
+ */
+export async function extractStructured(markdown, schema) {
+  if (!generator) throw new Error("LLM not initialized. Call initLLM() first.");
+  const config = MODEL_PRESETS[currentPreset];
+  if (config.type === "nuextract") {
+    return extractWithNuExtract(markdown, schema, config);
+  } else if (config.type === "qwen25" || config.type === "granite") {
+    return extractWithQwen25(markdown, schema, config);
+  } else {
+    return extractWithQwen3(markdown, schema, config);
+  }
+}
+/**
+ * NuExtract-specific extraction
+ */
+async function extractWithNuExtract(markdown, schema, config) {
+  const template = schemaToTemplate(schema);
+  const templateStr = JSON.stringify(template, null, 2);
+  const prompt = `<|input|>\n### Template:\n${templateStr}\n### Text:\n${markdown}\n\n<|output|>\n`;
+  const output = await generator(prompt, {
+    max_new_tokens: config.maxNewTokens,
+    do_sample: false,
+  });
+  const raw = output[0].generated_text.slice(prompt.length).trim();
+  return parseJSONSafe(raw);
+}
+/**
+ * Qwen2.5-specific extraction (no thinking mode)
+ */
+async function extractWithQwen25(markdown, schema, config) {
+  const schemaStr = JSON.stringify(schema, null, 2);
+  const messages = [
+    {
+      role: "system",
+      content: `You are a data extraction assistant. Extract structured information from the given content and return ONLY valid JSON matching the provided JSON Schema. No explanation, no markdown fences, just raw JSON.`,
+    },
+    {
+      role: "user",
+      content: `Extract data from the content below according to this JSON Schema:\n\n${schemaStr}\n\n---\nContent:\n${markdown}\n---\n\nReturn ONLY a valid JSON object matching the schema above.`,
+    },
+  ];
+  const output = await generator(messages, {
+    max_new_tokens: config.maxNewTokens,
+    temperature: 0.7,
+    top_p: 0.8,
+    top_k: 20,
+    do_sample: true,
+    repetition_penalty: 1.2,
+  });
+  const raw = output[0].generated_text.at(-1).content;
+  return normalizeExtracted(parseJSONSafe(raw));
+}
+/**
+ * Qwen3-specific extraction
+ */
+async function extractWithQwen3(markdown, schema, config) {
+  const schemaStr = JSON.stringify(schema, null, 2);
+  const messages = [
+    {
+      role: "system",
+      content: `You are a data extraction assistant. Extract structured information from the given content and return ONLY valid JSON matching the provided JSON Schema. No explanation, no markdown fences, no thinking, just raw JSON.`,
+    },
+    {
+      role: "user",
+      content: `Extract data from the content below according to this JSON Schema:\n\n${schemaStr}\n\n---\nContent:\n${markdown}\n---\n\nReturn ONLY a valid JSON object matching the schema above.`,
+    },
+  ];
+  const output = await generator(messages, {
+    max_new_tokens: config.maxNewTokens,
+    temperature: 0,
+    do_sample: false,
+    repetition_penalty: 1.05,
+    enable_thinking: false,
+  });
+  const raw = output[0].generated_text.at(-1).content;
+  return normalizeExtracted(parseJSONSafe(raw));
+}
+/**
+ * Call LLM with a custom prompt
+ */
+export async function queryLLM(systemPrompt, userPrompt, options = {}) {
+  if (!generator) throw new Error("LLM not initialized. Call initLLM() first.");
+  const config = MODEL_PRESETS[currentPreset];
+  const { noThink = true } = options;
+  const finalSystemPrompt = noThink
+    ? `${systemPrompt} /no_think`
+    : systemPrompt;
+  const messages = [
+    { role: "system", content: finalSystemPrompt },
+    { role: "user", content: userPrompt },
+  ];
+  const output = await generator(messages, {
+    max_new_tokens: config.maxNewTokens,
+    do_sample: false,
+  });
+  return output[0].generated_text.at(-1).content;
+}
+/**
+ * Normalize LLM output structure
+ * - If schema definition leaked into output -> extract properties content
+ * - If wrapped in array -> extract first element
+ */
+function normalizeExtracted(parsed) {
+  if (!parsed || parsed._error) return parsed;
+  // Schema leak: { type: "object", properties: { title: "...", ... } }
+  if (parsed.type && parsed.properties && typeof parsed.properties === "object") {
+    const inner = parsed.properties;
+    // If property values are actual data (not schema defs like { type: "string" })
+    const firstVal = Object.values(inner)[0];
+    if (firstVal && typeof firstVal !== "object") {
+      return inner;
+    }
+  }
+  // Array wrap: [{ title: "...", ... }] -> { title: "...", ... }
+  if (Array.isArray(parsed) && parsed.length === 1) {
+    return parsed[0];
+  }
+  return parsed;
+}
+/**
+ * Safely parse a JSON string from LLM output
+ */
+function parseJSONSafe(raw) {
+  // Remove think tags
+  let cleaned = raw.replace(/<think>[\s\S]*?<\/think>/g, "");
+  cleaned = cleaned.replace(/<think>[\s\S]*/g, "");
+  // Remove code fences
+  cleaned = cleaned.replace(/```(?:json)?\s*/gi, "").replace(/```/g, "");
+  cleaned = cleaned.trim();
+  try {
+    return JSON.parse(cleaned);
+  } catch {
+    // Find the last valid JSON object
+    const matches = [...cleaned.matchAll(/\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}/g)];
+    for (let i = matches.length - 1; i >= 0; i--) {
+      try {
+        return JSON.parse(matches[i][0]);
+      } catch { continue; }
+    }
+    // Try broadest match
+    const broad = cleaned.match(/\{[\s\S]*\}/);
+    if (broad) {
+      try {
+        return JSON.parse(broad[0]);
+      } catch {}
+    }
+    return { _raw: raw, _error: "Failed to parse JSON" };
+  }
+}
+export { MODEL_PRESETS };

package/src/pipeline.mjs ADDED Viewed

@@ -0,0 +1,165 @@
+// src/pipeline.mjs
+// Scraping -> Markdown conversion -> LLM structured extraction pipeline
+import {
+  fetchPage,
+  fetchPages,
+  launchBrowser,
+  closeBrowser,
+} from "./scraper.mjs";
+import { htmlToMarkdown, cleanMarkdown, truncateForLLM } from "./html2md.mjs";
+import { initLLM, extractStructured, queryLLM } from "./llm.mjs";
+import { tryStructuredExtract } from "./structured-extract.mjs";
+/**
+ * Default schema definition
+ */
+const DEFAULT_SCHEMA = {
+  title: "Page title (string)",
+  description: "Page summary in 1-2 sentences (string)",
+  main_topics: "List of main topics (array of strings)",
+  key_facts: "Important facts and data points (array of strings)",
+  entities: "People, organizations, and product names mentioned (array of strings)",
+  language: "Content language (string, e.g. 'ja', 'en')",
+};
+/**
+ * Scrape a single URL and extract structured data
+ * @param {string} url
+ * @param {object} options
+ */
+export async function scrapeAndExtract(url, options = {}) {
+  const {
+    schema = DEFAULT_SCHEMA,
+    preset = "balanced",
+    device = "webgpu",
+    maxTokens = 2048,
+    selector,
+    browserOptions = {},
+    scrapeOptions = {},
+  } = options;
+  await launchBrowser(browserOptions);
+  const { html, url: finalUrl, status } = await fetchPage(url, scrapeOptions);
+  if (!html) return { url, error: "Failed to fetch page", status };
+  // Try structured data (JSON-LD / OG) first
+  const structResult = tryStructuredExtract(html, schema);
+  if (structResult) {
+    return {
+      url: finalUrl, status,
+      extraction_source: structResult.source,
+      coverage: structResult.coverage,
+      extracted: structResult.extracted,
+    };
+  }
+  // Fallback: Markdown conversion -> cleanMarkdown -> LLM
+  const { markdown, title, excerpt } = htmlToMarkdown(html, finalUrl, { selector });
+  const llmInput = cleanMarkdown(markdown);
+  const metadata = {
+    url: finalUrl, status, title, excerpt,
+    markdown_length: markdown.length,
+    cleaned_length: llmInput.length,
+    extraction_source: "llm",
+  };
+  // LLM structured extraction
+  await initLLM(preset, device);
+  const extracted = await extractStructured(llmInput, schema);
+  return { ...metadata, extracted };
+}
+/**
+ * Batch process multiple URLs
+ */
+export async function batchScrapeAndExtract(urls, options = {}) {
+  const {
+    schema = DEFAULT_SCHEMA,
+    preset = "balanced",
+    device = "webgpu",
+    maxTokens = 2048,
+    concurrency = 3,
+    browserOptions = {},
+    scrapeOptions = {},
+  } = options;
+  await initLLM(preset, device);
+  await launchBrowser(browserOptions);
+  const results = [];
+  const pages = await fetchPages(urls, scrapeOptions, concurrency);
+  for (const page of pages) {
+    if (page.error || !page.html) {
+      results.push({ url: page.url, error: page.error || "Empty response", extracted: null });
+      continue;
+    }
+    // Try structured data first
+    const structResult = tryStructuredExtract(page.html, schema);
+    if (structResult) {
+      results.push({
+        url: page.url, status: page.status,
+        extraction_source: structResult.source,
+        coverage: structResult.coverage,
+        extracted: structResult.extracted,
+      });
+      continue;
+    }
+    // Fallback: Markdown -> LLM
+    const { markdown, title } = htmlToMarkdown(page.html, page.url);
+    const truncated = truncateForLLM(cleanMarkdown(markdown), maxTokens);
+    try {
+      const extracted = await extractStructured(truncated, schema);
+      results.push({ url: page.url, status: page.status, title, extraction_source: "llm", extracted });
+    } catch (error) {
+      results.push({ url: page.url, error: error.message, extracted: null });
+    }
+  }
+  return results;
+}
+/**
+ * Custom prompt query against page content
+ */
+export async function scrapeAndQuery(url, prompt, options = {}) {
+  const {
+    preset = "balanced",
+    device = "webgpu",
+    maxTokens = 2048,
+    browserOptions = {},
+    scrapeOptions = {},
+  } = options;
+  await launchBrowser(browserOptions);
+  const { html, url: finalUrl } = await fetchPage(url, scrapeOptions);
+  if (!html) return { url, error: "Failed to fetch" };
+  const { markdown } = htmlToMarkdown(html, finalUrl);
+  const content = truncateForLLM(cleanMarkdown(markdown), maxTokens);
+  await initLLM(preset, device);
+  const answer = await queryLLM(
+    "You are a helpful assistant analyzing web content. Answer based only on the provided content.",
+    `${prompt}\n\n---\nContent:\n${content}`
+  );
+  return { url, answer };
+}
+/**
+ * Cleanup (close browser)
+ */
+export async function cleanup() {
+  await closeBrowser();
+}
+export { DEFAULT_SCHEMA };