extract-cite 1.0.0

package/src/html-to-cite/extract-author.ts

@@ -0,0 +1,125 @@
+/**
+ * @fileoverview Utility for identifying and extracting author names from HTML metadata and content.
+ * Handles individual authors, multiple authors, and organizational authors.
+ */
+import { extractHumanName } from "./human-names-recognize";
+
+// https://www.scribbr.com/citation/generator/folders/2rx21jyIjZIKRrcwLk2oXE/lists/4OeJQ4euxzyTk9BiPTRjwn/
+const AUTHOR_META_TAGS = [
+  "byl",
+  "clmst",
+  "dc.author",
+  "dcsext.author",
+  "dc.creator",
+  "rbauthors",
+  "authors",
+  "sailthru.author",
+  "article:author",
+  "parsely-author",
+];
+
+const AUTHOR_SELECTORS = [
+  ".entry .entry-author",
+  ".author.vcard .fn",
+  ".author .vcard .fn",
+  ".byline.vcard .fn",
+  ".byline .vcard .fn",
+  ".byline .by .author",
+  ".byline .by",
+  ".byline .author",
+  ".post-author.vcard",
+  ".post-author .vcard",
+  "a[rel=author]",
+  "#by_author",
+  ".by_author",
+  "#entryAuthor",
+  ".entryAuthor",
+  ".byline a[href*=author]",
+  "#author .authorname",
+  ".author .authorname",
+  "#author",
+  ".author",
+  ".articleauthor",
+  ".ArticleAuthor",
+  ".byline",
+];
+
+const AUTHOR_MAX_LENGTH = 300;
+const BYLINE_REGEX = /^[\n\s]*By\s*:?\s*/i;
+const CLEAN_AUTHOR_RE = /^\s*(posted |written )?by\s*:?\s*(.*)/i;
+
+/**
+ * Extracts the author from the document and validates it as a human name
+ *
+ * @param {Document} document
+ * @returns {object|null} author_cite, author_short, author_type - or null if no valid author found
+*/
+export function extractAuthor(document) {
+
+
+  // 1. Check meta tags
+  for (const tag of AUTHOR_META_TAGS) {
+    const metaElement = document.querySelector(
+      `meta[name="${tag}"], meta[property="${tag}"]`
+    );
+    if (metaElement) {
+      const author = extractAndValidateHumanName(
+        metaElement.getAttribute("content")
+      );
+      if (author) return author;
+    }
+  }
+
+  // 2. Check selectors
+  for (const selector of AUTHOR_SELECTORS) {
+    const element = document.querySelector(selector);
+    if (element) {
+      const author = extractAndValidateHumanName(element.textContent);
+      if (author) return author;
+    }
+  }
+
+  // 4. Broader search in divs and spans
+  const elements = [
+    ...document.getElementsByTagName("div"),
+    ...document.getElementsByTagName("span"),
+  ];
+  for (const element of elements) {
+    if (
+      element.id.match(/author|byline/i) ||
+      element?.className?.match(/author|byline/i)
+    ) {
+      const author = extractAndValidateHumanName(element.innerText);
+      if (author) return author;
+    }
+  }
+
+
+  return null;
+}
+
+// Helper function to clean and validate author string
+const validateAuthor = (author) => {
+  if (!author) return null;
+
+  if (author.startsWith("@")) author = author.slice(1);
+  if (author.startsWith("http")) return null;
+
+  author = author?.replace(CLEAN_AUTHOR_RE, "$2").trim();
+  return author.length > 0 && author.length < AUTHOR_MAX_LENGTH
+    ? author
+    : null;
+};
+
+// Function to extract and validate human name
+const extractAndValidateHumanName = (author) => {
+  const validatedAuthor = validateAuthor(author);
+  if (validatedAuthor) {
+    return extractHumanName(validatedAuthor, {
+      formatCiteShortenAuthor: false
+    });
+  }
+  return null;
+};
+
+

package/src/html-to-cite/extract-cite.ts

@@ -0,0 +1,97 @@
+/**
+ * @fileoverview Orchestrator for extracting high-quality citations (author, date, title, source) from HTML.
+ * Validates names and sources against known patterns and benchmarks.
+ */
+import { parseHTML } from "linkedom";
+import { extractAuthor } from "./extract-author";
+import { extractDateQuick } from "./extract-date/extract-date-quick";
+import { extractDate } from "./extract-date/extract-date";
+import { extractSource } from "./extract-source";
+import { extractTitle } from "./extract-title";
+import { extractCiteFromMetadata } from "./metadata-to-cite";
+import { extractHumanName } from "./human-names-recognize";
+import { parseDate } from "chrono-node";
+
+export interface ExtractCiteResult {
+  author?: string;
+  author_cite?: string;
+  date?: string;
+  title?: string;
+  source?: string;
+}
+
+export interface ExtractCiteOptions {
+  url?: string;
+}
+
+/**
+ * ### \u1f4da\u1f48e Extract Expert Excerpt
+ * <img width="350px" src="https://i.imgur.com/4GOOM9s.jpeg" />
+ *
+ * Extract author, date, source, and title from HTML using meta tags
+ * and common class names. Validates human name from author string to check
+ * against common list of 90k first names, last names,and organizations to infer
+ * if it should be reversed starting by author last name (accounting for affixes/titles),
+ * since organizations are not reversed.
+ * [Article Extraction Benchmark](https://github.com/scrapinghub/article-extraction-benchmark?tab=readme-ov-file#results)
+ * @param {Document | string} document dom object or html string with article content
+ * @param {ExtractCiteOptions} [options={}]
+ * @returns {ExtractCiteResult | null} An object containing extracted citation information.
+ * @category Extract
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ */
+export function extractCite(
+  document: Document | string,
+  options: ExtractCiteOptions = {}
+) {
+  const { url = "" } = options;
+
+  if (!document) return null;
+
+  //if passing in html string, convert to dom object
+  if (typeof document === "string") document = parseHTML(document)?.document;
+
+  var { author, date, title, source } = extractCiteFromMetadata(document);
+
+  if (author?.length < 3 || author?.length > 100) author = null;
+
+  var { author_cite, author_short, author_type } =
+    extractAuthor(document) || extractHumanName(author);
+
+  date = extractDateQuick(document, url) || date;
+
+  //extract from ways of writing dates in natural language into standard format
+  date = parseDate(date)?.toISOString().split("T")[0] || null;
+
+  if (!date)
+    try {
+      date = extractDate(
+        document,
+        true,
+        true,
+        "%Y-%m-%d",
+        url,
+        false,
+        new Date("2002-01-01")
+      );
+    } catch (e) {
+      console.log(e);
+    }
+
+  date = parseDate(date)?.toISOString().split("T")[0] || null;
+
+  title = extractTitle(document) || title;
+  source = extractSource(document) || source;
+
+  // URL TO SOURCE
+  if (!source && url?.length > 20)
+    source = url
+      .split("//")[1]
+      .split("/")[0]
+      ?.replace("www.", "")
+      .split(" ")
+      .map((word) => word[0].toUpperCase() + word.slice(1))
+      .join(" ");
+
+  return { author, author_cite, date, title, source };
+}