extract-cite 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -0
- package/dist/extract-cite.cjs.js +2 -0
- package/dist/extract-cite.cjs.js.map +1 -0
- package/dist/extract-cite.es.js +4 -0
- package/dist/extract-cite.es.js.map +1 -0
- package/dist/html-to-cite/extract-author.d.ts +11 -0
- package/dist/html-to-cite/extract-cite.d.ts +33 -0
- package/dist/html-to-cite/extract-date/date-extractors.d.ts +40 -0
- package/dist/html-to-cite/extract-date/date-validators.d.ts +15 -0
- package/dist/html-to-cite/extract-date/extract-date-quick.d.ts +8 -0
- package/dist/html-to-cite/extract-date/extract-date.d.ts +26 -0
- package/dist/html-to-cite/extract-source.d.ts +11 -0
- package/dist/html-to-cite/extract-title.d.ts +11 -0
- package/dist/html-to-cite/human-names-recognize.d.ts +16 -0
- package/dist/html-to-cite/metadata-to-cite.d.ts +16 -0
- package/dist/html-to-cite/url-to-domain.d.ts +20 -0
- package/dist/index.d.ts +14 -0
- package/dist/llm/call-llm.d.ts +33 -0
- package/dist/llm/extract-cite-llm.d.ts +19 -0
- package/dist/llm/format-citation.d.ts +22 -0
- package/dist/llm/index.d.ts +7 -0
- package/dist/llm/page-text.d.ts +21 -0
- package/dist/llm/parse-reply.d.ts +26 -0
- package/dist/llm/prompt.d.ts +14 -0
- package/dist/llm/types.d.ts +115 -0
- package/package.json +61 -0
- package/src/html-to-cite/extract-author.ts +125 -0
- package/src/html-to-cite/extract-cite.ts +97 -0
- package/src/html-to-cite/extract-date/date-extractors.ts +509 -0
- package/src/html-to-cite/extract-date/date-validators.ts +191 -0
- package/src/html-to-cite/extract-date/extract-date-quick.ts +184 -0
- package/src/html-to-cite/extract-date/extract-date.ts +1051 -0
- package/src/html-to-cite/extract-source.ts +35 -0
- package/src/html-to-cite/extract-title.ts +78 -0
- package/src/html-to-cite/human-names-92k.json +1 -0
- package/src/html-to-cite/human-names-recognize.ts +396 -0
- package/src/html-to-cite/metadata-to-cite.ts +77 -0
- package/src/html-to-cite/url-to-domain.ts +50 -0
- package/src/index.ts +14 -0
- package/src/llm/call-llm.ts +112 -0
- package/src/llm/extract-cite-llm.ts +100 -0
- package/src/llm/format-citation.ts +399 -0
- package/src/llm/index.ts +7 -0
- package/src/llm/page-text.ts +106 -0
- package/src/llm/parse-reply.ts +279 -0
- package/src/llm/prompt.ts +95 -0
- package/src/llm/types.ts +174 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Extracts the author from the document and validates it as a human name
|
|
3
|
+
*
|
|
4
|
+
* @param {Document} document
|
|
5
|
+
* @returns {object|null} author_cite, author_short, author_type - or null if no valid author found
|
|
6
|
+
*/
|
|
7
|
+
export declare function extractAuthor(document: any): {
|
|
8
|
+
author_cite: any;
|
|
9
|
+
author_short: any;
|
|
10
|
+
author_type: number;
|
|
11
|
+
};
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export interface ExtractCiteResult {
|
|
2
|
+
author?: string;
|
|
3
|
+
author_cite?: string;
|
|
4
|
+
date?: string;
|
|
5
|
+
title?: string;
|
|
6
|
+
source?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface ExtractCiteOptions {
|
|
9
|
+
url?: string;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* ### \u1f4da\u1f48e Extract Expert Excerpt
|
|
13
|
+
* <img width="350px" src="https://i.imgur.com/4GOOM9s.jpeg" />
|
|
14
|
+
*
|
|
15
|
+
* Extract author, date, source, and title from HTML using meta tags
|
|
16
|
+
* and common class names. Validates human name from author string to check
|
|
17
|
+
* against common list of 90k first names, last names,and organizations to infer
|
|
18
|
+
* if it should be reversed starting by author last name (accounting for affixes/titles),
|
|
19
|
+
* since organizations are not reversed.
|
|
20
|
+
* [Article Extraction Benchmark](https://github.com/scrapinghub/article-extraction-benchmark?tab=readme-ov-file#results)
|
|
21
|
+
* @param {Document | string} document dom object or html string with article content
|
|
22
|
+
* @param {ExtractCiteOptions} [options={}]
|
|
23
|
+
* @returns {ExtractCiteResult | null} An object containing extracted citation information.
|
|
24
|
+
* @category Extract
|
|
25
|
+
* @author [vtempest (2025)](https://github.com/vtempest)
|
|
26
|
+
*/
|
|
27
|
+
export declare function extractCite(document: Document | string, options?: ExtractCiteOptions): {
|
|
28
|
+
author: string;
|
|
29
|
+
author_cite: any;
|
|
30
|
+
date: string;
|
|
31
|
+
title: string;
|
|
32
|
+
source: string;
|
|
33
|
+
};
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
declare const FAST_PREPEND = "";
|
|
2
|
+
declare const MIN_SEGMENT_LEN = 6;
|
|
3
|
+
declare const MAX_SEGMENT_LEN = 52;
|
|
4
|
+
declare const DATE_EXPRESSIONS: string;
|
|
5
|
+
declare const SLOW_PREPEND = "";
|
|
6
|
+
declare const FREE_TEXT_EXPRESSIONS = ".//*[self::div or self::h2 or self::h3 or self::h4 or self::li or self::p or self::span or self::time or self::ul]/text()";
|
|
7
|
+
declare const THREE_COMP_REGEX_A: RegExp;
|
|
8
|
+
declare const THREE_COMP_REGEX_B: RegExp;
|
|
9
|
+
declare const TWO_COMP_REGEX: RegExp;
|
|
10
|
+
declare const YEAR_PATTERN: RegExp;
|
|
11
|
+
declare const COPYRIGHT_PATTERN: RegExp;
|
|
12
|
+
declare const THREE_PATTERN: RegExp;
|
|
13
|
+
declare const THREE_CATCH: RegExp;
|
|
14
|
+
declare const THREE_LOOSE_PATTERN: RegExp;
|
|
15
|
+
declare const THREE_LOOSE_CATCH: RegExp;
|
|
16
|
+
declare const SELECT_YMD_PATTERN: RegExp;
|
|
17
|
+
declare const SELECT_YMD_YEAR: RegExp;
|
|
18
|
+
declare const YMD_YEAR: RegExp;
|
|
19
|
+
declare const DATESTRINGS_PATTERN: RegExp;
|
|
20
|
+
declare const DATESTRINGS_CATCH: RegExp;
|
|
21
|
+
declare const SLASHES_PATTERN: RegExp;
|
|
22
|
+
declare const SLASHES_YEAR: RegExp;
|
|
23
|
+
declare const YYYYMM_PATTERN: RegExp;
|
|
24
|
+
declare const YYYYMM_CATCH: RegExp;
|
|
25
|
+
declare const MMYYYY_PATTERN: RegExp;
|
|
26
|
+
declare const MMYYYY_YEAR: RegExp;
|
|
27
|
+
declare const SIMPLE_PATTERN: RegExp;
|
|
28
|
+
declare const YMD_PATTERN: RegExp;
|
|
29
|
+
declare const TIMESTAMP_PATTERN: RegExp;
|
|
30
|
+
declare function discard_unwanted(tree: any): any[];
|
|
31
|
+
declare function extract_url_date(testurl: any, options: any): string;
|
|
32
|
+
declare function regex_parse(string: any): Date;
|
|
33
|
+
declare function custom_parse(string: any, outputformat: any, min_date: any, max_date: any): any;
|
|
34
|
+
declare function external_date_parser(string: any, outputformat: any): string;
|
|
35
|
+
declare function try_date_expr(string: any, outputformat: any, extensive_search: any, min_date: any, max_date: any): any;
|
|
36
|
+
declare function img_search(tree: any, options: any): string;
|
|
37
|
+
declare function pattern_search(text: any, date_pattern: any, options: any): any;
|
|
38
|
+
declare function json_search(tree: any, options: any): any;
|
|
39
|
+
declare function idiosyncrasies_search(htmlstring: any, options: any): string;
|
|
40
|
+
export { discard_unwanted, extract_url_date, regex_parse, custom_parse, external_date_parser, try_date_expr, img_search, pattern_search, json_search, idiosyncrasies_search, DATE_EXPRESSIONS, FAST_PREPEND, SLOW_PREPEND, FREE_TEXT_EXPRESSIONS, MAX_SEGMENT_LEN, MIN_SEGMENT_LEN, YEAR_PATTERN, YMD_PATTERN, COPYRIGHT_PATTERN, TIMESTAMP_PATTERN, THREE_PATTERN, THREE_CATCH, THREE_LOOSE_PATTERN, THREE_LOOSE_CATCH, SELECT_YMD_PATTERN, SELECT_YMD_YEAR, YMD_YEAR, DATESTRINGS_PATTERN, DATESTRINGS_CATCH, SLASHES_PATTERN, SLASHES_YEAR, YYYYMM_PATTERN, YYYYMM_CATCH, MMYYYY_PATTERN, MMYYYY_YEAR, SIMPLE_PATTERN, THREE_COMP_REGEX_A, THREE_COMP_REGEX_B, TWO_COMP_REGEX, };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Validation and filtering logic for extracted date candidates.
|
|
3
|
+
* Ensures dates fall within plausible ranges and meet format requirements.
|
|
4
|
+
*/
|
|
5
|
+
declare function is_valid_date(date_input: any, outputformat: any, earliest: any, latest: any): boolean;
|
|
6
|
+
declare function is_valid_format(outputformat: any): boolean;
|
|
7
|
+
declare function plausible_year_filter(htmlstring: any, pattern: any, yearpat: any, earliest: any, latest: any, incomplete?: boolean): Map<any, any>;
|
|
8
|
+
declare function compare_values(reference: any, attempt: any, options: any): any;
|
|
9
|
+
declare function filter_ymd_candidate(bestmatch: any, pattern: any, original_date: any, copyear: any, outputformat: any, min_date: any, max_date: any): any;
|
|
10
|
+
declare function convert_date(datestring: any, inputformat: any, outputformat: any): any;
|
|
11
|
+
declare function check_extracted_reference(reference: any, options: any): string;
|
|
12
|
+
declare function check_date_input(date_object: any, default_date: any): any;
|
|
13
|
+
declare function get_min_date(min_date: any): any;
|
|
14
|
+
declare function get_max_date(max_date: any): any;
|
|
15
|
+
export { is_valid_date, is_valid_format, plausible_year_filter, compare_values, filter_ymd_candidate, convert_date, check_extracted_reference, check_date_input, get_min_date, get_max_date };
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Extract date from document using various methods
|
|
3
|
+
*
|
|
4
|
+
* @param {Document} document - DOM object with article content
|
|
5
|
+
* @param {string} url - URL of the page
|
|
6
|
+
* @returns {string|null} Extracted date or null if not found
|
|
7
|
+
*/
|
|
8
|
+
export declare function extractDateQuick(document: any, url: any): any;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
declare const DATE_ATTRIBUTES: Set<string>;
|
|
2
|
+
declare const NAME_MODIFIED: Set<string>;
|
|
3
|
+
declare const PROPERTY_MODIFIED: Set<string>;
|
|
4
|
+
declare const ITEMPROP_ATTRS_ORIGINAL: Set<string>;
|
|
5
|
+
declare const ITEMPROP_ATTRS_MODIFIED: Set<string>;
|
|
6
|
+
declare const ITEMPROP_ATTRS: Set<string>;
|
|
7
|
+
declare const CLASS_ATTRS: Set<string>;
|
|
8
|
+
declare const NON_DIGITS_REGEX: RegExp;
|
|
9
|
+
export { DATE_ATTRIBUTES, NAME_MODIFIED, PROPERTY_MODIFIED, ITEMPROP_ATTRS_ORIGINAL, ITEMPROP_ATTRS_MODIFIED, ITEMPROP_ATTRS, CLASS_ATTRS, NON_DIGITS_REGEX, };
|
|
10
|
+
/**
|
|
11
|
+
* Extract date from document using various methods
|
|
12
|
+
*
|
|
13
|
+
* @param {Document} htmlobject - DOM object with article content
|
|
14
|
+
* @param {boolean} [extensive_search=true] - perform extensive search if true
|
|
15
|
+
* @param {boolean} [original_date=false] - return original date if true
|
|
16
|
+
* @param {string} [outputformat="%Y-%m-%d"] - output format
|
|
17
|
+
* @param {string} [url=null] - URL of the page
|
|
18
|
+
* @param {boolean} [verbose=false] - log debug messages if true
|
|
19
|
+
* @param {Date} [min_date=null] - minimum date to consider
|
|
20
|
+
* @param {Date} [max_date=null] - maximum date to consider
|
|
21
|
+
* @param {boolean} [deferred_url_extractor=false] - if true, do not extract date from URL
|
|
22
|
+
* @returns {string|null} Extracted date or null if not found
|
|
23
|
+
* @author [vtempest (2025)](https://github.com/vtempest)
|
|
24
|
+
* Based on [Barbaresi (2020)](https://github.com/adbar/htmldate/)
|
|
25
|
+
*/
|
|
26
|
+
export declare function extractDate(htmlobject: any, extensive_search?: boolean, original_date?: boolean, outputformat?: string, url?: any, verbose?: boolean, min_date?: any, max_date?: any, deferred_url_extractor?: boolean): any;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Utility for extracting the publication/source name from a
|
|
3
|
+
* document's `og:site_name` meta tag or class-based markers.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Extract source from document using common class names
|
|
7
|
+
*
|
|
8
|
+
* @param {document} document document or dom object with article content
|
|
9
|
+
* @returns {object} source
|
|
10
|
+
*/
|
|
11
|
+
export declare function extractSource(document: any): any;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Utility for identifying, extracting, and normalizing document titles from HTML.
|
|
3
|
+
* Handles metadata, selectors, and breadcrumb cleaning.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Extract and clean title from document
|
|
7
|
+
*
|
|
8
|
+
* @param {Document} document - DOM object with article content
|
|
9
|
+
* @returns {string} Extracted and cleaned title
|
|
10
|
+
*/
|
|
11
|
+
export declare function extractTitle(document: any): string;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export interface ExtractHumanNameOptions {
|
|
2
|
+
formatCiteShortenAuthor?: boolean;
|
|
3
|
+
maxAuthorsBeforeEtAl?: number;
|
|
4
|
+
}
|
|
5
|
+
/**
|
|
6
|
+
* Validates and formats author names properly handling multiple authors and multi-word names
|
|
7
|
+
*
|
|
8
|
+
* @param {string} author - The author name string(s) to be processed
|
|
9
|
+
* @param {ExtractHumanNameOptions} [options={}] - Configuration options
|
|
10
|
+
* @returns {object} Formatted author information for citation
|
|
11
|
+
*/
|
|
12
|
+
export declare function extractHumanName(author: string, options?: ExtractHumanNameOptions): {
|
|
13
|
+
author_cite: any;
|
|
14
|
+
author_short: any;
|
|
15
|
+
author_type: number;
|
|
16
|
+
};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Extracts citation metadata (author, date, title, source) from
|
|
3
|
+
* a document's `<meta>` tags using a list of commonly used property/name attributes.
|
|
4
|
+
*/
|
|
5
|
+
export interface CiteMetadata {
|
|
6
|
+
author?: string;
|
|
7
|
+
date?: string;
|
|
8
|
+
title?: string;
|
|
9
|
+
source?: string;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Extract cite info from common property names in webpage's metadata
|
|
13
|
+
* @param {Document} doc dom object of document
|
|
14
|
+
* @returns {CiteMetadata} author, date, title, source
|
|
15
|
+
*/
|
|
16
|
+
export declare function extractCiteFromMetadata(doc: Document): CiteMetadata;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Utility for extracting and normalizing domain names from URLs.
|
|
3
|
+
* Handles subdomains and TLD cleaning for source attribution.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Extract TLD and hostname from domain in Regex. There's [two or more part
|
|
7
|
+
* TLDs](https://en.wikipedia.org/wiki/List_of_Internet_top-level_domains)
|
|
8
|
+
* so it is hard to tell if host.secondTLD.tld or host.tld is correct way
|
|
9
|
+
* to get root domain (e.g. abc.go.jp, abc.co.uk)
|
|
10
|
+
* @param {string} domain
|
|
11
|
+
* @returns {string} rootDomain
|
|
12
|
+
*/
|
|
13
|
+
export declare function convertURLToDomain(domain: any): any;
|
|
14
|
+
/**
|
|
15
|
+
* Checks if a string is a valid URL.
|
|
16
|
+
* @param {string} string
|
|
17
|
+
* @returns {boolean} true if the string is a valid URL
|
|
18
|
+
* @private
|
|
19
|
+
*/
|
|
20
|
+
export declare function isURLValid(string: any): boolean;
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview extract-cite: cite any webpage.
|
|
3
|
+
*
|
|
4
|
+
* - `extractCite` reads author, date, title and source straight off the HTML
|
|
5
|
+
* (meta tags, selectors, JSON-LD, URL, a 92k-name database). Instant, offline,
|
|
6
|
+
* partial.
|
|
7
|
+
* - `extractCiteLLM` then asks a model (OpenRouter by default) to finish the
|
|
8
|
+
* citation, score every part, flag the doubtful ones and read the authors'
|
|
9
|
+
* qualifications, and writes it as APA, MLA, Chicago, Harvard, IEEE and BibTeX.
|
|
10
|
+
*/
|
|
11
|
+
export * from './html-to-cite/extract-cite';
|
|
12
|
+
export * from './html-to-cite/url-to-domain';
|
|
13
|
+
export { extractHumanName } from './html-to-cite/human-names-recognize';
|
|
14
|
+
export * from './llm';
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview The one LLM call: an OpenAI-compatible chat-completions
|
|
3
|
+
* request, OpenRouter by default, with a JSON-object reply parsed back out.
|
|
4
|
+
*/
|
|
5
|
+
export declare const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1";
|
|
6
|
+
export declare const DEFAULT_MODEL = "anthropic/claude-haiku-4.5";
|
|
7
|
+
export declare class CiteLLMError extends Error {
|
|
8
|
+
status?: number;
|
|
9
|
+
constructor(message: string, status?: number);
|
|
10
|
+
}
|
|
11
|
+
export interface CallLLMOptions {
|
|
12
|
+
apiKey?: string;
|
|
13
|
+
model?: string;
|
|
14
|
+
baseUrl?: string;
|
|
15
|
+
system: string;
|
|
16
|
+
user: string;
|
|
17
|
+
timeoutMs?: number;
|
|
18
|
+
headers?: Record<string, string>;
|
|
19
|
+
fetch?: typeof fetch;
|
|
20
|
+
}
|
|
21
|
+
/** The API key from the environment, where there is an environment. */
|
|
22
|
+
export declare function envApiKey(): string | undefined;
|
|
23
|
+
/**
|
|
24
|
+
* Sends one system + user prompt and returns the reply parsed as a JSON object.
|
|
25
|
+
* Throws `CiteLLMError` for a missing key, an HTTP error, an empty reply or a
|
|
26
|
+
* reply that is not JSON.
|
|
27
|
+
*/
|
|
28
|
+
export declare function callLLM(options: CallLLMOptions): Promise<{
|
|
29
|
+
json: unknown;
|
|
30
|
+
model: string;
|
|
31
|
+
}>;
|
|
32
|
+
/** Parses a reply that may be wrapped in a code fence or surrounded by prose. */
|
|
33
|
+
export declare function parseJSONReply(content: string): unknown;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { ExtractCiteLLMOptions, ExtractCiteLLMResult } from './types';
|
|
2
|
+
/**
|
|
3
|
+
* Cites a page: regex first, then the model for what regex cannot do.
|
|
4
|
+
*
|
|
5
|
+
* ```ts
|
|
6
|
+
* const cite = await extractCiteLLM({
|
|
7
|
+
* url: "https://www.nytimes.com/…",
|
|
8
|
+
* apiKey: process.env.OPENROUTER_API_KEY, // the default provider is OpenRouter
|
|
9
|
+
* model: "anthropic/claude-haiku-4.5",
|
|
10
|
+
* });
|
|
11
|
+
* cite.formatted.apa?.text; // full APA 7 entry
|
|
12
|
+
* cite.needsReview; // fields to check by hand
|
|
13
|
+
* cite.citation.authors[0].qualifications; // bio facts, quoted from the page
|
|
14
|
+
* ```
|
|
15
|
+
*
|
|
16
|
+
* Throws when the page cannot be fetched or the model call fails; a missing
|
|
17
|
+
* author or date is not an error, it comes back flagged in `needsReview`.
|
|
18
|
+
*/
|
|
19
|
+
export declare function extractCiteLLM(options: ExtractCiteLLMOptions): Promise<ExtractCiteLLMResult>;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { CitationData, CitationStyle, FormattedCitation } from './types';
|
|
2
|
+
interface DateParts {
|
|
3
|
+
y?: number;
|
|
4
|
+
m?: number;
|
|
5
|
+
d?: number;
|
|
6
|
+
}
|
|
7
|
+
export declare function parseDateParts(iso: string | undefined): DateParts;
|
|
8
|
+
export declare function formatAPA(c: CitationData): string;
|
|
9
|
+
export declare function formatMLA(c: CitationData): string;
|
|
10
|
+
export declare function formatChicago(c: CitationData, accessed?: string): string;
|
|
11
|
+
export declare function formatHarvard(c: CitationData, accessed?: string): string;
|
|
12
|
+
export declare function formatIEEE(c: CitationData, accessed?: string): string;
|
|
13
|
+
export declare function bibtexKey(c: CitationData): string;
|
|
14
|
+
export declare function formatBibTeX(c: CitationData, accessed?: string): string;
|
|
15
|
+
/**
|
|
16
|
+
* Writes `citation` in one style. `accessedDate` (ISO) defaults to the one on
|
|
17
|
+
* the citation; styles that print an access date use it when the page is undated.
|
|
18
|
+
*/
|
|
19
|
+
export declare function formatCitation(citation: CitationData, style: CitationStyle): FormattedCitation;
|
|
20
|
+
/** Every requested style at once. */
|
|
21
|
+
export declare function formatCitations(citation: CitationData, styles?: CitationStyle[]): Partial<Record<CitationStyle, FormattedCitation>>;
|
|
22
|
+
export {};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export * from './types';
|
|
2
|
+
export * from './call-llm';
|
|
3
|
+
export * from './extract-cite-llm';
|
|
4
|
+
export * from './format-citation';
|
|
5
|
+
export { preparePage, fetchPageHTML, clipText } from './page-text';
|
|
6
|
+
export { parseCitationReply, normalizeDate } from './parse-reply';
|
|
7
|
+
export { SYSTEM_PROMPT, buildUserPrompt, missingFields } from './prompt';
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export interface PageInput {
|
|
2
|
+
/** Visible text, whitespace-collapsed and trimmed to the budget. */
|
|
3
|
+
text: string;
|
|
4
|
+
/** `name: content` lines for the meta tags that carry citation data. */
|
|
5
|
+
meta: string;
|
|
6
|
+
/** JSON-LD blocks, trimmed. */
|
|
7
|
+
jsonLd: string;
|
|
8
|
+
/** The page's `<title>`. */
|
|
9
|
+
pageTitle: string;
|
|
10
|
+
}
|
|
11
|
+
/** Fetches a page's HTML, throwing a readable error when it cannot. */
|
|
12
|
+
export declare function fetchPageHTML(url: string, doFetch?: typeof fetch, timeoutMs?: number): Promise<string>;
|
|
13
|
+
/**
|
|
14
|
+
* Head and tail of `text` within `max` characters: 3/4 from the start, 1/4 from
|
|
15
|
+
* the end, with a marker between. Short text is returned whole.
|
|
16
|
+
*/
|
|
17
|
+
export declare function clipText(text: string, max: number): string;
|
|
18
|
+
export declare function preparePage(html: string, options?: {
|
|
19
|
+
text?: string;
|
|
20
|
+
maxChars?: number;
|
|
21
|
+
}): PageInput;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { ExtractCiteResult } from '../html-to-cite/extract-cite';
|
|
2
|
+
import { CitationData, CitationField, ReviewItem } from './types';
|
|
3
|
+
/** A number clamped to 0–1; anything else counts as no confidence at all. */
|
|
4
|
+
export declare function clamp01(value: unknown): number;
|
|
5
|
+
/** Keeps `YYYY`, `YYYY-MM` or `YYYY-MM-DD` when it is a real date, else undefined. */
|
|
6
|
+
export declare function normalizeDate(value: unknown): string | undefined;
|
|
7
|
+
/** Is `needle` somewhere in `haystack`, ignoring case, punctuation and spacing? */
|
|
8
|
+
export declare function appearsIn(needle: string | undefined, haystack: string): boolean;
|
|
9
|
+
export interface ParsedCitation {
|
|
10
|
+
citation: CitationData;
|
|
11
|
+
confidence: Record<CitationField, number>;
|
|
12
|
+
needsReview: ReviewItem[];
|
|
13
|
+
/** Fields the model's value matches the regex pass on, or replaced. */
|
|
14
|
+
agreement: Partial<Record<CitationField, "agree" | "replaced" | "filled">>;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Normalizes and checks the model's reply.
|
|
18
|
+
* @param reply parsed JSON from the model
|
|
19
|
+
* @param context `pageText`: everything the model was shown (text, meta, JSON-LD) that a name or quote must appear in
|
|
20
|
+
* @param partial what the regex pass found, to compare against
|
|
21
|
+
*/
|
|
22
|
+
export declare function parseCitationReply(reply: unknown, context: {
|
|
23
|
+
pageText: string;
|
|
24
|
+
url?: string;
|
|
25
|
+
accessedDate?: string;
|
|
26
|
+
}, partial: ExtractCiteResult, reviewThreshold: number): ParsedCitation;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { ExtractCiteResult } from '../html-to-cite/extract-cite';
|
|
2
|
+
import { CitationField } from './types';
|
|
3
|
+
import { PageInput } from './page-text';
|
|
4
|
+
export declare const SYSTEM_PROMPT: string;
|
|
5
|
+
/** Which candidate fields the regex pass filled, for the prompt. */
|
|
6
|
+
export declare function describeCandidates(partial: ExtractCiteResult): Record<string, string | null>;
|
|
7
|
+
/** The citation fields the regex pass cannot supply at all. */
|
|
8
|
+
export declare function missingFields(partial: ExtractCiteResult): CitationField[];
|
|
9
|
+
export declare function buildUserPrompt(args: {
|
|
10
|
+
url?: string;
|
|
11
|
+
partial: ExtractCiteResult;
|
|
12
|
+
missing: CitationField[];
|
|
13
|
+
page: PageInput;
|
|
14
|
+
}): string;
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { ExtractCiteResult } from '../html-to-cite/extract-cite';
|
|
2
|
+
/** Output styles `formatCitation` can write. */
|
|
3
|
+
export declare const CITATION_STYLES: readonly ["apa", "mla", "chicago", "harvard", "ieee", "bibtex"];
|
|
4
|
+
export type CitationStyle = (typeof CITATION_STYLES)[number];
|
|
5
|
+
export type SourceType = "webpage" | "news-article" | "blog-post" | "journal-article" | "report" | "book" | "video" | "other";
|
|
6
|
+
export declare const SOURCE_TYPES: SourceType[];
|
|
7
|
+
/** What the page says about who an author is. Only ever from the document. */
|
|
8
|
+
export interface AuthorQualifications {
|
|
9
|
+
/** Job title or role, e.g. "Senior Fellow". */
|
|
10
|
+
jobTitle?: string;
|
|
11
|
+
/** Employer, university, think tank, newsroom. */
|
|
12
|
+
affiliation?: string;
|
|
13
|
+
/** Degrees and credentials, e.g. ["PhD", "JD"]. */
|
|
14
|
+
credentials?: string[];
|
|
15
|
+
/** Fields the author is described as an expert in. */
|
|
16
|
+
expertise?: string[];
|
|
17
|
+
/** Notable books, awards, past roles named in the bio. */
|
|
18
|
+
highlights?: string[];
|
|
19
|
+
/** The bio sentence(s) copied verbatim from the page. */
|
|
20
|
+
evidence?: string;
|
|
21
|
+
}
|
|
22
|
+
export interface CitationAuthor {
|
|
23
|
+
/** Name as printed, "Given Family" order. */
|
|
24
|
+
name: string;
|
|
25
|
+
given?: string;
|
|
26
|
+
family?: string;
|
|
27
|
+
/** Jr., III, … */
|
|
28
|
+
suffix?: string;
|
|
29
|
+
/** A group author (agency, newsroom, company): never inverted or abbreviated. */
|
|
30
|
+
isOrganization: boolean;
|
|
31
|
+
/** 0–1: how sure the model is this is a real author of this document. */
|
|
32
|
+
confidence: number;
|
|
33
|
+
qualifications?: AuthorQualifications;
|
|
34
|
+
}
|
|
35
|
+
/** Every part a bibliography entry can be built from. */
|
|
36
|
+
export interface CitationData {
|
|
37
|
+
sourceType: SourceType;
|
|
38
|
+
authors: CitationAuthor[];
|
|
39
|
+
title?: string;
|
|
40
|
+
/** Site, newspaper, magazine or journal the item sits in. */
|
|
41
|
+
containerTitle?: string;
|
|
42
|
+
publisher?: string;
|
|
43
|
+
/** ISO 8601 and as precise as the page is: `2024-03-05`, `2024-03` or `2024`. */
|
|
44
|
+
publishedDate?: string;
|
|
45
|
+
doi?: string;
|
|
46
|
+
volume?: string;
|
|
47
|
+
issue?: string;
|
|
48
|
+
pages?: string;
|
|
49
|
+
url?: string;
|
|
50
|
+
/** ISO date the citation was made; APA and MLA print it for pages that change. */
|
|
51
|
+
accessedDate?: string;
|
|
52
|
+
}
|
|
53
|
+
/** The fields that carry a confidence score. */
|
|
54
|
+
export type CitationField = "authors" | "title" | "containerTitle" | "publisher" | "publishedDate" | "doi" | "volume" | "issue" | "pages";
|
|
55
|
+
export declare const CITATION_FIELDS: CitationField[];
|
|
56
|
+
/** A field (or one author's qualifications) a human should check. */
|
|
57
|
+
export interface ReviewItem {
|
|
58
|
+
/** A citation field, or `qualifications:<author name>`. */
|
|
59
|
+
field: string;
|
|
60
|
+
confidence: number;
|
|
61
|
+
reason: string;
|
|
62
|
+
}
|
|
63
|
+
export interface FormattedCitation {
|
|
64
|
+
/** Plain text. */
|
|
65
|
+
text: string;
|
|
66
|
+
/** The same entry with `<i>` for italics, every value HTML-escaped. */
|
|
67
|
+
html: string;
|
|
68
|
+
}
|
|
69
|
+
export interface ExtractCiteLLMResult {
|
|
70
|
+
citation: CitationData;
|
|
71
|
+
/** Per-field confidence, 0–1. */
|
|
72
|
+
confidence: Record<CitationField, number>;
|
|
73
|
+
/** Mean of the fields APA needs: authors, date, title, container. */
|
|
74
|
+
overallConfidence: number;
|
|
75
|
+
/** Fields and author bios below the review threshold or that failed a check. */
|
|
76
|
+
needsReview: ReviewItem[];
|
|
77
|
+
/** Which fields the cheap regex/metadata pass found, before the model ran. */
|
|
78
|
+
partial: ExtractCiteResult;
|
|
79
|
+
/** Fields the regex pass left empty and the model was asked to fill. */
|
|
80
|
+
missing: CitationField[];
|
|
81
|
+
/** Who set each field: the regex pass, the model, or both (they agreed). */
|
|
82
|
+
origin: Partial<Record<CitationField, "regex" | "llm" | "both">>;
|
|
83
|
+
/** One entry per requested style. */
|
|
84
|
+
formatted: Partial<Record<CitationStyle, FormattedCitation>>;
|
|
85
|
+
model: string;
|
|
86
|
+
ms: number;
|
|
87
|
+
}
|
|
88
|
+
export interface ExtractCiteLLMOptions {
|
|
89
|
+
/** Page to cite. Fetched unless `html` or `text` is given. */
|
|
90
|
+
url?: string;
|
|
91
|
+
/** HTML you already have. */
|
|
92
|
+
html?: string;
|
|
93
|
+
/** Plain text of the page, used instead of text taken from `html`. */
|
|
94
|
+
text?: string;
|
|
95
|
+
/** API key. Default: `OPENROUTER_API_KEY` from the environment, when there is one. */
|
|
96
|
+
apiKey?: string;
|
|
97
|
+
/** Default `anthropic/claude-haiku-4.5`. Any model id the endpoint serves. */
|
|
98
|
+
model?: string;
|
|
99
|
+
/** Any OpenAI-compatible chat-completions API. Default OpenRouter. */
|
|
100
|
+
baseUrl?: string;
|
|
101
|
+
/** Styles to write. Default: all of them. */
|
|
102
|
+
styles?: CitationStyle[];
|
|
103
|
+
/** Fields scoring below this are flagged for review. Default 0.7. */
|
|
104
|
+
reviewThreshold?: number;
|
|
105
|
+
/** Characters of page text sent to the model. Default 12000. */
|
|
106
|
+
maxChars?: number;
|
|
107
|
+
/** Abort the model call after this many milliseconds. Default 60000. */
|
|
108
|
+
timeoutMs?: number;
|
|
109
|
+
/** ISO date printed as the access date. Default: today. */
|
|
110
|
+
accessedDate?: string;
|
|
111
|
+
/** Extra request headers (OpenRouter: `HTTP-Referer`, `X-Title`). */
|
|
112
|
+
headers?: Record<string, string>;
|
|
113
|
+
/** Replaces the global `fetch` for both the page and the model call. */
|
|
114
|
+
fetch?: typeof fetch;
|
|
115
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "extract-cite",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"module": "./dist/extract-cite.es.js",
|
|
5
|
+
"description": "Cite any webpage: regex/metadata extraction of author, date, title and source, plus an optional LLM pass (OpenRouter by default) that completes the APA parts, scores confidence per field, flags fields for review, extracts author qualifications, and formats APA, MLA, Chicago, Harvard, IEEE and BibTeX.",
|
|
6
|
+
"author": "vtempest <grokthiscontact@gmail.com>",
|
|
7
|
+
"license": "rights.institute/PROSPER",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/OpenSourceAGI/qwksearch-research-agent.git",
|
|
11
|
+
"directory": "packages/extract-cite"
|
|
12
|
+
},
|
|
13
|
+
"main": "./dist/extract-cite.cjs.js",
|
|
14
|
+
"types": "./dist/index.d.ts",
|
|
15
|
+
"exports": {
|
|
16
|
+
".": {
|
|
17
|
+
"types": "./dist/index.d.ts",
|
|
18
|
+
"import": "./dist/extract-cite.es.js",
|
|
19
|
+
"require": "./dist/extract-cite.cjs.js"
|
|
20
|
+
},
|
|
21
|
+
"./*": {
|
|
22
|
+
"types": "./src/*.ts",
|
|
23
|
+
"import": "./src/*",
|
|
24
|
+
"require": "./src/*"
|
|
25
|
+
}
|
|
26
|
+
},
|
|
27
|
+
"files": [
|
|
28
|
+
"dist",
|
|
29
|
+
"src"
|
|
30
|
+
],
|
|
31
|
+
"scripts": {
|
|
32
|
+
"build": "vite build",
|
|
33
|
+
"test": "vitest run",
|
|
34
|
+
"test:coverage": "vitest run --coverage"
|
|
35
|
+
},
|
|
36
|
+
"devDependencies": {
|
|
37
|
+
"@types/node": "^26.5.0",
|
|
38
|
+
"@vitest/coverage-v8": "^5.0.0",
|
|
39
|
+
"terser": "^5.51.2",
|
|
40
|
+
"typescript": "^7.0.2",
|
|
41
|
+
"vite": "^8.2.2",
|
|
42
|
+
"vite-plugin-dts": "^5.1.0",
|
|
43
|
+
"vitest": "^5.0.0"
|
|
44
|
+
},
|
|
45
|
+
"dependencies": {
|
|
46
|
+
"chrono-node": "^2.10.1",
|
|
47
|
+
"linkedom": "^0.18.13"
|
|
48
|
+
},
|
|
49
|
+
"keywords": [
|
|
50
|
+
"citation",
|
|
51
|
+
"apa",
|
|
52
|
+
"mla",
|
|
53
|
+
"chicago",
|
|
54
|
+
"bibliography",
|
|
55
|
+
"openrouter",
|
|
56
|
+
"llm",
|
|
57
|
+
"metadata",
|
|
58
|
+
"author",
|
|
59
|
+
"scraping"
|
|
60
|
+
]
|
|
61
|
+
}
|