@framers/agentos-ext-citation-check 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +119 -11
- package/dist/claims.d.ts +281 -0
- package/dist/claims.d.ts.map +1 -0
- package/dist/claims.js +478 -0
- package/dist/claims.js.map +1 -0
- package/dist/firecrawl.d.ts +77 -0
- package/dist/firecrawl.d.ts.map +1 -0
- package/dist/firecrawl.js +184 -0
- package/dist/firecrawl.js.map +1 -0
- package/dist/index.d.ts +8 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -1
- package/dist/index.js.map +1 -1
- package/dist/match.d.ts +2 -1
- package/dist/match.d.ts.map +1 -1
- package/dist/match.js +11 -2
- package/dist/match.js.map +1 -1
- package/dist/meta.d.ts +6 -1
- package/dist/meta.d.ts.map +1 -1
- package/dist/meta.js +14 -3
- package/dist/meta.js.map +1 -1
- package/dist/page.d.ts +45 -8
- package/dist/page.d.ts.map +1 -1
- package/dist/page.js +254 -39
- package/dist/page.js.map +1 -1
- package/dist/parse.d.ts +2 -1
- package/dist/parse.d.ts.map +1 -1
- package/dist/parse.js +54 -3
- package/dist/parse.js.map +1 -1
- package/dist/web-source.d.ts +36 -8
- package/dist/web-source.d.ts.map +1 -1
- package/dist/web-source.js +74 -16
- package/dist/web-source.js.map +1 -1
- package/manifest.json +2 -2
- package/package.json +8 -7
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file firecrawl.ts
|
|
3
|
+
* @description A reader for a page the page reader's own request found no text in, such as one a script draws, over
|
|
4
|
+
* Firecrawl's scrape endpoint with the host's key. It sends one request a page, carrying the page's address and
|
|
5
|
+
* nothing else of the page or of the search that found it, and answers the page's text as blocks the same way the
|
|
6
|
+
* reader makes its own, or null.
|
|
7
|
+
*
|
|
8
|
+
* Firecrawl's scrape endpoint, from its API reference (https://docs.firecrawl.dev/api-reference/endpoint/scrape, the
|
|
9
|
+
* OpenAPI document `v2-openapi.json`, read on 11 November 2026): `POST https://api.firecrawl.dev/v2/scrape` with
|
|
10
|
+
* `Authorization: Bearer <key>` and a JSON body whose one required field is `url`; `formats` is a list of format names
|
|
11
|
+
* (`html` asks for the page's cleaned HTML, its scripts and styles removed), `onlyMainContent` (true when left out) cuts
|
|
12
|
+
* the headers, navigation and footers, `timeout` is Firecrawl's own deadline in milliseconds, from 1,000 to 300,000
|
|
13
|
+
* (60,000 when left out), and `maxAge` lets it answer a cached copy younger than that many milliseconds (172,800,000, two
|
|
14
|
+
* days, when left out). A 200 answers `{ success, data: { html, metadata: { statusCode, url } } }`, where `statusCode`
|
|
15
|
+
* is the status the page itself answered and `url` the page's address after the redirects Firecrawl followed; an error
|
|
16
|
+
* answers 400, 402, 403, 404, 409, 429, 500 or 503. Firecrawl's pricing
|
|
17
|
+
* (https://www.firecrawl.dev/pricing, read the same day) counts a scrape as 1 credit a page, a page that answers an
|
|
18
|
+
* error status included, and a scrape that returns no result as none.
|
|
19
|
+
*
|
|
20
|
+
* @module agentos/extensions/research/citation-check/firecrawl
|
|
21
|
+
*/
|
|
22
|
+
import { originOf, pageBlocks, readCapped, whenAborted } from './page.js';
|
|
23
|
+
/** Firecrawl's scrape endpoint, where the reader sends when the host names no other. */
|
|
24
|
+
export const FIRECRAWL_SCRAPE_URL = 'https://api.firecrawl.dev/v2/scrape';
|
|
25
|
+
/** The shortest and the longest deadline Firecrawl's scrape takes, in milliseconds. */
|
|
26
|
+
const FIRECRAWL_TIMEOUT_FLOOR = 1_000;
|
|
27
|
+
const FIRECRAWL_TIMEOUT_CEILING = 300_000;
|
|
28
|
+
/** A whole number of at least 1. */
|
|
29
|
+
function whole(name, value) {
|
|
30
|
+
if (typeof value !== 'number' || !Number.isInteger(value) || value < 1)
|
|
31
|
+
throw new TypeError(`${name} must be a whole number of at least 1`);
|
|
32
|
+
return value;
|
|
33
|
+
}
|
|
34
|
+
/** Whether a host name is the loopback address: `localhost`, 127.0.0.0/8 or `[::1]`. */
|
|
35
|
+
function isLoopback(hostname) {
|
|
36
|
+
return hostname === 'localhost' || hostname === '[::1]' || /^127(?:\.\d{1,3}){3}$/u.test(hostname);
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* The endpoint the key may go to: an https address, or plain http to the loopback address (a stand-in in a test), so
|
|
40
|
+
* the key never crosses a network in the clear. A refusal names the option alone.
|
|
41
|
+
*/
|
|
42
|
+
function endpointOf(value) {
|
|
43
|
+
if (value === undefined)
|
|
44
|
+
return FIRECRAWL_SCRAPE_URL;
|
|
45
|
+
try {
|
|
46
|
+
const { protocol, hostname } = new URL(String(value));
|
|
47
|
+
if (protocol === 'https:' || (protocol === 'http:' && isLoopback(hostname)))
|
|
48
|
+
return String(value);
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
// Refused below.
|
|
52
|
+
}
|
|
53
|
+
throw new TypeError('the Firecrawl scrape endpoint must be an https address, or an http address on the loopback address');
|
|
54
|
+
}
|
|
55
|
+
/** Cancels a body without waiting: a stream whose cancel never settles must not hold the answer. */
|
|
56
|
+
function discard(response) {
|
|
57
|
+
if (response.body !== null && !response.body.locked)
|
|
58
|
+
void response.body.cancel().catch(() => undefined);
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* The page's HTML from a scrape's answer, or null for a failed scrape, an error page, a page Firecrawl ended on at
|
|
63
|
+
* another origin than the address asked for, or an answer with none.
|
|
64
|
+
*/
|
|
65
|
+
function htmlOf(answer, asked) {
|
|
66
|
+
if (answer === null || typeof answer !== 'object')
|
|
67
|
+
return null;
|
|
68
|
+
const { success, data } = answer;
|
|
69
|
+
if (success !== true || data === null || typeof data !== 'object')
|
|
70
|
+
return null;
|
|
71
|
+
const { html, metadata } = data;
|
|
72
|
+
const meta = metadata !== null && typeof metadata === 'object' ? metadata : {};
|
|
73
|
+
// Firecrawl answers a page that answered an error status with that page's text, which is no text of the page cited.
|
|
74
|
+
if (typeof meta.statusCode === 'number' && (meta.statusCode < 200 || meta.statusCode > 299))
|
|
75
|
+
return null;
|
|
76
|
+
// Firecrawl follows redirects itself and names the page it ended on; one at another origin is not the page cited, and
|
|
77
|
+
// neither the host's rule nor robots.txt cleared it.
|
|
78
|
+
if (typeof meta.url === 'string') {
|
|
79
|
+
const ended = originOf(meta.url);
|
|
80
|
+
if (ended === 'unparsable' || ended !== originOf(asked))
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
return typeof html === 'string' && html !== '' ? html : null;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* A {@link PageFallbackReader} over Firecrawl's scrape endpoint. `read(url, { signal })` sends one request for the
|
|
87
|
+
* address, the HTML of the whole page asked for, and answers its text as blocks through the reader's own HTML reading,
|
|
88
|
+
* held to `textBudget`; null for an error status, an answer over `maxBytes`, an answer that is not a scrape's, a page
|
|
89
|
+
* that answered an error status, no text, or the reader's own deadline. The caller's cancellation ends the request and
|
|
90
|
+
* is thrown.
|
|
91
|
+
*/
|
|
92
|
+
export class FirecrawlPageReader {
|
|
93
|
+
// Private fields stay out of `JSON.stringify` and `util.inspect`, so the key is shown nowhere.
|
|
94
|
+
#apiKey;
|
|
95
|
+
#url;
|
|
96
|
+
#fetchFn;
|
|
97
|
+
#timeoutMs;
|
|
98
|
+
#maxBytes;
|
|
99
|
+
#textBudget;
|
|
100
|
+
/**
|
|
101
|
+
* @param options - The key, the endpoint and the bounds.
|
|
102
|
+
* @throws {TypeError} For an empty key, an endpoint that is neither an https address nor an http address on the
|
|
103
|
+
* loopback address, or a bound that is not a whole number of at least 1; the message never holds the key.
|
|
104
|
+
*/
|
|
105
|
+
constructor(options) {
|
|
106
|
+
if (typeof options.apiKey !== 'string' || options.apiKey.trim() === '')
|
|
107
|
+
throw new TypeError('a Firecrawl reader needs an API key');
|
|
108
|
+
this.#apiKey = options.apiKey;
|
|
109
|
+
this.#url = endpointOf(options.url);
|
|
110
|
+
this.#fetchFn = options.fetchFn;
|
|
111
|
+
this.#timeoutMs = whole('timeoutMs', options.timeoutMs);
|
|
112
|
+
this.#maxBytes = whole('maxBytes', options.maxBytes);
|
|
113
|
+
this.#textBudget = whole('textBudget', options.textBudget);
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Reads a page's text through Firecrawl. The request and its body are raced against the deadline and the caller's
|
|
117
|
+
* signal, so a `fetch` of the host's that does not watch the signal still ends with them; a body is cancelled without
|
|
118
|
+
* waiting for the cancel to settle.
|
|
119
|
+
*
|
|
120
|
+
* @param url - The page's address, the one request's only page-borne field.
|
|
121
|
+
* @param options - The caller's signal, which ends the request.
|
|
122
|
+
* @throws The caller's cancellation reason, whenever it fired before the answer.
|
|
123
|
+
*/
|
|
124
|
+
async read(url, options) {
|
|
125
|
+
const caller = options.signal;
|
|
126
|
+
// A signal that has already fired sends nothing.
|
|
127
|
+
caller.throwIfAborted();
|
|
128
|
+
const signal = AbortSignal.any([caller, AbortSignal.timeout(this.#timeoutMs)]);
|
|
129
|
+
const ended = whenAborted(signal);
|
|
130
|
+
let blocks = null;
|
|
131
|
+
try {
|
|
132
|
+
const sent = this.#send(url, signal);
|
|
133
|
+
// A response that comes after the read ended is cancelled, so its connection is not left open.
|
|
134
|
+
void ended.promise.catch(() => sent.then(discard, () => undefined));
|
|
135
|
+
blocks = await this.#blocksOf(url, await Promise.race([sent, ended.promise]), signal);
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
// Every failure answers null; the caller's cancellation is thrown below.
|
|
139
|
+
}
|
|
140
|
+
finally {
|
|
141
|
+
ended.release();
|
|
142
|
+
}
|
|
143
|
+
caller.throwIfAborted();
|
|
144
|
+
return blocks;
|
|
145
|
+
}
|
|
146
|
+
/** The one request: the page's address, the format and the deadline, with the key in the `Authorization` header. */
|
|
147
|
+
#send(url, signal) {
|
|
148
|
+
const fetchFn = this.#fetchFn ?? globalThis.fetch;
|
|
149
|
+
return fetchFn(this.#url, {
|
|
150
|
+
method: 'POST',
|
|
151
|
+
headers: { authorization: `Bearer ${this.#apiKey}`, 'content-type': 'application/json', accept: 'application/json' },
|
|
152
|
+
// The whole page, as the reader reads its own fetches, rather than Firecrawl's main-content cut; and a fresh
|
|
153
|
+
// scrape, since Firecrawl answers a cached copy up to two days old unless `maxAge` says otherwise, and the
|
|
154
|
+
// document is dated by the reader's clock.
|
|
155
|
+
body: JSON.stringify({
|
|
156
|
+
url,
|
|
157
|
+
formats: ['html'],
|
|
158
|
+
onlyMainContent: false,
|
|
159
|
+
maxAge: 0,
|
|
160
|
+
timeout: Math.min(FIRECRAWL_TIMEOUT_CEILING, Math.max(FIRECRAWL_TIMEOUT_FLOOR, this.#timeoutMs)),
|
|
161
|
+
}),
|
|
162
|
+
// The key goes to the endpoint alone, never to an address a redirect names.
|
|
163
|
+
redirect: 'error',
|
|
164
|
+
signal,
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
/** The page's text from Firecrawl's answer for the address asked for, or null. */
|
|
168
|
+
async #blocksOf(url, response, signal) {
|
|
169
|
+
if (!response.ok)
|
|
170
|
+
return discard(response);
|
|
171
|
+
const declared = Number(response.headers.get('content-length') ?? '');
|
|
172
|
+
if (Number.isFinite(declared) && declared > this.#maxBytes)
|
|
173
|
+
return discard(response);
|
|
174
|
+
const bytes = await readCapped(response, this.#maxBytes, signal);
|
|
175
|
+
if (bytes === null)
|
|
176
|
+
return null;
|
|
177
|
+
const html = htmlOf(JSON.parse(new TextDecoder('utf-8').decode(bytes)), url);
|
|
178
|
+
if (html === null)
|
|
179
|
+
return null;
|
|
180
|
+
const blocks = pageBlocks('html', html, this.#textBudget);
|
|
181
|
+
return blocks.length === 0 ? null : blocks;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
//# sourceMappingURL=firecrawl.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"firecrawl.js","sourceRoot":"","sources":["../src/firecrawl.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAA2B,MAAM,WAAW,CAAC;AAEnG,wFAAwF;AACxF,MAAM,CAAC,MAAM,oBAAoB,GAAG,qCAAqC,CAAC;AAE1E,uFAAuF;AACvF,MAAM,uBAAuB,GAAG,KAAK,CAAC;AACtC,MAAM,yBAAyB,GAAG,OAAO,CAAC;AA2B1C,oCAAoC;AACpC,SAAS,KAAK,CAAC,IAAY,EAAE,KAAc;IACzC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,uCAAuC,CAAC,CAAC;IAC5I,OAAO,KAAK,CAAC;AACf,CAAC;AAED,wFAAwF;AACxF,SAAS,UAAU,CAAC,QAAgB;IAClC,OAAO,QAAQ,KAAK,WAAW,IAAI,QAAQ,KAAK,OAAO,IAAI,wBAAwB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AACrG,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,KAAc;IAChC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,oBAAoB,CAAC;IACrD,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACtD,IAAI,QAAQ,KAAK,QAAQ,IAAI,CAAC,QAAQ,KAAK,OAAO,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC;YAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACpG,CAAC;IAAC,MAAM,CAAC;QACP,iBAAiB;IACnB,CAAC;IACD,MAAM,IAAI,SAAS,CAAC,oGAAoG,CAAC,CAAC;AAC5H,CAAC;AAED,oGAAoG;AACpG,SAAS,OAAO,CAAC,QAAkB;IACjC,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM;QAAE,KAAK,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACxG,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,MAAM,CAAC,MAAe,EAAE,KAAa;IAC5C,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC/D,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,MAA+C,CAAC;IAC1E,IAAI,OAAO,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC/E,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAA8C,CAAC;IAC1E,MAAM,IAAI,GAAG,QAAQ,KAAK,IAAI,IAAI,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAE,QAAoD,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5H,oHAAoH;IACpH,IAAI,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,GAAG,IAAI,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACzG,sHAAsH;IACtH,qDAAqD;IACrD,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,KAAK,YAAY,IAAI,KAAK,KAAK,QAAQ,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;IACvE,CAAC;IACD,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AAC/D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,OAAO,mBAAmB;IAC9B,+FAA+F;IACtF,OAAO,CAAS;IAChB,IAAI,CAAS;IACb,QAAQ,CAAsC;IAC9C,UAAU,CAAS;IACnB,SAAS,CAAS;IAClB,WAAW,CAAS;IAE7B;;;;OAIG;IACH,YAAY,OAAmC;QAC7C,IAAI,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;QACnI,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;QAChC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QACxD,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAC7D,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,OAAgC;QACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,iDAAiD;QACjD,MAAM,CAAC,cAAc,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,MAAM,GAAoB,IAAI,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACrC,+FAA+F;YAC/F,KAAK,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;YACpE,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACxF,CAAC;QAAC,MAAM,CAAC;YACP,yEAAyE;QAC3E,CAAC;gBAAS,CAAC;YACT,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,CAAC;QACD,MAAM,CAAC,cAAc,EAAE,CAAC;QACxB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,oHAAoH;IACpH,KAAK,CAAC,GAAW,EAAE,MAAmB;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC;QAClD,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE;YACxB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,EAAE,kBAAkB,EAAE;YACpH,6GAA6G;YAC7G,2GAA2G;YAC3G,2CAA2C;YAC3C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,GAAG;gBACH,OAAO,EAAE,CAAC,MAAM,CAAC;gBACjB,eAAe,EAAE,KAAK;gBACtB,MAAM,EAAE,CAAC;gBACT,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,yBAAyB,EAAE,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;aACjG,CAAC;YACF,4EAA4E;YAC5E,QAAQ,EAAE,OAAO;YACjB,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAED,kFAAkF;IAClF,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,QAAkB,EAAE,MAAmB;QAClE,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,OAAO,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC;QACtE,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,IAAI,CAAC,SAAS;YAAE,OAAO,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrF,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACjE,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC7E,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC/B,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC1D,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IAC7C,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -6,7 +6,10 @@
|
|
|
6
6
|
* given), `renderCitationLog` writes the markdown for a "Citation check log" section, and `citationReportFails` decides
|
|
7
7
|
* the result. The pack's one tool is `citation_verify`; the `agentos-citations` command runs the same check. The page
|
|
8
8
|
* reader is exported (`readPage` with its transports, the host rule and robots.txt on every hop, the licence read),
|
|
9
|
-
* and `WebPageSourceProvider` makes the open web a public source over a search the host gives.
|
|
9
|
+
* and `WebPageSourceProvider` makes the open web a public source over a search the host gives. `FirecrawlPageReader`
|
|
10
|
+
* reads a page with no text through Firecrawl's scrape endpoint with the host's key. `ClaimChecker` checks a claim heard
|
|
11
|
+
* in a room against passages from any source, with one judge call (`claimJudge` on AgentOS's `generateObject`) and the
|
|
12
|
+
* grounding guard's decision by code.
|
|
10
13
|
*
|
|
11
14
|
* @module @framers/agentos-ext-citation-check
|
|
12
15
|
*/
|
|
@@ -16,15 +19,17 @@ import type { EmbedFn, NliFn } from './types.js';
|
|
|
16
19
|
export * from './types.js';
|
|
17
20
|
export { parseCitedClaims, CitationInputError, isHttpUrl, type CitedClaimFormat } from './parse.js';
|
|
18
21
|
export { normalizeForMatch, quoteFragments, locateQuote, lexicalNearest, withoutInvisible } from './match.js';
|
|
19
|
-
export { pageKey, readPage, fetchTransport, egressTransport, PageRefusal, UNREAD_LICENCE, type EgressProfile, type PageScraper, type PageReaderOptions, type PageRead, type PageFallbackReader, type Transport, type TransportAnswer, type TransportRules, } from './page.js';
|
|
22
|
+
export { pageKey, readPage, fetchTransport, egressTransport, PageRefusal, UNREAD_LICENCE, type EgressProfile, type PageScraper, type PageReaderOptions, type PageRead, type PageFallbackReader, type Transport, type TransportAnswer, type TransportRules, type TransportLimits, } from './page.js';
|
|
20
23
|
export { readRobots, robotsAllows, parseRobots, ROBOTS_PARSING_FLOOR, type RobotsRule, type RobotsFile, type RobotsGroup, type RobotsLine, type ReadRobotsOptions } from './robots.js';
|
|
21
24
|
export { readLicence } from './licence.js';
|
|
22
|
-
export { WebPageSourceProvider, DEFAULT_ROBOTS, type WebPageSourceOptions } from './web-source.js';
|
|
25
|
+
export { WebPageSourceProvider, DEFAULT_ROBOTS, type WebPageSourceOptions, type WebSearchAnswer } from './web-source.js';
|
|
26
|
+
export { FirecrawlPageReader, FIRECRAWL_SCRAPE_URL, type FirecrawlPageReaderOptions } from './firecrawl.js';
|
|
23
27
|
export { checkCitations, checkCitationDocument, CitationSetupError, type CitationCheckOptions } from './runner.js';
|
|
24
28
|
export { renderCitationLog, citationReportFails, FAILING_VERDICTS, type CitationReportOptions } from './report.js';
|
|
25
29
|
export { openAIEmbeddings, managerEmbeddings, type OpenAIEmbeddingOptions, type EmbeddingManagerLike } from './embeddings.js';
|
|
26
30
|
export { CitationVerifyTool, type CitationVerifyInput, type CitationVerifyOutput } from './tools/citationVerify.js';
|
|
27
31
|
export { runCitationCli, CITATION_CLI_USAGE, type CitationCliIO } from './cli.js';
|
|
32
|
+
export { ClaimChecker, claimJudge, claimJudgeSystem, claimJudgePrompt, CLAIM_QUOTE_MAX_WORDS, type SpokenClaim, type ClaimPassage, type ClaimUsage, type ClaimJudgement, type ClaimJudge, type ClaimCheckerOptions, type ClaimCheckOutcome, type ClaimDecision, } from './claims.js';
|
|
28
33
|
/** The pack's options, from its manifest entry or from the host's code. */
|
|
29
34
|
export interface CitationCheckPackOptions {
|
|
30
35
|
/** `none` (the default) or `openai`: grade a claim not found word for word with OpenAI embeddings through AgentOS. */
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAE5E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEjD,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,SAAS,EAAE,KAAK,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACpG,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,WAAW,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9G,OAAO,EACL,OAAO,EACP,QAAQ,EACR,cAAc,EACd,eAAe,EACf,WAAW,EACX,cAAc,EACd,KAAK,aAAa,EAClB,KAAK,WAAW,EAChB,KAAK,iBAAiB,EACtB,KAAK,QAAQ,EACb,KAAK,kBAAkB,EACvB,KAAK,SAAS,EACd,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,eAAe,GACrB,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,oBAAoB,EAAE,KAAK,UAAU,EAAE,KAAK,UAAU,EAAE,KAAK,WAAW,EAAE,KAAK,UAAU,EAAE,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACvL,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,qBAAqB,EAAE,cAAc,EAAE,KAAK,oBAAoB,EAAE,KAAK,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACzH,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,KAAK,0BAA0B,EAAE,MAAM,gBAAgB,CAAC;AAC5G,OAAO,EAAE,cAAc,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,KAAK,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnH,OAAO,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,KAAK,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACnH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,KAAK,sBAAsB,EAAE,KAAK,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAC9H,OAAO,EAAE,kBAAkB,EAAE,KAAK,mBAAmB,EAAE,KAAK,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACpH,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,UAAU,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,gBAAgB,EAChB,qBAAqB,EACrB,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,UAAU,EACf,KAAK,cAAc,EACnB,KAAK,UAAU,EACf,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,aAAa,GACnB,MAAM,aAAa,CAAC;AAErB,2EAA2E;AAC3E,MAAM,WAAW,wBAAwB;IACvC,sHAAsH;IACtH,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC/B,mEAAmE;IACnE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0HAA0H;IAC1H,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,0FAA0F;IAC1F,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC,yEAAyE;IACzE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,qHAAqH;IACrH,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,yEAAyE;IACzE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,GAAE,oBAAyB,GAAG,aAAa,CAgCrF;AAED,eAAe,mBAAmB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -6,7 +6,10 @@
|
|
|
6
6
|
* given), `renderCitationLog` writes the markdown for a "Citation check log" section, and `citationReportFails` decides
|
|
7
7
|
* the result. The pack's one tool is `citation_verify`; the `agentos-citations` command runs the same check. The page
|
|
8
8
|
* reader is exported (`readPage` with its transports, the host rule and robots.txt on every hop, the licence read),
|
|
9
|
-
* and `WebPageSourceProvider` makes the open web a public source over a search the host gives.
|
|
9
|
+
* and `WebPageSourceProvider` makes the open web a public source over a search the host gives. `FirecrawlPageReader`
|
|
10
|
+
* reads a page with no text through Firecrawl's scrape endpoint with the host's key. `ClaimChecker` checks a claim heard
|
|
11
|
+
* in a room against passages from any source, with one judge call (`claimJudge` on AgentOS's `generateObject`) and the
|
|
12
|
+
* grounding guard's decision by code.
|
|
10
13
|
*
|
|
11
14
|
* @module @framers/agentos-ext-citation-check
|
|
12
15
|
*/
|
|
@@ -19,11 +22,13 @@ export { pageKey, readPage, fetchTransport, egressTransport, PageRefusal, UNREAD
|
|
|
19
22
|
export { readRobots, robotsAllows, parseRobots, ROBOTS_PARSING_FLOOR } from './robots.js';
|
|
20
23
|
export { readLicence } from './licence.js';
|
|
21
24
|
export { WebPageSourceProvider, DEFAULT_ROBOTS } from './web-source.js';
|
|
25
|
+
export { FirecrawlPageReader, FIRECRAWL_SCRAPE_URL } from './firecrawl.js';
|
|
22
26
|
export { checkCitations, checkCitationDocument, CitationSetupError } from './runner.js';
|
|
23
27
|
export { renderCitationLog, citationReportFails, FAILING_VERDICTS } from './report.js';
|
|
24
28
|
export { openAIEmbeddings, managerEmbeddings } from './embeddings.js';
|
|
25
29
|
export { CitationVerifyTool } from './tools/citationVerify.js';
|
|
26
30
|
export { runCitationCli, CITATION_CLI_USAGE } from './cli.js';
|
|
31
|
+
export { ClaimChecker, claimJudge, claimJudgeSystem, claimJudgePrompt, CLAIM_QUOTE_MAX_WORDS, } from './claims.js';
|
|
27
32
|
/**
|
|
28
33
|
* The pack: one tool, `citation_verify`. With `embeddings: 'openai'` the OpenAI key comes from the `openai.apiKey`
|
|
29
34
|
* secret, or `OPENAI_API_KEY` when the secret is not set.
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAEnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAG/D,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,SAAS,EAAyB,MAAM,YAAY,CAAC;AACpG,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,WAAW,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9G,OAAO,EACL,OAAO,EACP,QAAQ,EACR,cAAc,EACd,eAAe,EACf,WAAW,EACX,cAAc,GAUf,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,oBAAoB,EAA+F,MAAM,aAAa,CAAC;AACvL,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,qBAAqB,EAAE,cAAc,EAAmD,MAAM,iBAAiB,CAAC;AACzH,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAmC,MAAM,gBAAgB,CAAC;AAC5G,OAAO,EAAE,cAAc,EAAE,qBAAqB,EAAE,kBAAkB,EAA6B,MAAM,aAAa,CAAC;AACnH,OAAO,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,gBAAgB,EAA8B,MAAM,aAAa,CAAC;AACnH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAA0D,MAAM,iBAAiB,CAAC;AAC9H,OAAO,EAAE,kBAAkB,EAAuD,MAAM,2BAA2B,CAAC;AACpH,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAsB,MAAM,UAAU,CAAC;AAClF,OAAO,EACL,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,gBAAgB,EAChB,qBAAqB,GAStB,MAAM,aAAa,CAAC;AA0BrB;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAAC,UAAgC,EAAE;IACpE,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAA6B,CAAC;IACpE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,MAAM,CAAC;IAChD,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,QAAQ;QAAE,MAAM,IAAI,SAAS,CAAC,0CAA0C,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAClJ,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QAC9F,MAAM,IAAI,SAAS,CAAC,uCAAuC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC/F,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,eAAe,CAAC,CAAC;IACpD,MAAM,OAAO,GACX,OAAO,CAAC,OAAO;QACf,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACpK,MAAM,IAAI,GAAG,IAAI,kBAAkB,CAAC;QAClC,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;QACpG,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpE,CAAC,CAAC;IACH,OAAO;QACL,IAAI,EAAE,qCAAqC;QAC3C,WAAW,EAAE;YACX;gBACE,EAAE,EAAE,IAAI,CAAC,IAAI;gBACb,IAAI,EAAE,MAAM;gBACZ,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,EAAE;gBAChC,OAAO,EAAE,IAAI;gBACb,eAAe,EAAE,CAAC,EAAE,EAAE,EAAE,eAAe,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;aAC3D;SACF;KACF,CAAC;AACJ,CAAC;AAED,eAAe,mBAAmB,CAAC"}
|
package/dist/match.d.ts
CHANGED
|
@@ -12,7 +12,8 @@ import type { CitationPassage } from './types.js';
|
|
|
12
12
|
export declare function withoutInvisible(text: string): string;
|
|
13
13
|
/**
|
|
14
14
|
* A text as the checks read it: NFKC, the invisible characters removed, typographic and angle quotation marks plain,
|
|
15
|
-
* every dash a hyphen-minus with no space around it
|
|
15
|
+
* every dash a hyphen-minus with no space around it (one with white space after it before a digit keeps that space),
|
|
16
|
+
* lower case, white space collapsed.
|
|
16
17
|
*/
|
|
17
18
|
export declare function normalizeForMatch(text: string): string;
|
|
18
19
|
/**
|
package/dist/match.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"match.d.ts","sourceRoot":"","sources":["../src/match.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAWlD,yDAAyD;AACzD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAErD;
|
|
1
|
+
{"version":3,"file":"match.d.ts","sourceRoot":"","sources":["../src/match.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAWlD,yDAAyD;AACzD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAErD;AAWD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKtD;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAK1D;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,MAAM,EAAE,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAe7G;AAWD;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,eAAe,EAAE,CAgB3G"}
|
package/dist/match.js
CHANGED
|
@@ -20,15 +20,24 @@ const EDGES = /^[\s"',.;:!?]+|[\s"',.;:!?]+$/gu;
|
|
|
20
20
|
export function withoutInvisible(text) {
|
|
21
21
|
return text.replace(INVISIBLE, '');
|
|
22
22
|
}
|
|
23
|
+
/**
|
|
24
|
+
* A dash run as the checks read it: a hyphen-minus with no space around it, except one with white space after it and a
|
|
25
|
+
* digit next, which keeps one space after it. That dash is punctuation (`1887 — 4,200 ships`), and a hyphen-minus
|
|
26
|
+
* against a digit would read as the number's sign, so a quotation that starts at the number would cut it.
|
|
27
|
+
*/
|
|
28
|
+
function dashAsRead(run, offset, text) {
|
|
29
|
+
return /\s$/u.test(run) && /[0-9]/u.test(text.charAt(offset + run.length)) ? '- ' : '-';
|
|
30
|
+
}
|
|
23
31
|
/**
|
|
24
32
|
* A text as the checks read it: NFKC, the invisible characters removed, typographic and angle quotation marks plain,
|
|
25
|
-
* every dash a hyphen-minus with no space around it
|
|
33
|
+
* every dash a hyphen-minus with no space around it (one with white space after it before a digit keeps that space),
|
|
34
|
+
* lower case, white space collapsed.
|
|
26
35
|
*/
|
|
27
36
|
export function normalizeForMatch(text) {
|
|
28
37
|
const folded = fold(withoutInvisible(text.normalize('NFKC')))
|
|
29
38
|
.replace(/[«»]/gu, '"')
|
|
30
39
|
.replace(/[‹›]/gu, "'");
|
|
31
|
-
return collapse(folded.replace(DASHES,
|
|
40
|
+
return collapse(folded.replace(DASHES, dashAsRead).toLowerCase());
|
|
32
41
|
}
|
|
33
42
|
/**
|
|
34
43
|
* A quotation's pieces as {@link normalizeForMatch} reads them: the words between its gaps (an ellipsis, a bracketed
|
package/dist/match.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"match.js","sourceRoot":"","sources":["../src/match.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,6CAA6C,CAAC;AAG7G,mHAAmH;AACnH,MAAM,SAAS,GAAG,qCAAqC,CAAC;AACxD,0FAA0F;AAC1F,MAAM,MAAM,GAAG,mDAAmD,CAAC;AACnE,uGAAuG;AACvG,MAAM,GAAG,GAAG,8BAA8B,CAAC;AAC3C,8GAA8G;AAC9G,MAAM,KAAK,GAAG,iCAAiC,CAAC;AAEhD,yDAAyD;AACzD,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;AACrC,CAAC;AAED
|
|
1
|
+
{"version":3,"file":"match.js","sourceRoot":"","sources":["../src/match.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,6CAA6C,CAAC;AAG7G,mHAAmH;AACnH,MAAM,SAAS,GAAG,qCAAqC,CAAC;AACxD,0FAA0F;AAC1F,MAAM,MAAM,GAAG,mDAAmD,CAAC;AACnE,uGAAuG;AACvG,MAAM,GAAG,GAAG,8BAA8B,CAAC;AAC3C,8GAA8G;AAC9G,MAAM,KAAK,GAAG,iCAAiC,CAAC;AAEhD,yDAAyD;AACzD,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;AACrC,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,GAAW,EAAE,MAAc,EAAE,IAAY;IAC3D,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;AAC1F,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;SAC1D,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC1B,OAAO,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;AACpE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,SAAiB;IAC9C,OAAO,iBAAiB,CAAC,SAAS,CAAC;SAChC,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;SACxC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAChD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,SAA4B;IACpE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;IACf,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC;IACb,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,iHAAiH;QACjH,+CAA+C;QAC/C,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG;YAAE,MAAM,IAAI,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;QACrD,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC/B,IAAI,KAAK,KAAK,CAAC,CAAC;YAAE,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC;QAC9C,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC;QACxB,MAAM,GAAG,GAAG,CAAC;IACf,CAAC;IACD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC3C,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,UAAU,CAAC,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5G,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa,EAAE,QAA2B,EAAE,KAAa;IACtF,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,MAAM,GAA+C,EAAE,CAAC;IAC9D,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC/B,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO;QAC3B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACf,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,IAAI,IAAI,MAAM;YAAE,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,MAAM,IAAI,CAAC,CAAC;QAC3D,IAAI,MAAM,KAAK,CAAC;YAAE,OAAO;QACzB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IACnF,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;AACzF,CAAC"}
|
package/dist/meta.d.ts
CHANGED
|
@@ -24,6 +24,8 @@ export interface MarkupScan {
|
|
|
24
24
|
metas: Map<string, string>;
|
|
25
25
|
/** Each `<link>` whose `rel` names `license`, its attributes, in document order; at most 1,000. */
|
|
26
26
|
links: Attributes[];
|
|
27
|
+
/** Each `<link>` whose `rel` names `icon` (`icon`, `shortcut icon`), its attributes, in document order; at most 32. */
|
|
28
|
+
icons: Attributes[];
|
|
27
29
|
/**
|
|
28
30
|
* Each `<a>` a licence is read from, in document order: one whose `rel` names `license`, with the start of its
|
|
29
31
|
* content as written, and one whose address, resolved against the page's own, is on creativecommons.org, with no
|
|
@@ -49,7 +51,10 @@ export declare function cleanAttribute(value: string): string;
|
|
|
49
51
|
* @param baseUrl - The page's own address, which an anchor's address is resolved against.
|
|
50
52
|
*/
|
|
51
53
|
export declare function scanMarkup(html: string, baseUrl: string): MarkupScan;
|
|
52
|
-
/**
|
|
54
|
+
/**
|
|
55
|
+
* Whether an element's `rel` names a link type: its tokens split on ASCII white space alone and read without ASCII case,
|
|
56
|
+
* as HTML reads them, so a no-break space (` ` or U+00A0) joins one token.
|
|
57
|
+
*/
|
|
53
58
|
export declare function relNames(attributes: Attributes, type: string): boolean;
|
|
54
59
|
/** An address as an attribute writes it, its references read and resolved against the page's own; null when it does not parse or is not http or https. */
|
|
55
60
|
export declare function resolveHttp(href: string | undefined, base: string): string | null;
|
package/dist/meta.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"meta.d.ts","sourceRoot":"","sources":["../src/meta.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,wEAAwE;AACxE,eAAO,MAAM,QAAQ,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"meta.d.ts","sourceRoot":"","sources":["../src/meta.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,wEAAwE;AACxE,eAAO,MAAM,QAAQ,MAAM,CAAC;AAkB5B,uGAAuG;AACvG,MAAM,MAAM,UAAU,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE7C,gDAAgD;AAChD,MAAM,WAAW,UAAU;IACzB,6EAA6E;IAC7E,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,mHAAmH;IACnH,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3B,mGAAmG;IACnG,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,uHAAuH;IACvH,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB;;;;OAIG;IACH,OAAO,EAAE,KAAK,CAAC;QAAE,UAAU,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACzD,6DAA6D;IAC7D,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,0EAA0E;AAC1E,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAMzD;AAED,8HAA8H;AAC9H,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEhD;AAED,wHAAwH;AACxH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEpD;AAmDD;;;;;GAKG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,UAAU,CA+DpE;AAED;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAMtE;AAED,0JAA0J;AAC1J,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CASjF;AAED,yGAAyG;AACzG,wBAAgB,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,GAAG,IAAI,CAIzD;AAED,qIAAqI;AACrI,wBAAgB,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CA0BrF;AAED,0GAA0G;AAC1G,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAIrE;AAQD,+GAA+G;AAC/G,wBAAgB,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,MAAM,GAAG,IAAI,CAepH;AAED,iHAAiH;AACjH,wBAAgB,UAAU,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAQhJ"}
|
package/dist/meta.js
CHANGED
|
@@ -25,6 +25,8 @@ const LD_BLOCKS = 20;
|
|
|
25
25
|
const LD_NODES = 500;
|
|
26
26
|
/** The most `<link rel="license">`s one page keeps, and the most `<a>`s of each kind a licence is read from. */
|
|
27
27
|
const LICENCE_ELEMENTS_MAX = 1_000;
|
|
28
|
+
/** The most `<link rel="icon">`s one page keeps. */
|
|
29
|
+
const ICON_LINKS_MAX = 32;
|
|
28
30
|
/** Elements whose content is not read for tags. */
|
|
29
31
|
const OPAQUE = new Set(['script', 'style', 'template', 'noscript', 'textarea', 'title']);
|
|
30
32
|
const ATTRIBUTE = /([^\s=/>"']+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/gu;
|
|
@@ -104,7 +106,7 @@ function onCreativeCommons(href, base) {
|
|
|
104
106
|
* @param baseUrl - The page's own address, which an anchor's address is resolved against.
|
|
105
107
|
*/
|
|
106
108
|
export function scanMarkup(html, baseUrl) {
|
|
107
|
-
const scan = { title: null, metas: new Map(), links: [], anchors: [], jsonLd: [] };
|
|
109
|
+
const scan = { title: null, metas: new Map(), links: [], icons: [], anchors: [], jsonLd: [] };
|
|
108
110
|
let licenceAnchors = 0;
|
|
109
111
|
let deedAnchors = 0;
|
|
110
112
|
let at = 0;
|
|
@@ -151,6 +153,8 @@ export function scanMarkup(html, baseUrl) {
|
|
|
151
153
|
if (name === 'link') {
|
|
152
154
|
if (marked && scan.links.length < LICENCE_ELEMENTS_MAX)
|
|
153
155
|
scan.links.push(attributes);
|
|
156
|
+
if (relNames(attributes, 'icon') && scan.icons.length < ICON_LINKS_MAX)
|
|
157
|
+
scan.icons.push(attributes);
|
|
154
158
|
}
|
|
155
159
|
else if (marked) {
|
|
156
160
|
if (licenceAnchors < LICENCE_ELEMENTS_MAX) {
|
|
@@ -180,9 +184,16 @@ export function scanMarkup(html, baseUrl) {
|
|
|
180
184
|
}
|
|
181
185
|
return scan;
|
|
182
186
|
}
|
|
183
|
-
/**
|
|
187
|
+
/**
|
|
188
|
+
* Whether an element's `rel` names a link type: its tokens split on ASCII white space alone and read without ASCII case,
|
|
189
|
+
* as HTML reads them, so a no-break space (` ` or U+00A0) joins one token.
|
|
190
|
+
*/
|
|
184
191
|
export function relNames(attributes, type) {
|
|
185
|
-
|
|
192
|
+
// Its character references read first: `rel="icon"` names the icon as `rel="icon"` does.
|
|
193
|
+
return decodeHTMLAttribute(attributes.get('rel') ?? '')
|
|
194
|
+
.replace(/[A-Z]/gu, (letter) => letter.toLowerCase())
|
|
195
|
+
.split(/[\t\n\f\r ]+/u)
|
|
196
|
+
.includes(type);
|
|
186
197
|
}
|
|
187
198
|
/** An address as an attribute writes it, its references read and resolved against the page's own; null when it does not parse or is not http or https. */
|
|
188
199
|
export function resolveHttp(href, base) {
|
package/dist/meta.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"meta.js","sourceRoot":"","sources":["../src/meta.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,6CAA6C,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,wEAAwE;AACxE,MAAM,CAAC,MAAM,QAAQ,GAAG,GAAG,CAAC;AAE5B,0GAA0G;AAC1G,MAAM,cAAc,GAAG,MAAM,CAAC;AAC9B,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,qFAAqF;AACrF,MAAM,SAAS,GAAG,EAAE,CAAC;AACrB,MAAM,QAAQ,GAAG,GAAG,CAAC;AACrB,gHAAgH;AAChH,MAAM,oBAAoB,GAAG,KAAK,CAAC;AACnC,mDAAmD;AACnD,MAAM,MAAM,GAAwB,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;AAE9G,MAAM,SAAS,GAAG,mEAAmE,CAAC;AACtF,MAAM,QAAQ,GAAG,8BAA8B,CAAC;AAuBhD,0EAA0E;AAC1E,MAAM,UAAU,OAAO,CAAC,IAAY,EAAE,GAAW;IAC/C,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IACpC,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC5C,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM;QAAE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7D,OAAO,GAAG,CAAC;AACb,CAAC;AAED,8HAA8H;AAC9H,MAAM,UAAU,SAAS,CAAC,MAAc;IACtC,OAAO,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACnG,CAAC;AAED,wHAAwH;AACxH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,QAAQ,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACjF,CAAC;AAED,iGAAiG;AACjG,SAAS,YAAY,CAAC,MAAc;IAClC,MAAM,GAAG,GAAe,IAAI,GAAG,EAAE,CAAC;IAClC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,GAAI,KAAK,CAAC,CAAC,CAAY,CAAC,WAAW,EAAE,CAAC;QAChD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,SAAS,MAAM,CAAC,IAAY,EAAE,IAAY;IACxC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,KAAK,IAAI,EAAE,GAAG,IAAI,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACjC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAChB,IAAI,IAAI,KAAK,KAAK;gBAAE,KAAK,GAAG,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;YACpD,KAAK,GAAG,IAAI,CAAC;YACb,WAAW,GAAG,KAAK,CAAC;YACpB,SAAS;QACX,CAAC;QACD,IAAI,IAAI,KAAK,IAAI;YAAE,WAAW,GAAG,IAAI,CAAC;aACjC,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI;YAAE,WAAW,GAAG,KAAK,CAAC;IAClH,CAAC;IACD,OAAO,CAAC,CAAC,CAAC;AACZ,CAAC;AAED,4GAA4G;AAC5G,SAAS,UAAU,CAAC,IAAY,EAAE,IAAY,EAAE,IAAY;IAC1D,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,MAAM,IAAI,MAAM,EAAE,KAAK,CAAC,CAAC;IAC9D,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IACzB,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjC,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;AAC5F,CAAC;AAED,sIAAsI;AACtI,SAAS,iBAAiB,CAAC,IAAwB,EAAE,IAAY;IAC/D,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACpC,OAAO,GAAG,KAAK,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,KAAK,qBAAqB,CAAC;AAC9G,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,OAAe;IACtD,MAAM,IAAI,GAAe,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IAC/F,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACnC,IAAI,IAAI,KAAK,CAAC,CAAC;YAAE,MAAM;QACvB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;YAC5C,IAAI,KAAK,KAAK,CAAC,CAAC;gBAAE,MAAM;YACxB,EAAE,GAAG,KAAK,GAAG,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QACD,QAAQ,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC;YACd,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAChC,6EAA6E;QAC7E,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,MAAM;QACtB,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;QACb,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QACvC,MAAM,IAAI,GAAI,KAAK,CAAC,CAAC,CAAY,CAAC,WAAW,EAAE,CAAC;QAChD,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YACpB,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACxD,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACvH,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC1C,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,IAAI,OAAO,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACrH,CAAC;aAAM,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YACtC,2EAA2E;YAC3E,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,SAAS;YACpC,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;YACxC,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YAC/C,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;gBACpB,IAAI,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,oBAAoB;oBAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACtF,CAAC;iBAAM,IAAI,MAAM,EAAE,CAAC;gBAClB,IAAI,cAAc,GAAG,oBAAoB,EAAE,CAAC;oBAC1C,cAAc,IAAI,CAAC,CAAC;oBACpB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,eAAe,CAAC,CAAC;oBAClD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBACzC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;gBACtF,CAAC;YACH,CAAC;iBAAM,IAAI,WAAW,GAAG,oBAAoB,IAAI,iBAAiB,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC;gBACpG,WAAW,IAAI,CAAC,CAAC;gBACjB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;aAAM,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC3E,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;gBAAE,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;YAClE,MAAM,QAAQ,GAAG,IAAI,KAAK,QAAQ,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,qBAAqB,CAAC;YAC9I,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,SAAS;gBAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC1E,IAAI,KAAK,KAAK,IAAI;gBAAE,MAAM;YAC1B,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC;QACjB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,QAAQ,CAAC,UAAsB,EAAE,IAAY;IAC3D,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAClF,CAAC;AAED,0JAA0J;AAC1J,MAAM,UAAU,WAAW,CAAC,IAAwB,EAAE,IAAY;IAChE,MAAM,OAAO,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3E,IAAI,OAAO,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACnC,OAAO,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACjF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,yGAAyG;AACzG,MAAM,UAAU,SAAS,CAAC,IAAgB;IACxC,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACrC,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC;IACvD,OAAO,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;AACrC,CAAC;AAED,qIAAqI;AACrI,MAAM,UAAU,WAAW,CAAC,MAAyB;IACnD,MAAM,KAAK,GAAmC,EAAE,CAAC;IACjD,MAAM,KAAK,GAAG,CAAC,KAAc,EAAE,KAAa,EAAQ,EAAE;QACpD,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ,IAAI,KAAK,GAAG,CAAC;YAAE,OAAO;QAClD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,KAAK,MAAM,IAAI,IAAI,KAAK;gBAAE,KAAK,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YACjD,OAAO;QACT,CAAC;QACD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO;QACxD,MAAM,IAAI,GAAG,KAAgC,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,IAAI,QAAQ,IAAI,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC;QAC/C,8EAA8E;QAC9E,MAAM,IAAI,GAAG,KAAK;aACf,IAAI,EAAE;aACN,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC;aACrC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC;YACH,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACP,yCAAyC;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,0GAA0G;AAC1G,MAAM,UAAU,MAAM,CAAC,IAA+B;IACpD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAClE,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACnC,OAAO,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACzC,CAAC;AAED,2HAA2H;AAC3H,SAAS,MAAM,CAAC,IAAY,EAAE,SAAkB;IAC9C,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACrH,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AACnC,CAAC;AAED,+GAA+G;AAC/G,MAAM,UAAU,WAAW,CAAC,KAA0B,EAAE,KAA6C;IACnG,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IACrC,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC9B,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;IACjC,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAc,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAChG,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,OAAO,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAQ,SAAuC,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAE,SAA8B,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACzK,MAAM,IAAI,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACtD,IAAI,IAAI,KAAK,IAAI;gBAAE,OAAO,IAAI,CAAC;QACjC,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,iHAAiH;AACjH,MAAM,UAAU,UAAU,CAAC,KAA0B,EAAE,KAA6C,EAAE,YAA2B;IAC/H,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC,CAAC;IAC1D,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,MAAM,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1F,IAAI,QAAQ,KAAK,IAAI;YAAE,OAAO,QAAQ,CAAC;IACzC,CAAC;IACD,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC;AAC9B,CAAC"}
|
|
1
|
+
{"version":3,"file":"meta.js","sourceRoot":"","sources":["../src/meta.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,6CAA6C,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,wEAAwE;AACxE,MAAM,CAAC,MAAM,QAAQ,GAAG,GAAG,CAAC;AAE5B,0GAA0G;AAC1G,MAAM,cAAc,GAAG,MAAM,CAAC;AAC9B,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,qFAAqF;AACrF,MAAM,SAAS,GAAG,EAAE,CAAC;AACrB,MAAM,QAAQ,GAAG,GAAG,CAAC;AACrB,gHAAgH;AAChH,MAAM,oBAAoB,GAAG,KAAK,CAAC;AACnC,oDAAoD;AACpD,MAAM,cAAc,GAAG,EAAE,CAAC;AAC1B,mDAAmD;AACnD,MAAM,MAAM,GAAwB,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;AAE9G,MAAM,SAAS,GAAG,mEAAmE,CAAC;AACtF,MAAM,QAAQ,GAAG,8BAA8B,CAAC;AAyBhD,0EAA0E;AAC1E,MAAM,UAAU,OAAO,CAAC,IAAY,EAAE,GAAW;IAC/C,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IACpC,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC5C,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM;QAAE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7D,OAAO,GAAG,CAAC;AACb,CAAC;AAED,8HAA8H;AAC9H,MAAM,UAAU,SAAS,CAAC,MAAc;IACtC,OAAO,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACnG,CAAC;AAED,wHAAwH;AACxH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,QAAQ,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACjF,CAAC;AAED,iGAAiG;AACjG,SAAS,YAAY,CAAC,MAAc;IAClC,MAAM,GAAG,GAAe,IAAI,GAAG,EAAE,CAAC;IAClC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,GAAI,KAAK,CAAC,CAAC,CAAY,CAAC,WAAW,EAAE,CAAC;QAChD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,SAAS,MAAM,CAAC,IAAY,EAAE,IAAY;IACxC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,KAAK,IAAI,EAAE,GAAG,IAAI,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACjC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAChB,IAAI,IAAI,KAAK,KAAK;gBAAE,KAAK,GAAG,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;YACpD,KAAK,GAAG,IAAI,CAAC;YACb,WAAW,GAAG,KAAK,CAAC;YACpB,SAAS;QACX,CAAC;QACD,IAAI,IAAI,KAAK,IAAI;YAAE,WAAW,GAAG,IAAI,CAAC;aACjC,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI;YAAE,WAAW,GAAG,KAAK,CAAC;IAClH,CAAC;IACD,OAAO,CAAC,CAAC,CAAC;AACZ,CAAC;AAED,4GAA4G;AAC5G,SAAS,UAAU,CAAC,IAAY,EAAE,IAAY,EAAE,IAAY;IAC1D,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,MAAM,IAAI,MAAM,EAAE,KAAK,CAAC,CAAC;IAC9D,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IACzB,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjC,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;AAC5F,CAAC;AAED,sIAAsI;AACtI,SAAS,iBAAiB,CAAC,IAAwB,EAAE,IAAY;IAC/D,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACpC,OAAO,GAAG,KAAK,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,KAAK,qBAAqB,CAAC;AAC9G,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,OAAe;IACtD,MAAM,IAAI,GAAe,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IAC1G,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACnC,IAAI,IAAI,KAAK,CAAC,CAAC;YAAE,MAAM;QACvB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;YAC5C,IAAI,KAAK,KAAK,CAAC,CAAC;gBAAE,MAAM;YACxB,EAAE,GAAG,KAAK,GAAG,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QACD,QAAQ,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC;YACd,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAChC,6EAA6E;QAC7E,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,MAAM;QACtB,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;QACb,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QACvC,MAAM,IAAI,GAAI,KAAK,CAAC,CAAC,CAAY,CAAC,WAAW,EAAE,CAAC;QAChD,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YACpB,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACxD,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACvH,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC1C,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,IAAI,OAAO,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACrH,CAAC;aAAM,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YACtC,2EAA2E;YAC3E,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,SAAS;YACpC,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;YACxC,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YAC/C,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;gBACpB,IAAI,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,oBAAoB;oBAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACpF,IAAI,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,cAAc;oBAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACtG,CAAC;iBAAM,IAAI,MAAM,EAAE,CAAC;gBAClB,IAAI,cAAc,GAAG,oBAAoB,EAAE,CAAC;oBAC1C,cAAc,IAAI,CAAC,CAAC;oBACpB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,eAAe,CAAC,CAAC;oBAClD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBACzC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;gBACtF,CAAC;YACH,CAAC;iBAAM,IAAI,WAAW,GAAG,oBAAoB,IAAI,iBAAiB,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC;gBACpG,WAAW,IAAI,CAAC,CAAC;gBACjB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;aAAM,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC3E,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;gBAAE,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;YAClE,MAAM,QAAQ,GAAG,IAAI,KAAK,QAAQ,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,qBAAqB,CAAC;YAC9I,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,SAAS;gBAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC1E,IAAI,KAAK,KAAK,IAAI;gBAAE,MAAM;YAC1B,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC;QACjB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,UAAsB,EAAE,IAAY;IAC3D,8FAA8F;IAC9F,OAAO,mBAAmB,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;SACpD,OAAO,CAAC,SAAS,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;SACpD,KAAK,CAAC,eAAe,CAAC;SACtB,QAAQ,CAAC,IAAI,CAAC,CAAC;AACpB,CAAC;AAED,0JAA0J;AAC1J,MAAM,UAAU,WAAW,CAAC,IAAwB,EAAE,IAAY;IAChE,MAAM,OAAO,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3E,IAAI,OAAO,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACnC,OAAO,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACjF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,yGAAyG;AACzG,MAAM,UAAU,SAAS,CAAC,IAAgB;IACxC,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACrC,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC;IACvD,OAAO,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;AACrC,CAAC;AAED,qIAAqI;AACrI,MAAM,UAAU,WAAW,CAAC,MAAyB;IACnD,MAAM,KAAK,GAAmC,EAAE,CAAC;IACjD,MAAM,KAAK,GAAG,CAAC,KAAc,EAAE,KAAa,EAAQ,EAAE;QACpD,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ,IAAI,KAAK,GAAG,CAAC;YAAE,OAAO;QAClD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,KAAK,MAAM,IAAI,IAAI,KAAK;gBAAE,KAAK,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YACjD,OAAO;QACT,CAAC;QACD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO;QACxD,MAAM,IAAI,GAAG,KAAgC,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,IAAI,QAAQ,IAAI,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC;QAC/C,8EAA8E;QAC9E,MAAM,IAAI,GAAG,KAAK;aACf,IAAI,EAAE;aACN,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC;aACrC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC;YACH,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACP,yCAAyC;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,0GAA0G;AAC1G,MAAM,UAAU,MAAM,CAAC,IAA+B;IACpD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAClE,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACnC,OAAO,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACzC,CAAC;AAED,2HAA2H;AAC3H,SAAS,MAAM,CAAC,IAAY,EAAE,SAAkB;IAC9C,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACrH,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AACnC,CAAC;AAED,+GAA+G;AAC/G,MAAM,UAAU,WAAW,CAAC,KAA0B,EAAE,KAA6C;IACnG,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IACrC,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC9B,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;IACjC,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAc,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAChG,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,OAAO,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAQ,SAAuC,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAE,SAA8B,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACzK,MAAM,IAAI,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACtD,IAAI,IAAI,KAAK,IAAI;gBAAE,OAAO,IAAI,CAAC;QACjC,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,iHAAiH;AACjH,MAAM,UAAU,UAAU,CAAC,KAA0B,EAAE,KAA6C,EAAE,YAA2B;IAC/H,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC,CAAC;IAC1D,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,MAAM,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1F,IAAI,QAAQ,KAAK,IAAI;YAAE,OAAO,QAAQ,CAAC;IACzC,CAAC;IACD,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC;AAC9B,CAAC"}
|
package/dist/page.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @file page.ts
|
|
3
3
|
* @description One page as a document and passages. A request goes out once per page, with a browser-like User-Agent
|
|
4
|
-
* from the web scraper pack, a deadline and a size cap, through Node's fetch
|
|
5
|
-
*
|
|
4
|
+
* from the web scraper pack, a deadline and a size cap, through Node's fetch or through AgentOS's checked egress (every
|
|
5
|
+
* address and every redirect checked against a policy), each following up to five redirects. Given rules, the reader checks the first
|
|
6
6
|
* address against the host's rule and the origin's robots.txt before it is requested, and the transport checks every
|
|
7
7
|
* redirect target the same way before its request. The answer is read by its content type: HTML and XML through
|
|
8
8
|
* public-sources' `htmlToBlocks` in page mode, other text by paragraphs, a PDF or another binary not at all; its
|
|
@@ -53,12 +53,19 @@ export interface TransportRules {
|
|
|
53
53
|
/** robots.txt as read before each request: a disallow refuses the address, a `PageRefusal` with `refused` and `robots`. */
|
|
54
54
|
robots?: RobotsRule;
|
|
55
55
|
}
|
|
56
|
+
/** What one request's caller will read of its answer. */
|
|
57
|
+
export interface TransportLimits {
|
|
58
|
+
/** The most bytes of the body the caller reads: a transport that holds the body before `read` refuses it past them. */
|
|
59
|
+
maxBytes?: number;
|
|
60
|
+
}
|
|
56
61
|
/**
|
|
57
62
|
* One GET of an address under a signal. Given rules, the transport applies them to the address and to every redirect
|
|
58
63
|
* target before its request, and throws {@link PageRefusal} for one they refuse; a transport of the host's own that
|
|
59
|
-
* follows redirects applies them too, or the reader checks the first address alone.
|
|
64
|
+
* follows redirects applies them too, or the reader checks the first address alone. Given limits, a transport that
|
|
65
|
+
* downloads the body before `read` (as the checked egress does) stops at them; one that streams it in `read` needs
|
|
66
|
+
* nothing more, since `read(cap)` stops there.
|
|
60
67
|
*/
|
|
61
|
-
export type Transport = (url: string, signal: AbortSignal, rules?: TransportRules) => Promise<TransportAnswer>;
|
|
68
|
+
export type Transport = (url: string, signal: AbortSignal, rules?: TransportRules, limits?: TransportLimits) => Promise<TransportAnswer>;
|
|
62
69
|
/** A failure a transport names itself, such as the egress policy's refusal. */
|
|
63
70
|
export declare class PageRefusal extends Error {
|
|
64
71
|
readonly failure: PageFailure;
|
|
@@ -76,9 +83,24 @@ export declare function pageKey(url: string): string;
|
|
|
76
83
|
/** An address's origin for a log line, which leaves out the path and the query where secrets can sit. */
|
|
77
84
|
export declare function originOf(url: string): string;
|
|
78
85
|
/**
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
|
|
86
|
+
* A promise that rejects with the signal's reason once it fires, after `onAbort` runs; `release` stops listening. It
|
|
87
|
+
* never counts as an unhandled rejection.
|
|
88
|
+
*/
|
|
89
|
+
export declare function whenAborted(signal: AbortSignal, onAbort?: () => void): {
|
|
90
|
+
promise: Promise<never>;
|
|
91
|
+
release: () => void;
|
|
92
|
+
};
|
|
93
|
+
/**
|
|
94
|
+
* A body read up to `cap` bytes: its bytes, or null past the cap with the rest of the stream cancelled. Given a signal,
|
|
95
|
+
* its abort cancels the stream and throws the signal's reason at once, also for a stream that answers neither.
|
|
96
|
+
*/
|
|
97
|
+
export declare function readCapped(response: Response, cap: number, signal?: AbortSignal): Promise<Uint8Array | null>;
|
|
98
|
+
/**
|
|
99
|
+
* Requests through a `fetch`, the body streamed up to the cap, up to five redirects followed one at a time and a sixth
|
|
100
|
+
* the failure `redirects`; given rules, each target is checked before its request. A target that is not http or https
|
|
101
|
+
* is refused (`scheme`). Node's fetch hands back a redirect asked for with `redirect: 'manual'` as it came
|
|
102
|
+
* (https://github.com/nodejs/undici/issues/1193); with no rules, a `fetch` that hides it, as a browser's does, is asked
|
|
103
|
+
* again to follow its redirects itself.
|
|
82
104
|
*/
|
|
83
105
|
export declare function fetchTransport(fetchFn: typeof globalThis.fetch, userAgent?: string): Transport;
|
|
84
106
|
/**
|
|
@@ -93,6 +115,8 @@ export declare function egressTransport(profile: EgressProfile, bounds: {
|
|
|
93
115
|
timeoutMs: number;
|
|
94
116
|
maxBytes: number;
|
|
95
117
|
}, userAgent?: string): Promise<Transport>;
|
|
118
|
+
/** A page's text as blocks, the characters a reader never sees removed. */
|
|
119
|
+
export declare function pageBlocks(kind: 'html' | 'text', text: string, budget: number): string[];
|
|
96
120
|
/** Reads a page the reader's own fetch found no text in, such as one a script draws. Answers the text as blocks, or null. */
|
|
97
121
|
export interface PageFallbackReader {
|
|
98
122
|
read(url: string, options: {
|
|
@@ -135,6 +159,18 @@ export interface PageReaderOptions {
|
|
|
135
159
|
* these rules: the reader asks it only for an address they cleared.
|
|
136
160
|
*/
|
|
137
161
|
fallback?: PageFallbackReader;
|
|
162
|
+
/**
|
|
163
|
+
* Whether to read the page's icon once its text is read: from the page's own origin alone, after its redirects (its
|
|
164
|
+
* first `<link rel="icon">` at that origin that is not an SVG, by its `type` or its address, else that origin's
|
|
165
|
+
* `/favicon.ico`, never a declared icon at another host), through the same transport, under `allowUrl` and `robots`,
|
|
166
|
+
* under a deadline of its own (`timeoutMs`, as each request has, so it can add up to `timeoutMs` to the read), at
|
|
167
|
+
* most `iconMaxBytes` read. A PNG, ICO, JPEG or WebP
|
|
168
|
+
* whose data address holds at most 8,192 characters is kept on the document and on each passage as `icon`; any other
|
|
169
|
+
* answer, a refusal or a failure leaves the icon out, and the page is read as it is without one. Not read when left out.
|
|
170
|
+
*/
|
|
171
|
+
icon?: boolean;
|
|
172
|
+
/** The most bytes of an icon read; 65,536 when left out. */
|
|
173
|
+
iconMaxBytes?: number;
|
|
138
174
|
}
|
|
139
175
|
/** A page as the check reads it: its status, and when it was read, its document, its blocks and its sentences as passages. */
|
|
140
176
|
export interface PageRead {
|
|
@@ -157,7 +193,8 @@ export interface PageRead {
|
|
|
157
193
|
* Reads one page: the rules checked when given, one request, then for a 200 with no text one read of its address after
|
|
158
194
|
* redirects through the fallback reader when given, and, when no rules are given, for a failure the scraper may mend (a
|
|
159
195
|
* deadline, no connection, an error status other than 404 and 410, a page with no text) one try through the fallback
|
|
160
|
-
* scraper. Answers a status for a page that is not read; throws only the
|
|
196
|
+
* scraper; then, with `icon`, one request for its icon. Answers a status for a page that is not read; throws only the
|
|
197
|
+
* caller's own cancellation.
|
|
161
198
|
*
|
|
162
199
|
* @param url - The address to fetch, from {@link pageKey}.
|
|
163
200
|
* @param options - How to read it.
|
package/dist/page.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"page.d.ts","sourceRoot":"","sources":["../src/page.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;
|
|
1
|
+
{"version":3,"file":"page.d.ts","sourceRoot":"","sources":["../src/page.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,EAAiC,KAAK,cAAc,EAAE,KAAK,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,qCAAqC,CAAC;AAClJ,OAAO,EAAkB,KAAK,WAAW,EAAE,KAAK,YAAY,EAAE,MAAM,kCAAkC,CAAC;AAKvG,OAAO,EAA4B,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AACxE,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1E;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CACnD;AAED,8HAA8H;AAC9H,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE/C,yCAAyC;AACzC,MAAM,WAAW,eAAe;IAC9B,gCAAgC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,0CAA0C;IAC1C,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACpC,wEAAwE;IACxE,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAC9C,oHAAoH;IACpH,UAAU,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAC9C,6CAA6C;IAC7C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,mGAAmG;IACnG,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IACpC,2HAA2H;IAC3H,MAAM,CAAC,EAAE,UAAU,CAAC;CACrB;AAED,yDAAyD;AACzD,MAAM,WAAW,eAAe;IAC9B,uHAAuH;IACvH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,SAAS,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,CAAC,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,eAAe,KAAK,OAAO,CAAC,eAAe,CAAC,CAAC;AAEzI,+EAA+E;AAC/E,qBAAa,WAAY,SAAQ,KAAK;IAMlC,QAAQ,CAAC,OAAO,EAAE,WAAW;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IANhC;;;OAGG;gBAEQ,OAAO,EAAE,WAAW,EACpB,MAAM,GAAE,MAAM,GAAG,IAAW;CAKxC;AAQD,+FAA+F;AAC/F,eAAO,MAAM,cAAc,EAAE,QAAQ,CAAC,aAAa,CAAgD,CAAC;AAEpG,8GAA8G;AAC9G,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAI3C;AAED,yGAAyG;AACzG,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAM5C;AAyCD;;;GAGG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,GAAE,MAAM,IAAsB,GAAG;IAAE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAAC,OAAO,EAAE,MAAM,IAAI,CAAA;CAAE,CAYxI;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAyBlH;AA4CD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,UAAU,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAwB9F;AAwCD;;;;;;;GAOG;AACH,wBAAsB,eAAe,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAkErJ;AAqFD,2EAA2E;AAC3E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAGxF;AAED,6HAA6H;AAC7H,MAAM,WAAW,kBAAkB;IACjC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE;QAAE,MAAM,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC;CAC/E;AAED,0BAA0B;AAC1B,MAAM,WAAW,iBAAiB;IAChC,6CAA6C;IAC7C,SAAS,EAAE,SAAS,CAAC;IACrB,+GAA+G;IAC/G,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;IAC5B,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,+CAA+C;IAC/C,GAAG,EAAE,MAAM,MAAM,CAAC;IAClB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB;;;OAGG;IACH,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IACpC;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAC9B;;;;;;;;OAQG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,4DAA4D;IAC5D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,8HAA8H;AAC9H,MAAM,WAAW,QAAQ;IACvB,0BAA0B;IAC1B,MAAM,EAAE,UAAU,CAAC;IACnB,kEAAkE;IAClE,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,0GAA0G;IAC1G,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B;;;;OAIG;IACH,QAAQ,EAAE,cAAc,GAAG,IAAI,CAAC;IAChC,yGAAyG;IACzG,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AA8SD;;;;;;;;;GASG;AACH,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAQzF"}
|