@framers/agentos-ext-citation-check 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -2
- package/dist/index.d.ts +7 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -2
- package/dist/index.js.map +1 -1
- package/dist/licence.d.ts +31 -0
- package/dist/licence.d.ts.map +1 -0
- package/dist/licence.js +122 -0
- package/dist/licence.js.map +1 -0
- package/dist/meta.d.ts +66 -0
- package/dist/meta.d.ts.map +1 -0
- package/dist/meta.js +283 -0
- package/dist/meta.js.map +1 -0
- package/dist/page.d.ts +78 -17
- package/dist/page.d.ts.map +1 -1
- package/dist/page.js +235 -37
- package/dist/page.js.map +1 -1
- package/dist/robots.d.ts +90 -0
- package/dist/robots.d.ts.map +1 -0
- package/dist/robots.js +249 -0
- package/dist/robots.js.map +1 -0
- package/dist/types.d.ts +9 -6
- package/dist/types.d.ts.map +1 -1
- package/dist/web-source.d.ts +95 -0
- package/dist/web-source.d.ts.map +1 -0
- package/dist/web-source.js +176 -0
- package/dist/web-source.js.map +1 -0
- package/manifest.json +4 -3
- package/package.json +10 -8
package/dist/meta.js
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file meta.ts
|
|
3
|
+
* @description What a page says about itself in its markup as served: its `<title>`, its `<meta>` tags, the links and
|
|
4
|
+
* anchors a licence is read from, and its JSON-LD. One pass over the markup, in document order, finds them: a comment
|
|
5
|
+
* is passed over whole (so a tag or a JSON-LD block inside one is not the page's), the content of a script, a style, a
|
|
6
|
+
* template, a `noscript`, a `textarea` and the title is not read for tags (so a comment opener or a tag written inside
|
|
7
|
+
* one is theirs), and a tag runs to the first `>` outside a quoted attribute value. Each step of the pass starts where
|
|
8
|
+
* the one before it ended, so the time grows with the page alone, however its tags are left open. Of the links and
|
|
9
|
+
* anchors it keeps those a licence is read from, at most 1,000 of each kind, and reads the text of a `rel="license"`
|
|
10
|
+
* anchor alone, so what it holds does not grow with the page. A value a script writes later is not seen. Character
|
|
11
|
+
* references are read with the `entities` package's HTML decoder.
|
|
12
|
+
*
|
|
13
|
+
* @module agentos/extensions/research/citation-check/meta
|
|
14
|
+
*/
|
|
15
|
+
import { collapse } from '@framers/agentos-ext-grounding-guard/values';
|
|
16
|
+
import { decodeHTML, decodeHTMLAttribute } from 'entities';
|
|
17
|
+
import { withoutInvisible } from './match.js';
|
|
18
|
+
/** The most characters a page's title or its publisher's name keeps. */
|
|
19
|
+
export const NAME_MAX = 255;
|
|
20
|
+
/** The most characters of a tag read for its attributes, and of an anchor's content read for its text. */
|
|
21
|
+
const TAG_SOURCE_MAX = 16_384;
|
|
22
|
+
const ANCHOR_TEXT_MAX = 2_000;
|
|
23
|
+
/** The most JSON-LD blocks read from one page, and the most nodes walked in them. */
|
|
24
|
+
const LD_BLOCKS = 20;
|
|
25
|
+
const LD_NODES = 500;
|
|
26
|
+
/** The most `<link rel="license">`s one page keeps, and the most `<a>`s of each kind a licence is read from. */
|
|
27
|
+
const LICENCE_ELEMENTS_MAX = 1_000;
|
|
28
|
+
/** Elements whose content is not read for tags. */
|
|
29
|
+
const OPAQUE = new Set(['script', 'style', 'template', 'noscript', 'textarea', 'title']);
|
|
30
|
+
const ATTRIBUTE = /([^\s=/>"']+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/gu;
|
|
31
|
+
const TAG_NAME = /\/?([A-Za-z][A-Za-z0-9-]*)/uy;
|
|
32
|
+
/** At most `max` characters of a text, never half of a surrogate pair. */
|
|
33
|
+
export function cutText(text, max) {
|
|
34
|
+
if (text.length <= max)
|
|
35
|
+
return text;
|
|
36
|
+
let cut = text.slice(0, max);
|
|
37
|
+
const last = cut.charCodeAt(cut.length - 1);
|
|
38
|
+
if (last >= 0xd800 && last <= 0xdbff)
|
|
39
|
+
cut = cut.slice(0, -1);
|
|
40
|
+
return cut;
|
|
41
|
+
}
|
|
42
|
+
/** Markup as a reader of it sees it: tags removed, references read, NFC, invisible characters gone, white space collapsed. */
|
|
43
|
+
export function cleanText(markup) {
|
|
44
|
+
return collapse(withoutInvisible(decodeHTML(markup.replace(/<[^>]*>/gu, ' ')).normalize('NFC')));
|
|
45
|
+
}
|
|
46
|
+
/** An attribute's value as a reader sees it: references read, NFC, invisible characters gone, white space collapsed. */
|
|
47
|
+
export function cleanAttribute(value) {
|
|
48
|
+
return collapse(withoutInvisible(decodeHTMLAttribute(value).normalize('NFC')));
|
|
49
|
+
}
|
|
50
|
+
/** A tag's attributes by name in lower case, the first of a name kept, the values as written. */
|
|
51
|
+
function attributesOf(source) {
|
|
52
|
+
const out = new Map();
|
|
53
|
+
for (const match of source.slice(0, TAG_SOURCE_MAX).matchAll(ATTRIBUTE)) {
|
|
54
|
+
const name = match[1].toLowerCase();
|
|
55
|
+
if (!out.has(name))
|
|
56
|
+
out.set(name, match[2] ?? match[3] ?? match[4] ?? '');
|
|
57
|
+
}
|
|
58
|
+
return out;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Where a tag whose attributes start at `from` ends: the first `>` outside a quoted value, a quote opening a value only
|
|
62
|
+
* after an `=`. -1 when the markup ends first.
|
|
63
|
+
*/
|
|
64
|
+
function tagEnd(html, from) {
|
|
65
|
+
let quote = 0;
|
|
66
|
+
let afterEquals = false;
|
|
67
|
+
for (let at = from; at < html.length; at += 1) {
|
|
68
|
+
const code = html.charCodeAt(at);
|
|
69
|
+
if (quote !== 0) {
|
|
70
|
+
if (code === quote)
|
|
71
|
+
quote = 0;
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
if (code === 0x3e)
|
|
75
|
+
return at;
|
|
76
|
+
if ((code === 0x22 || code === 0x27) && afterEquals) {
|
|
77
|
+
quote = code;
|
|
78
|
+
afterEquals = false;
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
if (code === 0x3d)
|
|
82
|
+
afterEquals = true;
|
|
83
|
+
else if (code !== 0x20 && code !== 0x09 && code !== 0x0a && code !== 0x0c && code !== 0x0d)
|
|
84
|
+
afterEquals = false;
|
|
85
|
+
}
|
|
86
|
+
return -1;
|
|
87
|
+
}
|
|
88
|
+
/** The element's closing tag from `from` on, as its start and its end, or null when the markup has none. */
|
|
89
|
+
function closingTag(html, name, from) {
|
|
90
|
+
const pattern = new RegExp(String.raw `<\/${name}\s*>`, 'giu');
|
|
91
|
+
pattern.lastIndex = from;
|
|
92
|
+
const match = pattern.exec(html);
|
|
93
|
+
return match === null ? null : { start: match.index, end: match.index + match[0].length };
|
|
94
|
+
}
|
|
95
|
+
/** Whether an address as an attribute writes it, resolved against the page's own, is an http or https page on creativecommons.org. */
|
|
96
|
+
function onCreativeCommons(href, base) {
|
|
97
|
+
const url = resolveHttp(href, base);
|
|
98
|
+
return url !== null && new URL(url).hostname.toLowerCase().replace(/^www\./u, '') === 'creativecommons.org';
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* One pass over a page's markup; see the module's description.
|
|
102
|
+
*
|
|
103
|
+
* @param html - The page's markup as served.
|
|
104
|
+
* @param baseUrl - The page's own address, which an anchor's address is resolved against.
|
|
105
|
+
*/
|
|
106
|
+
export function scanMarkup(html, baseUrl) {
|
|
107
|
+
const scan = { title: null, metas: new Map(), links: [], anchors: [], jsonLd: [] };
|
|
108
|
+
let licenceAnchors = 0;
|
|
109
|
+
let deedAnchors = 0;
|
|
110
|
+
let at = 0;
|
|
111
|
+
while (at < html.length) {
|
|
112
|
+
const open = html.indexOf('<', at);
|
|
113
|
+
if (open === -1)
|
|
114
|
+
break;
|
|
115
|
+
if (html.startsWith('<!--', open)) {
|
|
116
|
+
const close = html.indexOf('-->', open + 4);
|
|
117
|
+
if (close === -1)
|
|
118
|
+
break;
|
|
119
|
+
at = close + 3;
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
TAG_NAME.lastIndex = open + 1;
|
|
123
|
+
const named = TAG_NAME.exec(html);
|
|
124
|
+
if (named === null) {
|
|
125
|
+
at = open + 1;
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
const start = TAG_NAME.lastIndex;
|
|
129
|
+
const end = tagEnd(html, start);
|
|
130
|
+
// A tag the markup never closes holds the rest of it, as a browser reads it.
|
|
131
|
+
if (end === -1)
|
|
132
|
+
break;
|
|
133
|
+
at = end + 1;
|
|
134
|
+
if (named[0].startsWith('/'))
|
|
135
|
+
continue;
|
|
136
|
+
const name = named[1].toLowerCase();
|
|
137
|
+
if (name === 'meta') {
|
|
138
|
+
const attributes = attributesOf(html.slice(start, end));
|
|
139
|
+
const key = (attributes.get('property') ?? attributes.get('name') ?? attributes.get('itemprop'))?.trim().toLowerCase();
|
|
140
|
+
const content = attributes.get('content');
|
|
141
|
+
if (key !== undefined && key !== '' && content !== undefined && !scan.metas.has(key))
|
|
142
|
+
scan.metas.set(key, content);
|
|
143
|
+
}
|
|
144
|
+
else if (name === 'link' || name === 'a') {
|
|
145
|
+
const source = html.slice(start, end);
|
|
146
|
+
// A tag with no `=` gives no `rel` and no address, so it names no licence.
|
|
147
|
+
if (!source.includes('='))
|
|
148
|
+
continue;
|
|
149
|
+
const attributes = attributesOf(source);
|
|
150
|
+
const marked = relNames(attributes, 'license');
|
|
151
|
+
if (name === 'link') {
|
|
152
|
+
if (marked && scan.links.length < LICENCE_ELEMENTS_MAX)
|
|
153
|
+
scan.links.push(attributes);
|
|
154
|
+
}
|
|
155
|
+
else if (marked) {
|
|
156
|
+
if (licenceAnchors < LICENCE_ELEMENTS_MAX) {
|
|
157
|
+
licenceAnchors += 1;
|
|
158
|
+
const tail = html.slice(at, at + ANCHOR_TEXT_MAX);
|
|
159
|
+
const close = tail.search(/<\/a[\s>]/iu);
|
|
160
|
+
scan.anchors.push({ attributes, text: close === -1 ? tail : tail.slice(0, close) });
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
else if (deedAnchors < LICENCE_ELEMENTS_MAX && onCreativeCommons(attributes.get('href'), baseUrl)) {
|
|
164
|
+
deedAnchors += 1;
|
|
165
|
+
scan.anchors.push({ attributes, text: '' });
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
else if (OPAQUE.has(name)) {
|
|
169
|
+
const close = closingTag(html, name, at);
|
|
170
|
+
const content = html.slice(at, close === null ? html.length : close.start);
|
|
171
|
+
if (name === 'title' && scan.title === null)
|
|
172
|
+
scan.title = content;
|
|
173
|
+
const isJsonLd = name === 'script' && (attributesOf(html.slice(start, end)).get('type') ?? '').trim().toLowerCase() === 'application/ld+json';
|
|
174
|
+
if (isJsonLd && scan.jsonLd.length < LD_BLOCKS)
|
|
175
|
+
scan.jsonLd.push(content);
|
|
176
|
+
if (close === null)
|
|
177
|
+
break;
|
|
178
|
+
at = close.end;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
return scan;
|
|
182
|
+
}
|
|
183
|
+
/** Whether an element's `rel` names a link type, its tokens read without case. */
|
|
184
|
+
export function relNames(attributes, type) {
|
|
185
|
+
return (attributes.get('rel') ?? '').toLowerCase().split(/\s+/u).includes(type);
|
|
186
|
+
}
|
|
187
|
+
/** An address as an attribute writes it, its references read and resolved against the page's own; null when it does not parse or is not http or https. */
|
|
188
|
+
export function resolveHttp(href, base) {
|
|
189
|
+
const written = href === undefined ? '' : decodeHTMLAttribute(href).trim();
|
|
190
|
+
if (written === '')
|
|
191
|
+
return null;
|
|
192
|
+
try {
|
|
193
|
+
const url = new URL(written, base);
|
|
194
|
+
return url.protocol === 'http:' || url.protocol === 'https:' ? url.href : null;
|
|
195
|
+
}
|
|
196
|
+
catch {
|
|
197
|
+
return null;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
/** The page's `<title>`, its text read and cut to {@link NAME_MAX} characters; null when it has none. */
|
|
201
|
+
export function pageTitle(scan) {
|
|
202
|
+
if (scan.title === null)
|
|
203
|
+
return null;
|
|
204
|
+
const title = cutText(cleanText(scan.title), NAME_MAX);
|
|
205
|
+
return title === '' ? null : title;
|
|
206
|
+
}
|
|
207
|
+
/** The JSON-LD objects of the page's blocks, in document order: each block's top object or array, and the nodes of each `@graph`. */
|
|
208
|
+
export function jsonLdNodes(blocks) {
|
|
209
|
+
const nodes = [];
|
|
210
|
+
const visit = (value, depth) => {
|
|
211
|
+
if (nodes.length >= LD_NODES || depth > 6)
|
|
212
|
+
return;
|
|
213
|
+
if (Array.isArray(value)) {
|
|
214
|
+
for (const item of value)
|
|
215
|
+
visit(item, depth + 1);
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
if (value === null || typeof value !== 'object')
|
|
219
|
+
return;
|
|
220
|
+
const node = value;
|
|
221
|
+
nodes.push(node);
|
|
222
|
+
if ('@graph' in node)
|
|
223
|
+
visit(node['@graph'], depth + 1);
|
|
224
|
+
};
|
|
225
|
+
for (const block of blocks.slice(0, LD_BLOCKS)) {
|
|
226
|
+
// A script may wrap its JSON in a comment or a CDATA section for old parsers.
|
|
227
|
+
const body = block
|
|
228
|
+
.trim()
|
|
229
|
+
.replace(/^(?:<!--|<!\[CDATA\[)/u, '')
|
|
230
|
+
.replace(/(?:-->|\]\]>)$/u, '');
|
|
231
|
+
try {
|
|
232
|
+
visit(JSON.parse(body), 0);
|
|
233
|
+
}
|
|
234
|
+
catch {
|
|
235
|
+
// A block that is not JSON says nothing.
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
return nodes;
|
|
239
|
+
}
|
|
240
|
+
/** A date as milliseconds since 1970, or null for one that does not parse or names no four-digit year. */
|
|
241
|
+
export function dateOf(text) {
|
|
242
|
+
if (typeof text !== 'string' || !/\d{4}/u.test(text))
|
|
243
|
+
return null;
|
|
244
|
+
const at = Date.parse(text.trim());
|
|
245
|
+
return Number.isFinite(at) ? at : null;
|
|
246
|
+
}
|
|
247
|
+
/** A name as a page states it, cut to {@link NAME_MAX} characters; null when empty. An attribute's references are read. */
|
|
248
|
+
function nameOf(text, attribute) {
|
|
249
|
+
const name = cutText(attribute ? cleanAttribute(text) : collapse(withoutInvisible(text.normalize('NFC'))), NAME_MAX);
|
|
250
|
+
return name === '' ? null : name;
|
|
251
|
+
}
|
|
252
|
+
/** The site's name: `og:site_name`, else the first JSON-LD `publisher.name`; null when the page names none. */
|
|
253
|
+
export function publisherOf(metas, nodes) {
|
|
254
|
+
const og = metas.get('og:site_name');
|
|
255
|
+
if (og !== undefined) {
|
|
256
|
+
const name = nameOf(og, true);
|
|
257
|
+
if (name !== null)
|
|
258
|
+
return name;
|
|
259
|
+
}
|
|
260
|
+
for (const node of nodes) {
|
|
261
|
+
const publishers = Array.isArray(node.publisher) ? node.publisher : [node.publisher];
|
|
262
|
+
for (const publisher of publishers) {
|
|
263
|
+
const raw = typeof publisher === 'string' ? publisher : typeof publisher?.name === 'string' ? publisher.name : null;
|
|
264
|
+
const name = raw === null ? null : nameOf(raw, false);
|
|
265
|
+
if (name !== null)
|
|
266
|
+
return name;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
return null;
|
|
270
|
+
}
|
|
271
|
+
/** The page's own date: `article:modified_time`, else the first JSON-LD `dateModified`, else `Last-Modified`. */
|
|
272
|
+
export function pageDateOf(metas, nodes, lastModified) {
|
|
273
|
+
const stated = dateOf(metas.get('article:modified_time'));
|
|
274
|
+
if (stated !== null)
|
|
275
|
+
return stated;
|
|
276
|
+
for (const node of nodes) {
|
|
277
|
+
const modified = dateOf(typeof node.dateModified === 'string' ? node.dateModified : null);
|
|
278
|
+
if (modified !== null)
|
|
279
|
+
return modified;
|
|
280
|
+
}
|
|
281
|
+
return dateOf(lastModified);
|
|
282
|
+
}
|
|
283
|
+
//# sourceMappingURL=meta.js.map
|
package/dist/meta.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"meta.js","sourceRoot":"","sources":["../src/meta.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,6CAA6C,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,wEAAwE;AACxE,MAAM,CAAC,MAAM,QAAQ,GAAG,GAAG,CAAC;AAE5B,0GAA0G;AAC1G,MAAM,cAAc,GAAG,MAAM,CAAC;AAC9B,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,qFAAqF;AACrF,MAAM,SAAS,GAAG,EAAE,CAAC;AACrB,MAAM,QAAQ,GAAG,GAAG,CAAC;AACrB,gHAAgH;AAChH,MAAM,oBAAoB,GAAG,KAAK,CAAC;AACnC,mDAAmD;AACnD,MAAM,MAAM,GAAwB,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;AAE9G,MAAM,SAAS,GAAG,mEAAmE,CAAC;AACtF,MAAM,QAAQ,GAAG,8BAA8B,CAAC;AAuBhD,0EAA0E;AAC1E,MAAM,UAAU,OAAO,CAAC,IAAY,EAAE,GAAW;IAC/C,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IACpC,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC5C,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM;QAAE,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7D,OAAO,GAAG,CAAC;AACb,CAAC;AAED,8HAA8H;AAC9H,MAAM,UAAU,SAAS,CAAC,MAAc;IACtC,OAAO,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACnG,CAAC;AAED,wHAAwH;AACxH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,QAAQ,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACjF,CAAC;AAED,iGAAiG;AACjG,SAAS,YAAY,CAAC,MAAc;IAClC,MAAM,GAAG,GAAe,IAAI,GAAG,EAAE,CAAC;IAClC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,GAAI,KAAK,CAAC,CAAC,CAAY,CAAC,WAAW,EAAE,CAAC;QAChD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,SAAS,MAAM,CAAC,IAAY,EAAE,IAAY;IACxC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,KAAK,IAAI,EAAE,GAAG,IAAI,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACjC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAChB,IAAI,IAAI,KAAK,KAAK;gBAAE,KAAK,GAAG,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;YACpD,KAAK,GAAG,IAAI,CAAC;YACb,WAAW,GAAG,KAAK,CAAC;YACpB,SAAS;QACX,CAAC;QACD,IAAI,IAAI,KAAK,IAAI;YAAE,WAAW,GAAG,IAAI,CAAC;aACjC,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI;YAAE,WAAW,GAAG,KAAK,CAAC;IAClH,CAAC;IACD,OAAO,CAAC,CAAC,CAAC;AACZ,CAAC;AAED,4GAA4G;AAC5G,SAAS,UAAU,CAAC,IAAY,EAAE,IAAY,EAAE,IAAY;IAC1D,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,MAAM,IAAI,MAAM,EAAE,KAAK,CAAC,CAAC;IAC9D,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IACzB,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjC,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;AAC5F,CAAC;AAED,sIAAsI;AACtI,SAAS,iBAAiB,CAAC,IAAwB,EAAE,IAAY;IAC/D,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACpC,OAAO,GAAG,KAAK,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,KAAK,qBAAqB,CAAC;AAC9G,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,OAAe;IACtD,MAAM,IAAI,GAAe,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IAC/F,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACnC,IAAI,IAAI,KAAK,CAAC,CAAC;YAAE,MAAM;QACvB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;YAC5C,IAAI,KAAK,KAAK,CAAC,CAAC;gBAAE,MAAM;YACxB,EAAE,GAAG,KAAK,GAAG,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QACD,QAAQ,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC;YACd,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAChC,6EAA6E;QAC7E,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,MAAM;QACtB,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;QACb,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QACvC,MAAM,IAAI,GAAI,KAAK,CAAC,CAAC,CAAY,CAAC,WAAW,EAAE,CAAC;QAChD,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YACpB,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACxD,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACvH,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC1C,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,IAAI,OAAO,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACrH,CAAC;aAAM,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YACtC,2EAA2E;YAC3E,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,SAAS;YACpC,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;YACxC,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YAC/C,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;gBACpB,IAAI,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,oBAAoB;oBAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACtF,CAAC;iBAAM,IAAI,MAAM,EAAE,CAAC;gBAClB,IAAI,cAAc,GAAG,oBAAoB,EAAE,CAAC;oBAC1C,cAAc,IAAI,CAAC,CAAC;oBACpB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,eAAe,CAAC,CAAC;oBAClD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;oBACzC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;gBACtF,CAAC;YACH,CAAC;iBAAM,IAAI,WAAW,GAAG,oBAAoB,IAAI,iBAAiB,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC;gBACpG,WAAW,IAAI,CAAC,CAAC;gBACjB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;aAAM,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC3E,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;gBAAE,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;YAClE,MAAM,QAAQ,GAAG,IAAI,KAAK,QAAQ,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,qBAAqB,CAAC;YAC9I,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,SAAS;gBAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC1E,IAAI,KAAK,KAAK,IAAI;gBAAE,MAAM;YAC1B,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC;QACjB,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,QAAQ,CAAC,UAAsB,EAAE,IAAY;IAC3D,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAClF,CAAC;AAED,0JAA0J;AAC1J,MAAM,UAAU,WAAW,CAAC,IAAwB,EAAE,IAAY;IAChE,MAAM,OAAO,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3E,IAAI,OAAO,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACnC,OAAO,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACjF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,yGAAyG;AACzG,MAAM,UAAU,SAAS,CAAC,IAAgB;IACxC,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACrC,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC;IACvD,OAAO,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;AACrC,CAAC;AAED,qIAAqI;AACrI,MAAM,UAAU,WAAW,CAAC,MAAyB;IACnD,MAAM,KAAK,GAAmC,EAAE,CAAC;IACjD,MAAM,KAAK,GAAG,CAAC,KAAc,EAAE,KAAa,EAAQ,EAAE;QACpD,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ,IAAI,KAAK,GAAG,CAAC;YAAE,OAAO;QAClD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,KAAK,MAAM,IAAI,IAAI,KAAK;gBAAE,KAAK,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YACjD,OAAO;QACT,CAAC;QACD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO;QACxD,MAAM,IAAI,GAAG,KAAgC,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,IAAI,QAAQ,IAAI,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC;QAC/C,8EAA8E;QAC9E,MAAM,IAAI,GAAG,KAAK;aACf,IAAI,EAAE;aACN,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC;aACrC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC;YACH,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACP,yCAAyC;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,0GAA0G;AAC1G,MAAM,UAAU,MAAM,CAAC,IAA+B;IACpD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAClE,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACnC,OAAO,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACzC,CAAC;AAED,2HAA2H;AAC3H,SAAS,MAAM,CAAC,IAAY,EAAE,SAAkB;IAC9C,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACrH,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AACnC,CAAC;AAED,+GAA+G;AAC/G,MAAM,UAAU,WAAW,CAAC,KAA0B,EAAE,KAA6C;IACnG,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IACrC,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC9B,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;IACjC,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAc,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAChG,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,OAAO,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAQ,SAAuC,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAE,SAA8B,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACzK,MAAM,IAAI,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACtD,IAAI,IAAI,KAAK,IAAI;gBAAE,OAAO,IAAI,CAAC;QACjC,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,iHAAiH;AACjH,MAAM,UAAU,UAAU,CAAC,KAA0B,EAAE,KAA6C,EAAE,YAA2B;IAC/H,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC,CAAC;IAC1D,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,MAAM,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1F,IAAI,QAAQ,KAAK,IAAI;YAAE,OAAO,QAAQ,CAAC;IACzC,CAAC;IACD,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC;AAC9B,CAAC"}
|
package/dist/page.d.ts
CHANGED
|
@@ -1,18 +1,23 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @file page.ts
|
|
3
|
-
* @description One
|
|
4
|
-
* web scraper pack, a deadline and a size cap, through Node's fetch (redirects followed) or through AgentOS's
|
|
5
|
-
* egress (every address and every redirect checked against a policy)
|
|
6
|
-
*
|
|
3
|
+
* @description One page as a document and passages. A request goes out once per page, with a browser-like User-Agent
|
|
4
|
+
* from the web scraper pack, a deadline and a size cap, through Node's fetch (redirects followed) or through AgentOS's
|
|
5
|
+
* checked egress (every address and every redirect checked against a policy). Given rules, the reader checks the first
|
|
6
|
+
* address against the host's rule and the origin's robots.txt before it is requested, and the transport checks every
|
|
7
|
+
* redirect target the same way before its request. The answer is read by its content type: HTML and XML through
|
|
7
8
|
* public-sources' `htmlToBlocks` in page mode, other text by paragraphs, a PDF or another binary not at all; its
|
|
8
|
-
* charset from a byte order mark, the content type or a `<meta>` tag.
|
|
9
|
-
*
|
|
10
|
-
*
|
|
9
|
+
* charset from a byte order mark, the content type or a `<meta>` tag. The document carries the page's title, the
|
|
10
|
+
* licence it states, the publisher's name and the page's own date where the markup gives them. Its sentences become
|
|
11
|
+
* passages through public-sources' `citationSources`, cut by grounding-guard's `sentenceSpans`. A page that answered
|
|
12
|
+
* 200 with no text is read once through the host's fallback reader when one is given; a page that fails in a way a
|
|
13
|
+
* browser may mend is retried once through the web scraper pack's tiers. A page that fails is a status, never a throw;
|
|
14
|
+
* only the caller's own cancellation throws.
|
|
11
15
|
*
|
|
12
16
|
* @module agentos/extensions/research/citation-check/page
|
|
13
17
|
*/
|
|
14
|
-
import { type CitationSource } from '@framers/agentos-ext-public-sources';
|
|
18
|
+
import { type CitationSource, type SourceDocument, type SourceLicence } from '@framers/agentos-ext-public-sources';
|
|
15
19
|
import { type ScrapeInput, type ScrapeResult } from '@framers/agentos-ext-web-scraper';
|
|
20
|
+
import { type RobotsRule } from './robots.js';
|
|
16
21
|
import type { CitationLogger, PageFailure, PageStatus } from './types.js';
|
|
17
22
|
/**
|
|
18
23
|
* The fallback a failed page is retried through: the web scraper pack's `WebScraperService`, whose tiers go from a
|
|
@@ -33,11 +38,27 @@ export interface TransportAnswer {
|
|
|
33
38
|
header(name: string): string | null;
|
|
34
39
|
/** The body read up to `cap` bytes: its bytes, or null past the cap. */
|
|
35
40
|
read(cap: number): Promise<Uint8Array | null>;
|
|
41
|
+
/** The body's first `cap` bytes, the rest left unread. Both transports here give it; robots.txt is read with it. */
|
|
42
|
+
readPrefix?(cap: number): Promise<Uint8Array>;
|
|
36
43
|
/** Releases a body that will not be read. */
|
|
37
44
|
discard(): Promise<void>;
|
|
38
45
|
}
|
|
39
|
-
/**
|
|
40
|
-
|
|
46
|
+
/**
|
|
47
|
+
* The rules a transport applies to the address it is given and to every redirect target, before each request: the
|
|
48
|
+
* host's rule, and the robots.txt of each hop's origin, read through the same transport.
|
|
49
|
+
*/
|
|
50
|
+
export interface TransportRules {
|
|
51
|
+
/** The host's rule for an address: false refuses it, a `PageRefusal` with `refused` and `host`. */
|
|
52
|
+
allowUrl?: (url: string) => boolean;
|
|
53
|
+
/** robots.txt as read before each request: a disallow refuses the address, a `PageRefusal` with `refused` and `robots`. */
|
|
54
|
+
robots?: RobotsRule;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* One GET of an address under a signal. Given rules, the transport applies them to the address and to every redirect
|
|
58
|
+
* target before its request, and throws {@link PageRefusal} for one they refuse; a transport of the host's own that
|
|
59
|
+
* follows redirects applies them too, or the reader checks the first address alone.
|
|
60
|
+
*/
|
|
61
|
+
export type Transport = (url: string, signal: AbortSignal, rules?: TransportRules) => Promise<TransportAnswer>;
|
|
41
62
|
/** A failure a transport names itself, such as the egress policy's refusal. */
|
|
42
63
|
export declare class PageRefusal extends Error {
|
|
43
64
|
readonly failure: PageFailure;
|
|
@@ -48,16 +69,23 @@ export declare class PageRefusal extends Error {
|
|
|
48
69
|
*/
|
|
49
70
|
constructor(failure: PageFailure, detail?: string | null);
|
|
50
71
|
}
|
|
72
|
+
/** The licence of a page that states none the reader finds, and of a text that is not HTML. */
|
|
73
|
+
export declare const UNREAD_LICENCE: Readonly<SourceLicence>;
|
|
51
74
|
/** The address a page is fetched at: the cited address without its fragment, which never reaches a server. */
|
|
52
75
|
export declare function pageKey(url: string): string;
|
|
53
76
|
/** An address's origin for a log line, which leaves out the path and the query where secrets can sit. */
|
|
54
77
|
export declare function originOf(url: string): string;
|
|
55
|
-
/**
|
|
78
|
+
/**
|
|
79
|
+
* Requests through a `fetch`, the body streamed up to the cap. With no rules, `fetch` follows the redirects itself;
|
|
80
|
+
* given rules, up to five redirects are followed one at a time, each target checked before its request and one that
|
|
81
|
+
* is not http or https refused (`scheme`).
|
|
82
|
+
*/
|
|
56
83
|
export declare function fetchTransport(fetchFn: typeof globalThis.fetch, userAgent?: string): Transport;
|
|
57
84
|
/**
|
|
58
85
|
* Requests through AgentOS's checked egress (`@framers/agentos/safety/egress`, from 0.13.120): each address, and each
|
|
59
86
|
* address a redirect names, is checked against the policy before a byte is sent, its connection pinned to the address
|
|
60
|
-
* checked, and the answer bounded by the deadline and the size cap. Up to five redirects are followed
|
|
87
|
+
* checked, and the answer bounded by the deadline and the size cap. Up to five redirects are followed, a target that is
|
|
88
|
+
* not http or https refused (`scheme`); given rules, each address is checked against them before its request too.
|
|
61
89
|
*
|
|
62
90
|
* @throws {Error} When the installed AgentOS has no checked egress.
|
|
63
91
|
*/
|
|
@@ -65,11 +93,17 @@ export declare function egressTransport(profile: EgressProfile, bounds: {
|
|
|
65
93
|
timeoutMs: number;
|
|
66
94
|
maxBytes: number;
|
|
67
95
|
}, userAgent?: string): Promise<Transport>;
|
|
96
|
+
/** Reads a page the reader's own fetch found no text in, such as one a script draws. Answers the text as blocks, or null. */
|
|
97
|
+
export interface PageFallbackReader {
|
|
98
|
+
read(url: string, options: {
|
|
99
|
+
signal: AbortSignal;
|
|
100
|
+
}): Promise<string[] | null>;
|
|
101
|
+
}
|
|
68
102
|
/** How a page is read. */
|
|
69
103
|
export interface PageReaderOptions {
|
|
70
104
|
/** The request each attempt goes through. */
|
|
71
105
|
transport: Transport;
|
|
72
|
-
/** The fallback for a failure it may mend, or null for none. */
|
|
106
|
+
/** The fallback for a failure it may mend, or null for none; not used when `allowUrl` or `robots` is given. */
|
|
73
107
|
scraper: PageScraper | null;
|
|
74
108
|
/** The deadline of one request, in milliseconds. */
|
|
75
109
|
timeoutMs: number;
|
|
@@ -83,8 +117,26 @@ export interface PageReaderOptions {
|
|
|
83
117
|
now: () => number;
|
|
84
118
|
/** Where the reader writes its events. */
|
|
85
119
|
logger?: CitationLogger;
|
|
120
|
+
/**
|
|
121
|
+
* The host's rule for an address, run on the first address and on every redirect target before its request: false
|
|
122
|
+
* ends the page as `refused` with `detail: 'host'`, the address never requested.
|
|
123
|
+
*/
|
|
124
|
+
allowUrl?: (url: string) => boolean;
|
|
125
|
+
/**
|
|
126
|
+
* robots.txt by RFC 9309, read before the first address and before every redirect target: a disallow ends the page as
|
|
127
|
+
* `refused` with `detail: 'robots'`, the address never requested; a robots.txt the egress policy refuses ends it with
|
|
128
|
+
* the policy's reason, as the same read with no robots rule does. None is read when left out.
|
|
129
|
+
*/
|
|
130
|
+
robots?: RobotsRule;
|
|
131
|
+
/**
|
|
132
|
+
* The reader for a page that answered 200 with no text, called once for the address the reader's own request reached
|
|
133
|
+
* under the rules (every hop checked), with the caller's signal, its blocks held to `textBudget`. With none, or when it
|
|
134
|
+
* answers null, the page is `unreadable` with `no_text`. A fallback that requests the page itself does so outside
|
|
135
|
+
* these rules: the reader asks it only for an address they cleared.
|
|
136
|
+
*/
|
|
137
|
+
fallback?: PageFallbackReader;
|
|
86
138
|
}
|
|
87
|
-
/** A page as the check reads it: its status, and when it was read, its blocks and its sentences as passages. */
|
|
139
|
+
/** A page as the check reads it: its status, and when it was read, its document, its blocks and its sentences as passages. */
|
|
88
140
|
export interface PageRead {
|
|
89
141
|
/** How the page fared. */
|
|
90
142
|
status: PageStatus;
|
|
@@ -92,11 +144,20 @@ export interface PageRead {
|
|
|
92
144
|
blocks: string[];
|
|
93
145
|
/** Its sentences, shaped as AgentOS's `VerificationSource`, in page order; empty when it was not read. */
|
|
94
146
|
passages: CitationSource[];
|
|
147
|
+
/**
|
|
148
|
+
* The page as a document: its address after redirects, its `<title>` cut to 255 characters (the host when it has
|
|
149
|
+
* none), the licence it states (`not read` when the reader finds none), the publisher's name and its own date where
|
|
150
|
+
* it states them, and its blocks; null when it was not read.
|
|
151
|
+
*/
|
|
152
|
+
document: SourceDocument | null;
|
|
153
|
+
/** The page's own date in milliseconds, as on the document; null when it states none or was not read. */
|
|
154
|
+
pageDate: number | null;
|
|
95
155
|
}
|
|
96
156
|
/**
|
|
97
|
-
* Reads one page: one request, then for a
|
|
98
|
-
*
|
|
99
|
-
*
|
|
157
|
+
* Reads one page: the rules checked when given, one request, then for a 200 with no text one read of its address after
|
|
158
|
+
* redirects through the fallback reader when given, and, when no rules are given, for a failure the scraper may mend (a
|
|
159
|
+
* deadline, no connection, an error status other than 404 and 410, a page with no text) one try through the fallback
|
|
160
|
+
* scraper. Answers a status for a page that is not read; throws only the caller's own cancellation.
|
|
100
161
|
*
|
|
101
162
|
* @param url - The address to fetch, from {@link pageKey}.
|
|
102
163
|
* @param options - How to read it.
|
package/dist/page.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"page.d.ts","sourceRoot":"","sources":["../src/page.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"page.d.ts","sourceRoot":"","sources":["../src/page.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,EAAiC,KAAK,cAAc,EAAE,KAAK,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,qCAAqC,CAAC;AAClJ,OAAO,EAAkB,KAAK,WAAW,EAAE,KAAK,YAAY,EAAE,MAAM,kCAAkC,CAAC;AAKvG,OAAO,EAA4B,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AACxE,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1E;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CACnD;AAED,8HAA8H;AAC9H,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE/C,yCAAyC;AACzC,MAAM,WAAW,eAAe;IAC9B,gCAAgC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,0CAA0C;IAC1C,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACpC,wEAAwE;IACxE,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAC9C,oHAAoH;IACpH,UAAU,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAC9C,6CAA6C;IAC7C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,mGAAmG;IACnG,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IACpC,2HAA2H;IAC3H,MAAM,CAAC,EAAE,UAAU,CAAC;CACrB;AAED;;;;GAIG;AACH,MAAM,MAAM,SAAS,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,CAAC,EAAE,cAAc,KAAK,OAAO,CAAC,eAAe,CAAC,CAAC;AAE/G,+EAA+E;AAC/E,qBAAa,WAAY,SAAQ,KAAK;IAMlC,QAAQ,CAAC,OAAO,EAAE,WAAW;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IANhC;;;OAGG;gBAEQ,OAAO,EAAE,WAAW,EACpB,MAAM,GAAE,MAAM,GAAG,IAAW;CAKxC;AAQD,+FAA+F;AAC/F,eAAO,MAAM,cAAc,EAAE,QAAQ,CAAC,aAAa,CAAgD,CAAC;AAEpG,8GAA8G;AAC9G,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAI3C;AAED,yGAAyG;AACzG,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAM5C;AAsGD;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,UAAU,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAsB9F;AAwCD;;;;;;;GAOG;AACH,wBAAsB,eAAe,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAwCrJ;AAoFD,6HAA6H;AAC7H,MAAM,WAAW,kBAAkB;IACjC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE;QAAE,MAAM,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC;CAC/E;AAED,0BAA0B;AAC1B,MAAM,WAAW,iBAAiB;IAChC,6CAA6C;IAC7C,SAAS,EAAE,SAAS,CAAC;IACrB,+GAA+G;IAC/G,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;IAC5B,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,+CAA+C;IAC/C,GAAG,EAAE,MAAM,MAAM,CAAC;IAClB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB;;;OAGG;IACH,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IACpC;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,kBAAkB,CAAC;CAC/B;AAED,8HAA8H;AAC9H,MAAM,WAAW,QAAQ;IACvB,0BAA0B;IAC1B,MAAM,EAAE,UAAU,CAAC;IACnB,kEAAkE;IAClE,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,0GAA0G;IAC1G,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B;;;;OAIG;IACH,QAAQ,EAAE,cAAc,GAAG,IAAI,CAAC;IAChC,yGAAyG;IACzG,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAoLD;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAgCzF"}
|