@framers/agentos-ext-citation-check 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -2
- package/dist/index.d.ts +7 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -2
- package/dist/index.js.map +1 -1
- package/dist/licence.d.ts +31 -0
- package/dist/licence.d.ts.map +1 -0
- package/dist/licence.js +122 -0
- package/dist/licence.js.map +1 -0
- package/dist/meta.d.ts +66 -0
- package/dist/meta.d.ts.map +1 -0
- package/dist/meta.js +283 -0
- package/dist/meta.js.map +1 -0
- package/dist/page.d.ts +78 -17
- package/dist/page.d.ts.map +1 -1
- package/dist/page.js +235 -37
- package/dist/page.js.map +1 -1
- package/dist/robots.d.ts +90 -0
- package/dist/robots.d.ts.map +1 -0
- package/dist/robots.js +249 -0
- package/dist/robots.js.map +1 -0
- package/dist/types.d.ts +9 -6
- package/dist/types.d.ts.map +1 -1
- package/dist/web-source.d.ts +95 -0
- package/dist/web-source.d.ts.map +1 -0
- package/dist/web-source.js +176 -0
- package/dist/web-source.js.map +1 -0
- package/manifest.json +4 -3
- package/package.json +10 -8
package/dist/robots.js
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file robots.ts
|
|
3
|
+
* @description robots.txt as RFC 9309 gives it (https://www.rfc-editor.org/rfc/rfc9309.html). The file of a page's
|
|
4
|
+
* origin is read through the transport the page is read through, under a deadline of its own, and parsed to its
|
|
5
|
+
* first `maxBytes` bytes (the RFC's floor is 500 KiB; a line the limit cuts is left out). Its groups are matched by
|
|
6
|
+
* product token without case, the `*` group when none names the token; within the groups that apply, the longest
|
|
7
|
+
* matching rule decides, an `Allow` over a `Disallow` of the same length, `*` matching any run of characters and a
|
|
8
|
+
* final `$` the end of the path and query. A 400-range answer allows every page (2.3.1.3); a 500-range answer, a
|
|
9
|
+
* failed request or a passed deadline refuses every page (2.3.1.4); a file past five redirects is taken as
|
|
10
|
+
* unavailable (2.3.1.2). A refusal of the transport's egress policy, for the file's address or a hop of it, is the
|
|
11
|
+
* policy's answer and not the site's: it is thrown, so the page ends with the policy's reason. An origin's answer is
|
|
12
|
+
* kept for `cacheMs` per transport and parsing limit, save one the host's rule decided, since the next read may be
|
|
13
|
+
* under another rule.
|
|
14
|
+
*
|
|
15
|
+
* @module agentos/extensions/research/citation-check/robots
|
|
16
|
+
*/
|
|
17
|
+
/** The parsing limit RFC 9309's section 2.5 sets as the least a reader parses: 500 KiB, written as 512,000 bytes. */
|
|
18
|
+
export const ROBOTS_PARSING_FLOOR = 512_000;
|
|
19
|
+
const ALLOW_ALL = Object.freeze({ kind: 'allow-all' });
|
|
20
|
+
const DISALLOW_ALL = Object.freeze({ kind: 'disallow-all' });
|
|
21
|
+
/** The characters RFC 3986 calls unreserved, which a percent-encoding of them stands for. */
|
|
22
|
+
const UNRESERVED = /^[A-Za-z0-9\-._~]$/u;
|
|
23
|
+
/** The most origins one transport keeps an answer for; the oldest goes first. */
|
|
24
|
+
const CACHE_ENTRIES = 1_000;
|
|
25
|
+
/**
|
|
26
|
+
* A path in one form for comparison: characters outside printable ASCII percent-encoded in UTF-8, an encoded
|
|
27
|
+
* unreserved character decoded, and every other encoding in upper case (RFC 9309, 2.2.2).
|
|
28
|
+
*/
|
|
29
|
+
function normalizePath(path) {
|
|
30
|
+
let encoded = '';
|
|
31
|
+
for (const char of path) {
|
|
32
|
+
const code = char.codePointAt(0);
|
|
33
|
+
if (code > 0x20 && code < 0x7f) {
|
|
34
|
+
encoded += char;
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
try {
|
|
38
|
+
encoded += encodeURIComponent(char);
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
// A lone surrogate has no UTF-8 form; it is left out.
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return encoded.replace(/%([0-9A-Fa-f]{2})/gu, (_whole, hex) => {
|
|
45
|
+
const char = String.fromCharCode(Number.parseInt(hex, 16));
|
|
46
|
+
return UNRESERVED.test(char) ? char : `%${hex.toUpperCase()}`;
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Parses a robots.txt's text into its groups. Consecutive `User-agent` lines open one group, which its `Allow` and
|
|
51
|
+
* `Disallow` lines fill; a rule before any `User-agent` line, an empty rule and every other line (`Sitemap`,
|
|
52
|
+
* `Crawl-delay`, a comment) are passed over.
|
|
53
|
+
*/
|
|
54
|
+
export function parseRobots(text) {
|
|
55
|
+
const groups = [];
|
|
56
|
+
let current = null;
|
|
57
|
+
let readingAgents = false;
|
|
58
|
+
for (const raw of text.replace(/^/u, '').split(/\r\n|\r|\n/u)) {
|
|
59
|
+
const line = raw.replace(/#.*$/u, '').trim();
|
|
60
|
+
const colon = line.indexOf(':');
|
|
61
|
+
if (colon <= 0)
|
|
62
|
+
continue;
|
|
63
|
+
const field = line.slice(0, colon).trim().toLowerCase();
|
|
64
|
+
const value = line.slice(colon + 1).trim();
|
|
65
|
+
if (field === 'user-agent') {
|
|
66
|
+
if (current === null || !readingAgents) {
|
|
67
|
+
current = { agents: [], rules: [] };
|
|
68
|
+
groups.push(current);
|
|
69
|
+
readingAgents = true;
|
|
70
|
+
}
|
|
71
|
+
if (value !== '')
|
|
72
|
+
current.agents.push(value.toLowerCase());
|
|
73
|
+
}
|
|
74
|
+
else if (field === 'allow' || field === 'disallow') {
|
|
75
|
+
if (current === null)
|
|
76
|
+
continue;
|
|
77
|
+
readingAgents = false;
|
|
78
|
+
if (value !== '')
|
|
79
|
+
current.rules.push({ allow: field === 'allow', pattern: normalizePath(value) });
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return { kind: 'rules', groups };
|
|
83
|
+
}
|
|
84
|
+
/** Whether a group's `User-agent` value names the token: the same word without case, or the word with a version after it. */
|
|
85
|
+
function namesToken(agent, token) {
|
|
86
|
+
return agent === token || (token !== '*' && agent.startsWith(`${token}/`));
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Whether a rule's pattern matches a path and query from their start: `*` any run of characters, a final `$` the end.
|
|
90
|
+
* Each piece between wildcards is found at its first place after the one before it, which is where it leaves the most
|
|
91
|
+
* path for the pieces after it, so the time grows with the pattern and the path and never with the ways to split them.
|
|
92
|
+
*/
|
|
93
|
+
function matches(pattern, path) {
|
|
94
|
+
const anchored = pattern.endsWith('$');
|
|
95
|
+
const pieces = (anchored ? pattern.slice(0, -1) : pattern).split('*');
|
|
96
|
+
const first = pieces[0];
|
|
97
|
+
if (!path.startsWith(first))
|
|
98
|
+
return false;
|
|
99
|
+
if (pieces.length === 1)
|
|
100
|
+
return !anchored || path.length === first.length;
|
|
101
|
+
let at = first.length;
|
|
102
|
+
for (const piece of pieces.slice(1, -1)) {
|
|
103
|
+
if (piece === '')
|
|
104
|
+
continue;
|
|
105
|
+
const found = path.indexOf(piece, at);
|
|
106
|
+
if (found === -1)
|
|
107
|
+
return false;
|
|
108
|
+
at = found + piece.length;
|
|
109
|
+
}
|
|
110
|
+
const last = pieces[pieces.length - 1];
|
|
111
|
+
if (anchored)
|
|
112
|
+
return path.length - last.length >= at && path.endsWith(last);
|
|
113
|
+
return path.indexOf(last, at) !== -1;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Whether robots.txt lets a reader with this product token read the address. `/robots.txt` itself is always allowed.
|
|
117
|
+
*
|
|
118
|
+
* @param file - The file, from {@link readRobots} or {@link parseRobots}.
|
|
119
|
+
* @param token - The reader's product token; `*` obeys the `*` group alone.
|
|
120
|
+
* @param url - The address to read.
|
|
121
|
+
*/
|
|
122
|
+
export function robotsAllows(file, token, url) {
|
|
123
|
+
let target;
|
|
124
|
+
try {
|
|
125
|
+
target = new URL(url);
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
if (target.pathname === '/robots.txt')
|
|
131
|
+
return true;
|
|
132
|
+
if (file.kind === 'allow-all')
|
|
133
|
+
return true;
|
|
134
|
+
if (file.kind === 'disallow-all')
|
|
135
|
+
return false;
|
|
136
|
+
const path = normalizePath(`${target.pathname}${target.search}`) || '/';
|
|
137
|
+
const wanted = token.trim().toLowerCase();
|
|
138
|
+
let groups = file.groups.filter((group) => group.agents.some((agent) => namesToken(agent, wanted)));
|
|
139
|
+
if (groups.length === 0)
|
|
140
|
+
groups = file.groups.filter((group) => group.agents.includes('*'));
|
|
141
|
+
let best = null;
|
|
142
|
+
for (const group of groups) {
|
|
143
|
+
for (const rule of group.rules) {
|
|
144
|
+
if (!matches(rule.pattern, path))
|
|
145
|
+
continue;
|
|
146
|
+
const longer = best === null || rule.pattern.length > best.pattern.length;
|
|
147
|
+
const allowOverDisallow = best !== null && rule.pattern.length === best.pattern.length && rule.allow && !best.allow;
|
|
148
|
+
if (longer || allowOverDisallow)
|
|
149
|
+
best = rule;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return best === null || best.allow;
|
|
153
|
+
}
|
|
154
|
+
/** The answers each transport keeps, by origin and parsing limit, with when each was read. */
|
|
155
|
+
const kept = new WeakMap();
|
|
156
|
+
/** A transport's refusal, by its failure and its detail, or null for another error. */
|
|
157
|
+
function refusalOf(error) {
|
|
158
|
+
if (!(error instanceof Error) || error.name !== 'PageRefusal')
|
|
159
|
+
return null;
|
|
160
|
+
const { failure, detail } = error;
|
|
161
|
+
return { failure, detail };
|
|
162
|
+
}
|
|
163
|
+
/** A file's bytes as text, held to the parsing limit; past it, the line the limit cuts is left out. */
|
|
164
|
+
function robotsText(bytes, limit) {
|
|
165
|
+
let held = bytes;
|
|
166
|
+
if (bytes.byteLength > limit) {
|
|
167
|
+
held = bytes.subarray(0, limit);
|
|
168
|
+
const lastBreak = Math.max(held.lastIndexOf(0x0a), held.lastIndexOf(0x0d));
|
|
169
|
+
held = held.subarray(0, lastBreak + 1);
|
|
170
|
+
}
|
|
171
|
+
return new TextDecoder('utf-8').decode(held);
|
|
172
|
+
}
|
|
173
|
+
/** Requests the file and reads what its answer means. */
|
|
174
|
+
async function fetchRobots(robotsUrl, options) {
|
|
175
|
+
const deadline = AbortSignal.timeout(options.robots.timeoutMs);
|
|
176
|
+
const signal = options.signal ? AbortSignal.any([deadline, options.signal]) : deadline;
|
|
177
|
+
let answer;
|
|
178
|
+
try {
|
|
179
|
+
answer = await options.transport(robotsUrl, signal, options.allowUrl !== undefined ? { allowUrl: options.allowUrl } : undefined);
|
|
180
|
+
}
|
|
181
|
+
catch (error) {
|
|
182
|
+
if (options.signal?.aborted)
|
|
183
|
+
throw options.signal.reason;
|
|
184
|
+
const refusal = refusalOf(error);
|
|
185
|
+
if (refusal?.failure === 'redirects')
|
|
186
|
+
return { file: ALLOW_ALL, keep: true };
|
|
187
|
+
if (refusal?.failure === 'refused') {
|
|
188
|
+
// The host's rule refused a hop: the rule is the caller's, so the next read, under its own rule, asks again.
|
|
189
|
+
if (refusal.detail === 'host')
|
|
190
|
+
return { file: DISALLOW_ALL, keep: false };
|
|
191
|
+
// The egress policy refused the file's address or a hop of it before a byte was sent: the policy decided, not the
|
|
192
|
+
// site, so its refusal ends the page with its own reason, and nothing is kept for it.
|
|
193
|
+
throw error;
|
|
194
|
+
}
|
|
195
|
+
return { file: DISALLOW_ALL, keep: true };
|
|
196
|
+
}
|
|
197
|
+
try {
|
|
198
|
+
if (answer.status >= 200 && answer.status < 300) {
|
|
199
|
+
const limit = options.robots.maxBytes;
|
|
200
|
+
// One byte past the limit says whether the file runs past it.
|
|
201
|
+
const bytes = answer.readPrefix !== undefined ? await answer.readPrefix(limit + 1) : await answer.read(limit + 1);
|
|
202
|
+
if (bytes === null)
|
|
203
|
+
return { file: DISALLOW_ALL, keep: true };
|
|
204
|
+
return { file: parseRobots(robotsText(bytes, limit)), keep: true };
|
|
205
|
+
}
|
|
206
|
+
await answer.discard();
|
|
207
|
+
return { file: answer.status >= 500 ? DISALLOW_ALL : ALLOW_ALL, keep: true };
|
|
208
|
+
}
|
|
209
|
+
catch {
|
|
210
|
+
await answer.discard().catch(() => undefined);
|
|
211
|
+
if (options.signal?.aborted)
|
|
212
|
+
throw options.signal.reason;
|
|
213
|
+
return { file: DISALLOW_ALL, keep: true };
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Reads the robots.txt of an address's origin through the transport, or the answer the transport keeps for it.
|
|
218
|
+
* A 2xx answer is parsed; a 400-range answer allows every page; a 500-range answer, a failed request, a passed
|
|
219
|
+
* deadline or a hop the host rule refuses refuses every page; a file past five redirects allows every page. An answer
|
|
220
|
+
* is kept for `cacheMs` per transport, origin and parsing limit, save one the host's rule decided.
|
|
221
|
+
*
|
|
222
|
+
* @param url - Any address of the origin.
|
|
223
|
+
* @param options - The transport, the rule, the host rule for the file's hops, the caller's cancellation.
|
|
224
|
+
* @throws The caller's cancellation reason, when `signal` aborts; a `TypeError` for an address that does not parse;
|
|
225
|
+
* the transport's `PageRefusal` (`refused`, with a detail other than `host`) when its egress policy refuses the file's
|
|
226
|
+
* address or a hop of it, which is kept for no later read.
|
|
227
|
+
*/
|
|
228
|
+
export async function readRobots(url, options) {
|
|
229
|
+
const origin = new URL(url).origin;
|
|
230
|
+
const now = options.now ?? Date.now;
|
|
231
|
+
let answers = kept.get(options.transport);
|
|
232
|
+
if (answers === undefined) {
|
|
233
|
+
answers = new Map();
|
|
234
|
+
kept.set(options.transport, answers);
|
|
235
|
+
}
|
|
236
|
+
// A file parsed to a shorter limit may lack a rule a longer one reads, so each limit keeps its own answer.
|
|
237
|
+
const key = `${options.robots.maxBytes} ${origin}`;
|
|
238
|
+
const held = answers.get(key);
|
|
239
|
+
if (held !== undefined && now() - held.at < options.robots.cacheMs)
|
|
240
|
+
return held.file;
|
|
241
|
+
const { file, keep } = await fetchRobots(`${origin}/robots.txt`, options);
|
|
242
|
+
answers.delete(key);
|
|
243
|
+
if (keep)
|
|
244
|
+
answers.set(key, { at: now(), file });
|
|
245
|
+
if (answers.size > CACHE_ENTRIES)
|
|
246
|
+
answers.delete(answers.keys().next().value);
|
|
247
|
+
return file;
|
|
248
|
+
}
|
|
249
|
+
//# sourceMappingURL=robots.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"robots.js","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAgBH,qHAAqH;AACrH,MAAM,CAAC,MAAM,oBAAoB,GAAG,OAAO,CAAC;AAiB5C,MAAM,SAAS,GAAe,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,WAAoB,EAAE,CAAC,CAAC;AAC5E,MAAM,YAAY,GAAe,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,cAAuB,EAAE,CAAC,CAAC;AAElF,6FAA6F;AAC7F,MAAM,UAAU,GAAG,qBAAqB,CAAC;AACzC,iFAAiF;AACjF,MAAM,aAAa,GAAG,KAAK,CAAC;AAE5B;;;GAGG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAW,CAAC;QAC3C,IAAI,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,IAAI,EAAE,CAAC;YAC/B,OAAO,IAAI,IAAI,CAAC;YAChB,SAAS;QACX,CAAC;QACD,IAAI,CAAC;YACH,OAAO,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QAAC,MAAM,CAAC;YACP,sDAAsD;QACxD,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAC,MAAM,EAAE,GAAW,EAAE,EAAE;QACpE,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;QAC3D,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC;IAChE,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,IAAI,OAAO,GAAuB,IAAI,CAAC;IACvC,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;QAC/D,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,KAAK,IAAI,CAAC;YAAE,SAAS;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACxD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC3B,IAAI,OAAO,KAAK,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;gBACvC,OAAO,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,aAAa,GAAG,IAAI,CAAC;YACvB,CAAC;YACD,IAAI,KAAK,KAAK,EAAE;gBAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QAC7D,CAAC;aAAM,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;YACrD,IAAI,OAAO,KAAK,IAAI;gBAAE,SAAS;YAC/B,aAAa,GAAG,KAAK,CAAC;YACtB,IAAI,KAAK,KAAK,EAAE;gBAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,KAAK,OAAO,EAAE,OAAO,EAAE,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACpG,CAAC;IACH,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AACnC,CAAC;AAED,6HAA6H;AAC7H,SAAS,UAAU,CAAC,KAAa,EAAE,KAAa;IAC9C,OAAO,KAAK,KAAK,KAAK,IAAI,CAAC,KAAK,KAAK,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,OAAe,EAAE,IAAY;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACtE,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAW,CAAC;IAClC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,CAAC;IAC1E,IAAI,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC;IACtB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,IAAI,KAAK,KAAK,EAAE;YAAE,SAAS;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACtC,IAAI,KAAK,KAAK,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAC/B,EAAE,GAAG,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC5B,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAW,CAAC;IACjD,IAAI,QAAQ;QAAE,OAAO,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5E,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;AACvC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,IAAgB,EAAE,KAAa,EAAE,GAAW;IACvE,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,aAAa;QAAE,OAAO,IAAI,CAAC;IACnD,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IAC3C,IAAI,IAAI,CAAC,IAAI,KAAK,cAAc;QAAE,OAAO,KAAK,CAAC;IAC/C,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,GAAG,CAAC;IACxE,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC1C,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IACpG,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5F,IAAI,IAAI,GAAsB,IAAI,CAAC;IACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC;gBAAE,SAAS;YAC3C,MAAM,MAAM,GAAG,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAC1E,MAAM,iBAAiB,GAAG,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;YACpH,IAAI,MAAM,IAAI,iBAAiB;gBAAE,IAAI,GAAG,IAAI,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC;AACrC,CAAC;AAgBD,8FAA8F;AAC9F,MAAM,IAAI,GAAG,IAAI,OAAO,EAA4D,CAAC;AAErF,uFAAuF;AACvF,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa;QAAE,OAAO,IAAI,CAAC;IAC3E,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,KAAwD,CAAC;IACrF,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC7B,CAAC;AAQD,uGAAuG;AACvG,SAAS,UAAU,CAAC,KAAiB,EAAE,KAAa;IAClD,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,KAAK,CAAC,UAAU,GAAG,KAAK,EAAE,CAAC;QAC7B,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAChC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3E,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AAC/C,CAAC;AAED,yDAAyD;AACzD,KAAK,UAAU,WAAW,CAAC,SAAiB,EAAE,OAA0B;IACtE,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IACvF,IAAI,MAAuB,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACnI,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QACzD,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,OAAO,EAAE,OAAO,KAAK,WAAW;YAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC7E,IAAI,OAAO,EAAE,OAAO,KAAK,SAAS,EAAE,CAAC;YACnC,6GAA6G;YAC7G,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;gBAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YAC1E,kHAAkH;YAClH,sFAAsF;YACtF,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC5C,CAAC;IACD,IAAI,CAAC;QACH,IAAI,MAAM,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;YACtC,8DAA8D;YAC9D,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YAClH,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAC9D,OAAO,EAAE,IAAI,EAAE,WAAW,CAAC,UAAU,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACrE,CAAC;QACD,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACvB,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC/E,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QACzD,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC5C,CAAC;AACH,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW,EAAE,OAA0B;IACtE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACpC,IAAI,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC1C,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IACvC,CAAC;IACD,2GAA2G;IAC3G,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,EAAE,CAAC;IACnD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,IAAI,KAAK,SAAS,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC,IAAI,CAAC;IACrF,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,WAAW,CAAC,GAAG,MAAM,aAAa,EAAE,OAAO,CAAC,CAAC;IAC1E,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACpB,IAAI,IAAI;QAAE,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,IAAI,OAAO,CAAC,IAAI,GAAG,aAAa;QAAE,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAe,CAAC,CAAC;IACxF,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -77,9 +77,9 @@ export interface CitationCheck {
|
|
|
77
77
|
}
|
|
78
78
|
/**
|
|
79
79
|
* Why a page was not read. A page is unreachable for `timeout` (no answer by the deadline), `network` (no connection,
|
|
80
|
-
* or a name that does not resolve), `status` (an answer that is not 2xx), `refused` (the egress policy
|
|
81
|
-
* address) or `redirects` (too many redirects); it is unreadable for
|
|
82
|
-
* (another type that is not text) or `no_text` (no text a reader sees).
|
|
80
|
+
* or a name that does not resolve), `status` (an answer that is not 2xx), `refused` (the egress policy, the host's rule
|
|
81
|
+
* or robots.txt refused the address or a redirect target) or `redirects` (too many redirects); it is unreadable for
|
|
82
|
+
* `too_large` (past the size cap), `pdf`, `binary` (another type that is not text) or `no_text` (no text a reader sees).
|
|
83
83
|
*/
|
|
84
84
|
export type PageFailure = 'timeout' | 'network' | 'status' | 'refused' | 'redirects' | 'too_large' | 'pdf' | 'binary' | 'no_text';
|
|
85
85
|
/** How one page fared. */
|
|
@@ -94,9 +94,12 @@ export interface PageStatus {
|
|
|
94
94
|
status: number | null;
|
|
95
95
|
/** Why it was not read; null when it was. */
|
|
96
96
|
failure: PageFailure | null;
|
|
97
|
-
/**
|
|
98
|
-
|
|
99
|
-
|
|
97
|
+
/**
|
|
98
|
+
* More about the failure: the content type of a binary answer, the egress policy's reason, `host` for an address the
|
|
99
|
+
* host's rule refused, `robots` for one robots.txt disallows; null otherwise.
|
|
100
|
+
*/
|
|
101
|
+
detail: 'host' | 'robots' | (string & {}) | null;
|
|
102
|
+
/** How its text was read: the plain request, or a fallback (the host's reader or the web scraper); null when it was not read. */
|
|
100
103
|
via: 'fetch' | 'fallback' | null;
|
|
101
104
|
/** How many sentences it gave. */
|
|
102
105
|
passages: number;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,kEAAkE;AAClE,MAAM,WAAW,UAAU;IACzB,oEAAoE;IACpE,IAAI,EAAE,MAAM,CAAC;IACb,qFAAqF;IACrF,KAAK,EAAE,MAAM,CAAC;IACd,iEAAiE;IACjE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,sHAAsH;IACtH,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,gBAAgB,GAAG,WAAW,GAAG,aAAa,GAAG,YAAY,GAAG,SAAS,CAAC;AAElH,wDAAwD;AACxD,eAAO,MAAM,iBAAiB,EAAE,SAAS,eAAe,EAUvD,CAAC;AAEF,6GAA6G;AAC7G,MAAM,MAAM,cAAc,GAAG,OAAO,GAAG,WAAW,GAAG,SAAS,CAAC;AAE/D,0DAA0D;AAC1D,MAAM,WAAW,eAAe;IAC9B,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,+GAA+G;IAC/G,KAAK,EAAE,MAAM,CAAC;CACf;AAED,8DAA8D;AAC9D,MAAM,WAAW,aAAa;IAC5B,wCAAwC;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,mEAAmE;IACnE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,6FAA6F;IAC7F,KAAK,EAAE,OAAO,CAAC;IACf,6DAA6D;IAC7D,OAAO,EAAE,eAAe,CAAC;IACzB,yFAAyF;IACzF,MAAM,EAAE,cAAc,GAAG,IAAI,CAAC;IAC9B;;;OAGG;IACH,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,uGAAuG;IACvG,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,wGAAwG;IACxG,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB;;;OAGG;IACH,IAAI,EAAE,UAAU,GAAG,IAAI,CAAC;CACzB;AAED;;;;;GAKG;AACH,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,WAAW,GAAG,KAAK,GAAG,QAAQ,GAAG,SAAS,CAAC;AAElI,0BAA0B;AAC1B,MAAM,WAAW,UAAU;IACzB,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IACZ,6CAA6C;IAC7C,OAAO,EAAE,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;IAC/C,mFAAmF;IACnF,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,qDAAqD;IACrD,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,6CAA6C;IAC7C,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;IAC5B
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,kEAAkE;AAClE,MAAM,WAAW,UAAU;IACzB,oEAAoE;IACpE,IAAI,EAAE,MAAM,CAAC;IACb,qFAAqF;IACrF,KAAK,EAAE,MAAM,CAAC;IACd,iEAAiE;IACjE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,sHAAsH;IACtH,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,gBAAgB,GAAG,WAAW,GAAG,aAAa,GAAG,YAAY,GAAG,SAAS,CAAC;AAElH,wDAAwD;AACxD,eAAO,MAAM,iBAAiB,EAAE,SAAS,eAAe,EAUvD,CAAC;AAEF,6GAA6G;AAC7G,MAAM,MAAM,cAAc,GAAG,OAAO,GAAG,WAAW,GAAG,SAAS,CAAC;AAE/D,0DAA0D;AAC1D,MAAM,WAAW,eAAe;IAC9B,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,+GAA+G;IAC/G,KAAK,EAAE,MAAM,CAAC;CACf;AAED,8DAA8D;AAC9D,MAAM,WAAW,aAAa;IAC5B,wCAAwC;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,mEAAmE;IACnE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,6FAA6F;IAC7F,KAAK,EAAE,OAAO,CAAC;IACf,6DAA6D;IAC7D,OAAO,EAAE,eAAe,CAAC;IACzB,yFAAyF;IACzF,MAAM,EAAE,cAAc,GAAG,IAAI,CAAC;IAC9B;;;OAGG;IACH,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,uGAAuG;IACvG,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,wGAAwG;IACxG,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB;;;OAGG;IACH,IAAI,EAAE,UAAU,GAAG,IAAI,CAAC;CACzB;AAED;;;;;GAKG;AACH,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,WAAW,GAAG,KAAK,GAAG,QAAQ,GAAG,SAAS,CAAC;AAElI,0BAA0B;AAC1B,MAAM,WAAW,UAAU;IACzB,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IACZ,6CAA6C;IAC7C,OAAO,EAAE,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;IAC/C,mFAAmF;IACnF,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,qDAAqD;IACrD,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,6CAA6C;IAC7C,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;IAC5B;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC;IACjD,iIAAiI;IACjI,GAAG,EAAE,OAAO,GAAG,UAAU,GAAG,IAAI,CAAC;IACjC,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,qDAAqD;AACrD,MAAM,WAAW,eAAe;IAC9B,oCAAoC;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,mEAAmE;IACnE,MAAM,EAAE,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IACxC,uCAAuC;IACvC,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChD,+DAA+D;IAC/D,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B;AAED,0BAA0B;AAC1B,MAAM,WAAW,cAAc;IAC7B,mDAAmD;IACnD,SAAS,EAAE,MAAM,CAAC;IAClB,6EAA6E;IAC7E,OAAO,EAAE,SAAS,GAAG,WAAW,CAAC;IACjC,2DAA2D;IAC3D,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,kFAAkF;IAClF,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,sFAAsF;IACtF,OAAO,EAAE,eAAe,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC5D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC5D,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC7D,KAAK,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC/D;AAED,0GAA0G;AAC1G,MAAM,MAAM,OAAO,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAE/D,6FAA6F;AAC7F,MAAM,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,KAAK,OAAO,CAAC;IAAE,KAAK,EAAE,YAAY,GAAG,eAAe,GAAG,SAAS,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC"}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file web-source.ts
|
|
3
|
+
* @description The open web as a public source. The host gives the search: a phrase in, the addresses of pages that
|
|
4
|
+
* may hold it out, best first, from whatever search service it uses. The provider answers each address as a hit and
|
|
5
|
+
* reads a hit's page through the page reader: under the host's rule for every address and redirect target, given the
|
|
6
|
+
* phrase whose search found the page, and under robots.txt by RFC 9309, through AgentOS's checked egress unless the
|
|
7
|
+
* host gives a transport. The page answers as a document with the host's name without "www." as its site, its own
|
|
8
|
+
* title, the licence it states, the publisher's name and its own date; a page that answered 200 with no text is read
|
|
9
|
+
* once through the host's fallback reader when one is given. A page that is not read is a skip, never a throw; only
|
|
10
|
+
* the caller's own cancellation throws.
|
|
11
|
+
*
|
|
12
|
+
* @module agentos/extensions/research/citation-check/web-source
|
|
13
|
+
*/
|
|
14
|
+
import { type CallOptions, type PublicSourceProvider, type ReadOutcome, type SearchOutcome, type SourceHit } from '@framers/agentos-ext-public-sources';
|
|
15
|
+
import { type EgressProfile, type PageFallbackReader, type Transport } from './page.js';
|
|
16
|
+
import type { RobotsRule } from './robots.js';
|
|
17
|
+
import type { CitationLogger } from './types.js';
|
|
18
|
+
/** robots.txt as the provider reads it when the host names no rule: the `*` group, 2,000 ms, 512,000 bytes, ten minutes kept. */
|
|
19
|
+
export declare const DEFAULT_ROBOTS: Readonly<RobotsRule>;
|
|
20
|
+
/** How the provider searches and reads. */
|
|
21
|
+
export interface WebPageSourceOptions {
|
|
22
|
+
/** The host's search: a short phrase in, page addresses out, best first. A throw answers the search as failed. */
|
|
23
|
+
search: (phrase: string, signal?: AbortSignal) => Promise<readonly string[]>;
|
|
24
|
+
/** The request pages go through; AgentOS's checked egress with `egress` when left out. */
|
|
25
|
+
transport?: Transport;
|
|
26
|
+
/** The checked egress policy when no transport is given: `hosted` (the default), public https addresses alone, or `local`. */
|
|
27
|
+
egress?: EgressProfile;
|
|
28
|
+
/** The User-Agent the checked egress sends when no transport is given; a browser's when left out. */
|
|
29
|
+
userAgent?: string;
|
|
30
|
+
/**
|
|
31
|
+
* The host's rule for each address and each redirect target, with the phrase whose search answered the hit read (the
|
|
32
|
+
* empty string for a hit object this provider's search did not answer, a copy of one included): false refuses it
|
|
33
|
+
* before it is requested.
|
|
34
|
+
*/
|
|
35
|
+
allowUrl?: (url: string, phrase: string) => boolean;
|
|
36
|
+
/** robots.txt as read before each address; {@link DEFAULT_ROBOTS} when left out, none with null. */
|
|
37
|
+
robots?: RobotsRule | null;
|
|
38
|
+
/** The reader for a page that answered 200 with no text. */
|
|
39
|
+
fallback?: PageFallbackReader;
|
|
40
|
+
/** The deadline of one request, the most bytes of a body, the most characters of text: 15,000, 5,000,000 and 300,000. */
|
|
41
|
+
timeoutMs?: number;
|
|
42
|
+
maxBytes?: number;
|
|
43
|
+
textBudget?: number;
|
|
44
|
+
/** The source's name as a reader sees it; `the open web` when left out. */
|
|
45
|
+
site?: string;
|
|
46
|
+
/** The clock of each document's `readAt`; `Date.now` when left out. */
|
|
47
|
+
now?: () => number;
|
|
48
|
+
/** Where the reader writes its events. */
|
|
49
|
+
logger?: CitationLogger;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* A `PublicSourceProvider` over the open web: `search` asks the host's search for addresses, `read` reads a page. It
|
|
53
|
+
* queues nothing: the host reads at most as many pages at once as it calls `read` for.
|
|
54
|
+
*/
|
|
55
|
+
export declare class WebPageSourceProvider implements PublicSourceProvider {
|
|
56
|
+
readonly site: string;
|
|
57
|
+
private readonly searchFn;
|
|
58
|
+
private readonly options;
|
|
59
|
+
private readonly timeoutMs;
|
|
60
|
+
private readonly maxBytes;
|
|
61
|
+
private readonly textBudget;
|
|
62
|
+
private readonly robots;
|
|
63
|
+
private readonly now;
|
|
64
|
+
private transport;
|
|
65
|
+
/** The phrase each hit this provider's search answered was found by, held as long as the hit is. */
|
|
66
|
+
private readonly phrases;
|
|
67
|
+
/**
|
|
68
|
+
* @param options - The host's search and how pages are read.
|
|
69
|
+
* @throws {TypeError} For a search that is not a function, or a bound that is not a whole number of at least 1.
|
|
70
|
+
*/
|
|
71
|
+
constructor(options: WebPageSourceOptions);
|
|
72
|
+
/** The transport, built once: the host's, or AgentOS's checked egress. */
|
|
73
|
+
private transportOf;
|
|
74
|
+
/**
|
|
75
|
+
* Searches for a short phrase: a cancellation that came first is thrown before anything runs, then `onSend` is
|
|
76
|
+
* awaited once and the host's search runs, and its addresses answer
|
|
77
|
+
* as hits (`key` the address without its fragment, `title` the address), each page once, at most `limit` (3 when
|
|
78
|
+
* left out), an address that is not http or https passed over. `limited` when `onSend` throws a limiter's refusal
|
|
79
|
+
* (nothing was sent), `failed` when it throws anything else or the search throws. The caller's own cancellation is
|
|
80
|
+
* thrown.
|
|
81
|
+
*/
|
|
82
|
+
search(phrase: string, options?: CallOptions & {
|
|
83
|
+
limit?: number;
|
|
84
|
+
}): Promise<SearchOutcome>;
|
|
85
|
+
/**
|
|
86
|
+
* Reads a hit's page: `document` with `site` the host without "www.", `revision` null, `readAt` the clock's, and the
|
|
87
|
+
* page's title, licence, publisher and date as read, or `skipped` with the status when one came. The caller's own
|
|
88
|
+
* cancellation is thrown.
|
|
89
|
+
*
|
|
90
|
+
* @throws The caller's cancellation reason; an `Error` when no transport is given and the installed AgentOS has no
|
|
91
|
+
* checked egress.
|
|
92
|
+
*/
|
|
93
|
+
read(hit: SourceHit, options?: Omit<CallOptions, 'onSend'>): Promise<ReadOutcome>;
|
|
94
|
+
}
|
|
95
|
+
//# sourceMappingURL=web-source.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"web-source.d.ts","sourceRoot":"","sources":["../src/web-source.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAEL,KAAK,WAAW,EAChB,KAAK,oBAAoB,EACzB,KAAK,WAAW,EAChB,KAAK,aAAa,EAClB,KAAK,SAAS,EACf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAsC,KAAK,aAAa,EAAE,KAAK,kBAAkB,EAAE,KAAK,SAAS,EAAE,MAAM,WAAW,CAAC;AAC5H,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjD,iIAAiI;AACjI,eAAO,MAAM,cAAc,EAAE,QAAQ,CAAC,UAAU,CAAwF,CAAC;AAEzI,2CAA2C;AAC3C,MAAM,WAAW,oBAAoB;IACnC,kHAAkH;IAClH,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC,CAAC;IAC7E,0FAA0F;IAC1F,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,8HAA8H;IAC9H,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,qGAAqG;IACrG,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;IACpD,oGAAoG;IACpG,MAAM,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;IAC3B,4DAA4D;IAC5D,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAC9B,yHAAyH;IACzH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,2EAA2E;IAC3E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,uEAAuE;IACvE,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB;AAsCD;;;GAGG;AACH,qBAAa,qBAAsB,YAAW,oBAAoB;IAChE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiC;IAC1D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuB;IAC/C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,SAAS,CAAmC;IACpD,oGAAoG;IACpG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoC;IAE5D;;;OAGG;gBACS,OAAO,EAAE,oBAAoB;IAYzC,0EAA0E;IAC1E,OAAO,CAAC,WAAW;IAMnB;;;;;;;OAOG;IACG,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,WAAW,GAAG;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,aAAa,CAAC;IAmCpG;;;;;;;OAOG;IACG,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,GAAE,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAM,GAAG,OAAO,CAAC,WAAW,CAAC;CAuB5F"}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file web-source.ts
|
|
3
|
+
* @description The open web as a public source. The host gives the search: a phrase in, the addresses of pages that
|
|
4
|
+
* may hold it out, best first, from whatever search service it uses. The provider answers each address as a hit and
|
|
5
|
+
* reads a hit's page through the page reader: under the host's rule for every address and redirect target, given the
|
|
6
|
+
* phrase whose search found the page, and under robots.txt by RFC 9309, through AgentOS's checked egress unless the
|
|
7
|
+
* host gives a transport. The page answers as a document with the host's name without "www." as its site, its own
|
|
8
|
+
* title, the licence it states, the publisher's name and its own date; a page that answered 200 with no text is read
|
|
9
|
+
* once through the host's fallback reader when one is given. A page that is not read is a skip, never a throw; only
|
|
10
|
+
* the caller's own cancellation throws.
|
|
11
|
+
*
|
|
12
|
+
* @module agentos/extensions/research/citation-check/web-source
|
|
13
|
+
*/
|
|
14
|
+
import { LimiterRefused, } from '@framers/agentos-ext-public-sources';
|
|
15
|
+
import { egressTransport, pageKey, readPage } from './page.js';
|
|
16
|
+
/** robots.txt as the provider reads it when the host names no rule: the `*` group, 2,000 ms, 512,000 bytes, ten minutes kept. */
|
|
17
|
+
export const DEFAULT_ROBOTS = Object.freeze({ token: '*', timeoutMs: 2_000, maxBytes: 512_000, cacheMs: 600_000 });
|
|
18
|
+
/** The most hits a search answers when the call names no limit. */
|
|
19
|
+
const DEFAULT_LIMIT = 3;
|
|
20
|
+
/** A whole number of at least 1, or the default when left out. */
|
|
21
|
+
function whole(name, value, fallback) {
|
|
22
|
+
if (value === undefined)
|
|
23
|
+
return fallback;
|
|
24
|
+
if (!Number.isInteger(value) || value < 1)
|
|
25
|
+
throw new TypeError(`${name} must be a whole number of at least 1`);
|
|
26
|
+
return value;
|
|
27
|
+
}
|
|
28
|
+
/** Whether an error is a limiter's refusal, from any copy of public-sources. */
|
|
29
|
+
function isLimiterRefusal(error) {
|
|
30
|
+
return error instanceof LimiterRefused || (error instanceof Error && error.name === 'LimiterRefused');
|
|
31
|
+
}
|
|
32
|
+
/** An address as a hit's key: http or https, its fragment removed; null for any other. */
|
|
33
|
+
function keyOf(address) {
|
|
34
|
+
if (typeof address !== 'string')
|
|
35
|
+
return null;
|
|
36
|
+
try {
|
|
37
|
+
const key = pageKey(address.trim());
|
|
38
|
+
const { protocol } = new URL(key);
|
|
39
|
+
return protocol === 'http:' || protocol === 'https:' ? key : null;
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/** A page's host without a leading "www.", in lower case. */
|
|
46
|
+
function siteOf(url) {
|
|
47
|
+
try {
|
|
48
|
+
return new URL(url).hostname.toLowerCase().replace(/^www\./u, '');
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
return url;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* A `PublicSourceProvider` over the open web: `search` asks the host's search for addresses, `read` reads a page. It
|
|
56
|
+
* queues nothing: the host reads at most as many pages at once as it calls `read` for.
|
|
57
|
+
*/
|
|
58
|
+
export class WebPageSourceProvider {
|
|
59
|
+
site;
|
|
60
|
+
searchFn;
|
|
61
|
+
options;
|
|
62
|
+
timeoutMs;
|
|
63
|
+
maxBytes;
|
|
64
|
+
textBudget;
|
|
65
|
+
robots;
|
|
66
|
+
now;
|
|
67
|
+
transport = null;
|
|
68
|
+
/** The phrase each hit this provider's search answered was found by, held as long as the hit is. */
|
|
69
|
+
phrases = new WeakMap();
|
|
70
|
+
/**
|
|
71
|
+
* @param options - The host's search and how pages are read.
|
|
72
|
+
* @throws {TypeError} For a search that is not a function, or a bound that is not a whole number of at least 1.
|
|
73
|
+
*/
|
|
74
|
+
constructor(options) {
|
|
75
|
+
if (typeof options.search !== 'function')
|
|
76
|
+
throw new TypeError('a web page source needs a search function');
|
|
77
|
+
this.options = options;
|
|
78
|
+
this.searchFn = options.search;
|
|
79
|
+
this.site = options.site ?? 'the open web';
|
|
80
|
+
this.timeoutMs = whole('timeoutMs', options.timeoutMs, 15_000);
|
|
81
|
+
this.maxBytes = whole('maxBytes', options.maxBytes, 5_000_000);
|
|
82
|
+
this.textBudget = whole('textBudget', options.textBudget, 300_000);
|
|
83
|
+
this.robots = options.robots === undefined ? DEFAULT_ROBOTS : options.robots;
|
|
84
|
+
this.now = options.now ?? Date.now;
|
|
85
|
+
}
|
|
86
|
+
/** The transport, built once: the host's, or AgentOS's checked egress. */
|
|
87
|
+
transportOf() {
|
|
88
|
+
if (this.options.transport !== undefined)
|
|
89
|
+
return Promise.resolve(this.options.transport);
|
|
90
|
+
this.transport ??= egressTransport(this.options.egress ?? 'hosted', { timeoutMs: this.timeoutMs, maxBytes: this.maxBytes }, this.options.userAgent);
|
|
91
|
+
return this.transport;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Searches for a short phrase: a cancellation that came first is thrown before anything runs, then `onSend` is
|
|
95
|
+
* awaited once and the host's search runs, and its addresses answer
|
|
96
|
+
* as hits (`key` the address without its fragment, `title` the address), each page once, at most `limit` (3 when
|
|
97
|
+
* left out), an address that is not http or https passed over. `limited` when `onSend` throws a limiter's refusal
|
|
98
|
+
* (nothing was sent), `failed` when it throws anything else or the search throws. The caller's own cancellation is
|
|
99
|
+
* thrown.
|
|
100
|
+
*/
|
|
101
|
+
async search(phrase, options = {}) {
|
|
102
|
+
const limit = options.limit ?? DEFAULT_LIMIT;
|
|
103
|
+
// A search cancelled before it starts sends nothing, so nothing is charged for it.
|
|
104
|
+
options.signal?.throwIfAborted();
|
|
105
|
+
if (options.onSend !== undefined) {
|
|
106
|
+
try {
|
|
107
|
+
await options.onSend();
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
if (options.signal?.aborted)
|
|
111
|
+
throw options.signal.reason;
|
|
112
|
+
return isLimiterRefusal(error) ? { kind: 'limited' } : { kind: 'failed', status: null };
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
options.signal?.throwIfAborted();
|
|
116
|
+
let addresses;
|
|
117
|
+
try {
|
|
118
|
+
addresses = await this.searchFn(phrase, options.signal);
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
if (options.signal?.aborted)
|
|
122
|
+
throw options.signal.reason;
|
|
123
|
+
return { kind: 'failed', status: null };
|
|
124
|
+
}
|
|
125
|
+
if (options.signal?.aborted)
|
|
126
|
+
throw options.signal.reason;
|
|
127
|
+
const hits = [];
|
|
128
|
+
const seen = new Set();
|
|
129
|
+
for (const address of Array.isArray(addresses) ? addresses : []) {
|
|
130
|
+
if (hits.length >= limit)
|
|
131
|
+
break;
|
|
132
|
+
const key = keyOf(address);
|
|
133
|
+
if (key === null || seen.has(key))
|
|
134
|
+
continue;
|
|
135
|
+
seen.add(key);
|
|
136
|
+
const hit = { key, title: address.trim() };
|
|
137
|
+
this.phrases.set(hit, phrase);
|
|
138
|
+
hits.push(hit);
|
|
139
|
+
}
|
|
140
|
+
return { kind: 'hits', hits };
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Reads a hit's page: `document` with `site` the host without "www.", `revision` null, `readAt` the clock's, and the
|
|
144
|
+
* page's title, licence, publisher and date as read, or `skipped` with the status when one came. The caller's own
|
|
145
|
+
* cancellation is thrown.
|
|
146
|
+
*
|
|
147
|
+
* @throws The caller's cancellation reason; an `Error` when no transport is given and the installed AgentOS has no
|
|
148
|
+
* checked egress.
|
|
149
|
+
*/
|
|
150
|
+
async read(hit, options = {}) {
|
|
151
|
+
options.signal?.throwIfAborted();
|
|
152
|
+
const url = keyOf(hit.key);
|
|
153
|
+
if (url === null)
|
|
154
|
+
return { kind: 'skipped', status: null };
|
|
155
|
+
const phrase = this.phrases.get(hit) ?? '';
|
|
156
|
+
const { allowUrl, fallback, logger } = this.options;
|
|
157
|
+
const transport = await this.transportOf();
|
|
158
|
+
const page = await readPage(url, {
|
|
159
|
+
transport,
|
|
160
|
+
scraper: null,
|
|
161
|
+
timeoutMs: this.timeoutMs,
|
|
162
|
+
maxBytes: this.maxBytes,
|
|
163
|
+
textBudget: this.textBudget,
|
|
164
|
+
now: this.now,
|
|
165
|
+
...(options.signal !== undefined ? { signal: options.signal } : {}),
|
|
166
|
+
...(logger !== undefined ? { logger } : {}),
|
|
167
|
+
...(allowUrl !== undefined ? { allowUrl: (address) => allowUrl(address, phrase) } : {}),
|
|
168
|
+
...(this.robots !== null ? { robots: this.robots } : {}),
|
|
169
|
+
...(fallback !== undefined ? { fallback } : {}),
|
|
170
|
+
});
|
|
171
|
+
if (page.document === null)
|
|
172
|
+
return { kind: 'skipped', status: page.status.status };
|
|
173
|
+
return { kind: 'document', document: { ...page.document, site: siteOf(page.document.url), revision: null } };
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
//# sourceMappingURL=web-source.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"web-source.js","sourceRoot":"","sources":["../src/web-source.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EACL,cAAc,GAMf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,QAAQ,EAA+D,MAAM,WAAW,CAAC;AAI5H,iIAAiI;AACjI,MAAM,CAAC,MAAM,cAAc,GAAyB,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;AAkCzI,mEAAmE;AACnE,MAAM,aAAa,GAAG,CAAC,CAAC;AAExB,kEAAkE;AAClE,SAAS,KAAK,CAAC,IAAY,EAAE,KAAyB,EAAE,QAAgB;IACtE,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,uCAAuC,CAAC,CAAC;IAC/G,OAAO,KAAK,CAAC;AACf,CAAC;AAED,gFAAgF;AAChF,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,KAAK,YAAY,cAAc,IAAI,CAAC,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,CAAC,CAAC;AACxG,CAAC;AAED,0FAA0F;AAC1F,SAAS,KAAK,CAAC,OAAgB;IAC7B,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACpC,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,OAAO,QAAQ,KAAK,OAAO,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IACpE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,6DAA6D;AAC7D,SAAS,MAAM,CAAC,GAAW;IACzB,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACpE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,qBAAqB;IACvB,IAAI,CAAS;IACL,QAAQ,CAAiC;IACzC,OAAO,CAAuB;IAC9B,SAAS,CAAS;IAClB,QAAQ,CAAS;IACjB,UAAU,CAAS;IACnB,MAAM,CAAoB;IAC1B,GAAG,CAAe;IAC3B,SAAS,GAA8B,IAAI,CAAC;IACpD,oGAAoG;IACnF,OAAO,GAAG,IAAI,OAAO,EAAqB,CAAC;IAE5D;;;OAGG;IACH,YAAY,OAA6B;QACvC,IAAI,OAAO,OAAO,CAAC,MAAM,KAAK,UAAU;YAAE,MAAM,IAAI,SAAS,CAAC,2CAA2C,CAAC,CAAC;QAC3G,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;QAC/B,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,cAAc,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,WAAW,EAAE,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC/D,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC/D,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACnE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QAC7E,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACrC,CAAC;IAED,0EAA0E;IAClE,WAAW;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACzF,IAAI,CAAC,SAAS,KAAK,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACpJ,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,MAAM,CAAC,MAAc,EAAE,UAA4C,EAAE;QACzE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,aAAa,CAAC;QAC7C,mFAAmF;QACnF,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC;YACzB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;oBAAE,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;gBACzD,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAC1F,CAAC;QACH,CAAC;QACD,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,IAAI,SAA6B,CAAC;QAClC,IAAI,CAAC;YACH,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC1D,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;gBAAE,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;YACzD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC1C,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QACzD,MAAM,IAAI,GAAgB,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YAChE,IAAI,IAAI,CAAC,MAAM,IAAI,KAAK;gBAAE,MAAM;YAChC,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;YAC3B,IAAI,GAAG,KAAK,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAC5C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,MAAM,GAAG,GAAc,EAAE,GAAG,EAAE,KAAK,EAAG,OAAkB,CAAC,IAAI,EAAE,EAAE,CAAC;YAClE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAChC,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,IAAI,CAAC,GAAc,EAAE,UAAuC,EAAE;QAClE,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,GAAG,KAAK,IAAI;YAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC3D,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAC3C,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACpD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE;YAC/B,SAAS;YACT,OAAO,EAAE,IAAI;YACb,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3C,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,OAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/F,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChD,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,QAAQ,KAAK,IAAI;YAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnF,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;IAC/G,CAAC;CACF"}
|
package/manifest.json
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
"$schema": "https://agentos.sh/schemas/extension-manifest-v1.json",
|
|
3
3
|
"id": "com.framers.research.citation-check",
|
|
4
4
|
"name": "Citation Check",
|
|
5
|
-
"version": "0.
|
|
6
|
-
"description": "Checks the quoted claims of a document against the pages they cite, with AgentOS's citation tooling: word for word first, then by embeddings when a model is configured",
|
|
5
|
+
"version": "0.2.0",
|
|
6
|
+
"description": "Checks the quoted claims of a document against the pages they cite, with AgentOS's citation tooling: word for word first, then by embeddings when a model is configured. Reads public pages under robots.txt and a host's rules, and serves the open web as a public source",
|
|
7
7
|
"author": {
|
|
8
8
|
"name": "Frame",
|
|
9
9
|
"email": "team@frame.dev",
|
|
@@ -14,7 +14,8 @@
|
|
|
14
14
|
"citations",
|
|
15
15
|
"fact-checking",
|
|
16
16
|
"grounding",
|
|
17
|
-
"research"
|
|
17
|
+
"research",
|
|
18
|
+
"robots-txt"
|
|
18
19
|
],
|
|
19
20
|
"agentosVersion": ">=0.13.120",
|
|
20
21
|
"categories": ["research"],
|