@framers/agentos-ext-citation-check 0.1.4 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +98 -5
- package/dist/firecrawl.d.ts +77 -0
- package/dist/firecrawl.d.ts.map +1 -0
- package/dist/firecrawl.js +184 -0
- package/dist/firecrawl.js.map +1 -0
- package/dist/index.d.ts +9 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -2
- package/dist/index.js.map +1 -1
- package/dist/licence.d.ts +31 -0
- package/dist/licence.d.ts.map +1 -0
- package/dist/licence.js +122 -0
- package/dist/licence.js.map +1 -0
- package/dist/match.d.ts +2 -1
- package/dist/match.d.ts.map +1 -1
- package/dist/match.js +11 -2
- package/dist/match.js.map +1 -1
- package/dist/meta.d.ts +66 -0
- package/dist/meta.d.ts.map +1 -0
- package/dist/meta.js +283 -0
- package/dist/meta.js.map +1 -0
- package/dist/page.d.ts +95 -17
- package/dist/page.d.ts.map +1 -1
- package/dist/page.js +291 -50
- package/dist/page.js.map +1 -1
- package/dist/parse.d.ts +2 -1
- package/dist/parse.d.ts.map +1 -1
- package/dist/parse.js +54 -3
- package/dist/parse.js.map +1 -1
- package/dist/robots.d.ts +90 -0
- package/dist/robots.d.ts.map +1 -0
- package/dist/robots.js +249 -0
- package/dist/robots.js.map +1 -0
- package/dist/types.d.ts +9 -6
- package/dist/types.d.ts.map +1 -1
- package/dist/web-source.d.ts +95 -0
- package/dist/web-source.d.ts.map +1 -0
- package/dist/web-source.js +176 -0
- package/dist/web-source.js.map +1 -0
- package/manifest.json +4 -3
- package/package.json +10 -8
package/dist/page.d.ts
CHANGED
|
@@ -1,18 +1,23 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @file page.ts
|
|
3
|
-
* @description One
|
|
4
|
-
* web scraper pack, a deadline and a size cap, through Node's fetch
|
|
5
|
-
*
|
|
6
|
-
*
|
|
3
|
+
* @description One page as a document and passages. A request goes out once per page, with a browser-like User-Agent
|
|
4
|
+
* from the web scraper pack, a deadline and a size cap, through Node's fetch or through AgentOS's checked egress (every
|
|
5
|
+
* address and every redirect checked against a policy), each following up to five redirects. Given rules, the reader checks the first
|
|
6
|
+
* address against the host's rule and the origin's robots.txt before it is requested, and the transport checks every
|
|
7
|
+
* redirect target the same way before its request. The answer is read by its content type: HTML and XML through
|
|
7
8
|
* public-sources' `htmlToBlocks` in page mode, other text by paragraphs, a PDF or another binary not at all; its
|
|
8
|
-
* charset from a byte order mark, the content type or a `<meta>` tag.
|
|
9
|
-
*
|
|
10
|
-
*
|
|
9
|
+
* charset from a byte order mark, the content type or a `<meta>` tag. The document carries the page's title, the
|
|
10
|
+
* licence it states, the publisher's name and the page's own date where the markup gives them. Its sentences become
|
|
11
|
+
* passages through public-sources' `citationSources`, cut by grounding-guard's `sentenceSpans`. A page that answered
|
|
12
|
+
* 200 with no text is read once through the host's fallback reader when one is given; a page that fails in a way a
|
|
13
|
+
* browser may mend is retried once through the web scraper pack's tiers. A page that fails is a status, never a throw;
|
|
14
|
+
* only the caller's own cancellation throws.
|
|
11
15
|
*
|
|
12
16
|
* @module agentos/extensions/research/citation-check/page
|
|
13
17
|
*/
|
|
14
|
-
import { type CitationSource } from '@framers/agentos-ext-public-sources';
|
|
18
|
+
import { type CitationSource, type SourceDocument, type SourceLicence } from '@framers/agentos-ext-public-sources';
|
|
15
19
|
import { type ScrapeInput, type ScrapeResult } from '@framers/agentos-ext-web-scraper';
|
|
20
|
+
import { type RobotsRule } from './robots.js';
|
|
16
21
|
import type { CitationLogger, PageFailure, PageStatus } from './types.js';
|
|
17
22
|
/**
|
|
18
23
|
* The fallback a failed page is retried through: the web scraper pack's `WebScraperService`, whose tiers go from a
|
|
@@ -33,11 +38,27 @@ export interface TransportAnswer {
|
|
|
33
38
|
header(name: string): string | null;
|
|
34
39
|
/** The body read up to `cap` bytes: its bytes, or null past the cap. */
|
|
35
40
|
read(cap: number): Promise<Uint8Array | null>;
|
|
41
|
+
/** The body's first `cap` bytes, the rest left unread. Both transports here give it; robots.txt is read with it. */
|
|
42
|
+
readPrefix?(cap: number): Promise<Uint8Array>;
|
|
36
43
|
/** Releases a body that will not be read. */
|
|
37
44
|
discard(): Promise<void>;
|
|
38
45
|
}
|
|
39
|
-
/**
|
|
40
|
-
|
|
46
|
+
/**
|
|
47
|
+
* The rules a transport applies to the address it is given and to every redirect target, before each request: the
|
|
48
|
+
* host's rule, and the robots.txt of each hop's origin, read through the same transport.
|
|
49
|
+
*/
|
|
50
|
+
export interface TransportRules {
|
|
51
|
+
/** The host's rule for an address: false refuses it, a `PageRefusal` with `refused` and `host`. */
|
|
52
|
+
allowUrl?: (url: string) => boolean;
|
|
53
|
+
/** robots.txt as read before each request: a disallow refuses the address, a `PageRefusal` with `refused` and `robots`. */
|
|
54
|
+
robots?: RobotsRule;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* One GET of an address under a signal. Given rules, the transport applies them to the address and to every redirect
|
|
58
|
+
* target before its request, and throws {@link PageRefusal} for one they refuse; a transport of the host's own that
|
|
59
|
+
* follows redirects applies them too, or the reader checks the first address alone.
|
|
60
|
+
*/
|
|
61
|
+
export type Transport = (url: string, signal: AbortSignal, rules?: TransportRules) => Promise<TransportAnswer>;
|
|
41
62
|
/** A failure a transport names itself, such as the egress policy's refusal. */
|
|
42
63
|
export declare class PageRefusal extends Error {
|
|
43
64
|
readonly failure: PageFailure;
|
|
@@ -48,16 +69,38 @@ export declare class PageRefusal extends Error {
|
|
|
48
69
|
*/
|
|
49
70
|
constructor(failure: PageFailure, detail?: string | null);
|
|
50
71
|
}
|
|
72
|
+
/** The licence of a page that states none the reader finds, and of a text that is not HTML. */
|
|
73
|
+
export declare const UNREAD_LICENCE: Readonly<SourceLicence>;
|
|
51
74
|
/** The address a page is fetched at: the cited address without its fragment, which never reaches a server. */
|
|
52
75
|
export declare function pageKey(url: string): string;
|
|
53
76
|
/** An address's origin for a log line, which leaves out the path and the query where secrets can sit. */
|
|
54
77
|
export declare function originOf(url: string): string;
|
|
55
|
-
/**
|
|
78
|
+
/**
|
|
79
|
+
* A promise that rejects with the signal's reason once it fires, after `onAbort` runs; `release` stops listening. It
|
|
80
|
+
* never counts as an unhandled rejection.
|
|
81
|
+
*/
|
|
82
|
+
export declare function whenAborted(signal: AbortSignal, onAbort?: () => void): {
|
|
83
|
+
promise: Promise<never>;
|
|
84
|
+
release: () => void;
|
|
85
|
+
};
|
|
86
|
+
/**
|
|
87
|
+
* A body read up to `cap` bytes: its bytes, or null past the cap with the rest of the stream cancelled. Given a signal,
|
|
88
|
+
* its abort cancels the stream and throws the signal's reason at once, also for a stream that answers neither.
|
|
89
|
+
*/
|
|
90
|
+
export declare function readCapped(response: Response, cap: number, signal?: AbortSignal): Promise<Uint8Array | null>;
|
|
91
|
+
/**
|
|
92
|
+
* Requests through a `fetch`, the body streamed up to the cap, up to five redirects followed one at a time and a sixth
|
|
93
|
+
* the failure `redirects`; given rules, each target is checked before its request. A target that is not http or https
|
|
94
|
+
* is refused (`scheme`). Node's fetch hands back a redirect asked for with `redirect: 'manual'` as it came
|
|
95
|
+
* (https://github.com/nodejs/undici/issues/1193); with no rules, a `fetch` that hides it, as a browser's does, is asked
|
|
96
|
+
* again to follow its redirects itself.
|
|
97
|
+
*/
|
|
56
98
|
export declare function fetchTransport(fetchFn: typeof globalThis.fetch, userAgent?: string): Transport;
|
|
57
99
|
/**
|
|
58
100
|
* Requests through AgentOS's checked egress (`@framers/agentos/safety/egress`, from 0.13.120): each address, and each
|
|
59
101
|
* address a redirect names, is checked against the policy before a byte is sent, its connection pinned to the address
|
|
60
|
-
* checked, and the answer bounded by the deadline and the size cap. Up to five redirects are followed
|
|
102
|
+
* checked, and the answer bounded by the deadline and the size cap. Up to five redirects are followed, a target that is
|
|
103
|
+
* not http or https refused (`scheme`); given rules, each address is checked against them before its request too.
|
|
61
104
|
*
|
|
62
105
|
* @throws {Error} When the installed AgentOS has no checked egress.
|
|
63
106
|
*/
|
|
@@ -65,11 +108,19 @@ export declare function egressTransport(profile: EgressProfile, bounds: {
|
|
|
65
108
|
timeoutMs: number;
|
|
66
109
|
maxBytes: number;
|
|
67
110
|
}, userAgent?: string): Promise<Transport>;
|
|
111
|
+
/** A page's text as blocks, the characters a reader never sees removed. */
|
|
112
|
+
export declare function pageBlocks(kind: 'html' | 'text', text: string, budget: number): string[];
|
|
113
|
+
/** Reads a page the reader's own fetch found no text in, such as one a script draws. Answers the text as blocks, or null. */
|
|
114
|
+
export interface PageFallbackReader {
|
|
115
|
+
read(url: string, options: {
|
|
116
|
+
signal: AbortSignal;
|
|
117
|
+
}): Promise<string[] | null>;
|
|
118
|
+
}
|
|
68
119
|
/** How a page is read. */
|
|
69
120
|
export interface PageReaderOptions {
|
|
70
121
|
/** The request each attempt goes through. */
|
|
71
122
|
transport: Transport;
|
|
72
|
-
/** The fallback for a failure it may mend, or null for none. */
|
|
123
|
+
/** The fallback for a failure it may mend, or null for none; not used when `allowUrl` or `robots` is given. */
|
|
73
124
|
scraper: PageScraper | null;
|
|
74
125
|
/** The deadline of one request, in milliseconds. */
|
|
75
126
|
timeoutMs: number;
|
|
@@ -83,8 +134,26 @@ export interface PageReaderOptions {
|
|
|
83
134
|
now: () => number;
|
|
84
135
|
/** Where the reader writes its events. */
|
|
85
136
|
logger?: CitationLogger;
|
|
137
|
+
/**
|
|
138
|
+
* The host's rule for an address, run on the first address and on every redirect target before its request: false
|
|
139
|
+
* ends the page as `refused` with `detail: 'host'`, the address never requested.
|
|
140
|
+
*/
|
|
141
|
+
allowUrl?: (url: string) => boolean;
|
|
142
|
+
/**
|
|
143
|
+
* robots.txt by RFC 9309, read before the first address and before every redirect target: a disallow ends the page as
|
|
144
|
+
* `refused` with `detail: 'robots'`, the address never requested; a robots.txt the egress policy refuses ends it with
|
|
145
|
+
* the policy's reason, as the same read with no robots rule does. None is read when left out.
|
|
146
|
+
*/
|
|
147
|
+
robots?: RobotsRule;
|
|
148
|
+
/**
|
|
149
|
+
* The reader for a page that answered 200 with no text, called once for the address the reader's own request reached
|
|
150
|
+
* under the rules (every hop checked), with the caller's signal, its blocks held to `textBudget`. With none, or when it
|
|
151
|
+
* answers null, the page is `unreadable` with `no_text`. A fallback that requests the page itself does so outside
|
|
152
|
+
* these rules: the reader asks it only for an address they cleared.
|
|
153
|
+
*/
|
|
154
|
+
fallback?: PageFallbackReader;
|
|
86
155
|
}
|
|
87
|
-
/** A page as the check reads it: its status, and when it was read, its blocks and its sentences as passages. */
|
|
156
|
+
/** A page as the check reads it: its status, and when it was read, its document, its blocks and its sentences as passages. */
|
|
88
157
|
export interface PageRead {
|
|
89
158
|
/** How the page fared. */
|
|
90
159
|
status: PageStatus;
|
|
@@ -92,11 +161,20 @@ export interface PageRead {
|
|
|
92
161
|
blocks: string[];
|
|
93
162
|
/** Its sentences, shaped as AgentOS's `VerificationSource`, in page order; empty when it was not read. */
|
|
94
163
|
passages: CitationSource[];
|
|
164
|
+
/**
|
|
165
|
+
* The page as a document: its address after redirects, its `<title>` cut to 255 characters (the host when it has
|
|
166
|
+
* none), the licence it states (`not read` when the reader finds none), the publisher's name and its own date where
|
|
167
|
+
* it states them, and its blocks; null when it was not read.
|
|
168
|
+
*/
|
|
169
|
+
document: SourceDocument | null;
|
|
170
|
+
/** The page's own date in milliseconds, as on the document; null when it states none or was not read. */
|
|
171
|
+
pageDate: number | null;
|
|
95
172
|
}
|
|
96
173
|
/**
|
|
97
|
-
* Reads one page: one request, then for a
|
|
98
|
-
*
|
|
99
|
-
*
|
|
174
|
+
* Reads one page: the rules checked when given, one request, then for a 200 with no text one read of its address after
|
|
175
|
+
* redirects through the fallback reader when given, and, when no rules are given, for a failure the scraper may mend (a
|
|
176
|
+
* deadline, no connection, an error status other than 404 and 410, a page with no text) one try through the fallback
|
|
177
|
+
* scraper. Answers a status for a page that is not read; throws only the caller's own cancellation.
|
|
100
178
|
*
|
|
101
179
|
* @param url - The address to fetch, from {@link pageKey}.
|
|
102
180
|
* @param options - How to read it.
|
package/dist/page.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"page.d.ts","sourceRoot":"","sources":["../src/page.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"page.d.ts","sourceRoot":"","sources":["../src/page.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,EAAiC,KAAK,cAAc,EAAE,KAAK,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,qCAAqC,CAAC;AAClJ,OAAO,EAAkB,KAAK,WAAW,EAAE,KAAK,YAAY,EAAE,MAAM,kCAAkC,CAAC;AAKvG,OAAO,EAA4B,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AACxE,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1E;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CACnD;AAED,8HAA8H;AAC9H,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE/C,yCAAyC;AACzC,MAAM,WAAW,eAAe;IAC9B,gCAAgC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,0CAA0C;IAC1C,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACpC,wEAAwE;IACxE,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAC9C,oHAAoH;IACpH,UAAU,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAC9C,6CAA6C;IAC7C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,mGAAmG;IACnG,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IACpC,2HAA2H;IAC3H,MAAM,CAAC,EAAE,UAAU,CAAC;CACrB;AAED;;;;GAIG;AACH,MAAM,MAAM,SAAS,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,CAAC,EAAE,cAAc,KAAK,OAAO,CAAC,eAAe,CAAC,CAAC;AAE/G,+EAA+E;AAC/E,qBAAa,WAAY,SAAQ,KAAK;IAMlC,QAAQ,CAAC,OAAO,EAAE,WAAW;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IANhC;;;OAGG;gBAEQ,OAAO,EAAE,WAAW,EACpB,MAAM,GAAE,MAAM,GAAG,IAAW;CAKxC;AAQD,+FAA+F;AAC/F,eAAO,MAAM,cAAc,EAAE,QAAQ,CAAC,aAAa,CAAgD,CAAC;AAEpG,8GAA8G;AAC9G,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAI3C;AAED,yGAAyG;AACzG,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAM5C;AAyCD;;;GAGG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,GAAE,MAAM,IAAsB,GAAG;IAAE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAAC,OAAO,EAAE,MAAM,IAAI,CAAA;CAAE,CAYxI;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAyBlH;AA4CD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,UAAU,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAwB9F;AAwCD;;;;;;;GAOG;AACH,wBAAsB,eAAe,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAwCrJ;AAqFD,2EAA2E;AAC3E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAGxF;AAED,6HAA6H;AAC7H,MAAM,WAAW,kBAAkB;IACjC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE;QAAE,MAAM,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC;CAC/E;AAED,0BAA0B;AAC1B,MAAM,WAAW,iBAAiB;IAChC,6CAA6C;IAC7C,SAAS,EAAE,SAAS,CAAC;IACrB,+GAA+G;IAC/G,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;IAC5B,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,+CAA+C;IAC/C,GAAG,EAAE,MAAM,MAAM,CAAC;IAClB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB;;;OAGG;IACH,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IACpC;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,kBAAkB,CAAC;CAC/B;AAED,8HAA8H;AAC9H,MAAM,WAAW,QAAQ;IACvB,0BAA0B;IAC1B,MAAM,EAAE,UAAU,CAAC;IACnB,kEAAkE;IAClE,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,0GAA0G;IAC1G,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B;;;;OAIG;IACH,QAAQ,EAAE,cAAc,GAAG,IAAI,CAAC;IAChC,yGAAyG;IACzG,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAoLD;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAgCzF"}
|