@framers/agentos-ext-citation-check 0.1.4 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/page.d.ts CHANGED
@@ -1,18 +1,23 @@
1
1
  /**
2
2
  * @file page.ts
3
- * @description One cited page as passages. A request goes out once per page, with a browser-like User-Agent from the
4
- * web scraper pack, a deadline and a size cap, through Node's fetch (redirects followed) or through AgentOS's checked
5
- * egress (every address and every redirect checked against a policy); a page that fails in a way a browser may mend is
6
- * retried once through the web scraper pack's tiers. The answer is read by its content type: HTML and XML through
3
+ * @description One page as a document and passages. A request goes out once per page, with a browser-like User-Agent
4
+ * from the web scraper pack, a deadline and a size cap, through Node's fetch or through AgentOS's checked egress (every
5
+ * address and every redirect checked against a policy), each following up to five redirects. Given rules, the reader checks the first
6
+ * address against the host's rule and the origin's robots.txt before it is requested, and the transport checks every
7
+ * redirect target the same way before its request. The answer is read by its content type: HTML and XML through
7
8
  * public-sources' `htmlToBlocks` in page mode, other text by paragraphs, a PDF or another binary not at all; its
8
- * charset from a byte order mark, the content type or a `<meta>` tag. Its sentences become passages through
9
- * public-sources' `citationSources`, cut by grounding-guard's `sentenceSpans`. A page that fails is a status, never a
10
- * throw; only the caller's own cancellation throws.
9
+ * charset from a byte order mark, the content type or a `<meta>` tag. The document carries the page's title, the
10
+ * licence it states, the publisher's name and the page's own date where the markup gives them. Its sentences become
11
+ * passages through public-sources' `citationSources`, cut by grounding-guard's `sentenceSpans`. A page that answered
12
+ * 200 with no text is read once through the host's fallback reader when one is given; a page that fails in a way a
13
+ * browser may mend is retried once through the web scraper pack's tiers. A page that fails is a status, never a throw;
14
+ * only the caller's own cancellation throws.
11
15
  *
12
16
  * @module agentos/extensions/research/citation-check/page
13
17
  */
14
- import { type CitationSource } from '@framers/agentos-ext-public-sources';
18
+ import { type CitationSource, type SourceDocument, type SourceLicence } from '@framers/agentos-ext-public-sources';
15
19
  import { type ScrapeInput, type ScrapeResult } from '@framers/agentos-ext-web-scraper';
20
+ import { type RobotsRule } from './robots.js';
16
21
  import type { CitationLogger, PageFailure, PageStatus } from './types.js';
17
22
  /**
18
23
  * The fallback a failed page is retried through: the web scraper pack's `WebScraperService`, whose tiers go from a
@@ -33,11 +38,27 @@ export interface TransportAnswer {
33
38
  header(name: string): string | null;
34
39
  /** The body read up to `cap` bytes: its bytes, or null past the cap. */
35
40
  read(cap: number): Promise<Uint8Array | null>;
41
+ /** The body's first `cap` bytes, the rest left unread. Both transports here give it; robots.txt is read with it. */
42
+ readPrefix?(cap: number): Promise<Uint8Array>;
36
43
  /** Releases a body that will not be read. */
37
44
  discard(): Promise<void>;
38
45
  }
39
- /** One GET of an address under a signal. */
40
- export type Transport = (url: string, signal: AbortSignal) => Promise<TransportAnswer>;
46
+ /**
47
+ * The rules a transport applies to the address it is given and to every redirect target, before each request: the
48
+ * host's rule, and the robots.txt of each hop's origin, read through the same transport.
49
+ */
50
+ export interface TransportRules {
51
+ /** The host's rule for an address: false refuses it, a `PageRefusal` with `refused` and `host`. */
52
+ allowUrl?: (url: string) => boolean;
53
+ /** robots.txt as read before each request: a disallow refuses the address, a `PageRefusal` with `refused` and `robots`. */
54
+ robots?: RobotsRule;
55
+ }
56
+ /**
57
+ * One GET of an address under a signal. Given rules, the transport applies them to the address and to every redirect
58
+ * target before its request, and throws {@link PageRefusal} for one they refuse; a transport of the host's own that
59
+ * follows redirects applies them too, or the reader checks the first address alone.
60
+ */
61
+ export type Transport = (url: string, signal: AbortSignal, rules?: TransportRules) => Promise<TransportAnswer>;
41
62
  /** A failure a transport names itself, such as the egress policy's refusal. */
42
63
  export declare class PageRefusal extends Error {
43
64
  readonly failure: PageFailure;
@@ -48,16 +69,38 @@ export declare class PageRefusal extends Error {
48
69
  */
49
70
  constructor(failure: PageFailure, detail?: string | null);
50
71
  }
72
+ /** The licence of a page that states none the reader finds, and of a text that is not HTML. */
73
+ export declare const UNREAD_LICENCE: Readonly<SourceLicence>;
51
74
  /** The address a page is fetched at: the cited address without its fragment, which never reaches a server. */
52
75
  export declare function pageKey(url: string): string;
53
76
  /** An address's origin for a log line, which leaves out the path and the query where secrets can sit. */
54
77
  export declare function originOf(url: string): string;
55
- /** Requests through a `fetch`: redirects followed, the body streamed up to the cap. */
78
+ /**
79
+ * A promise that rejects with the signal's reason once it fires, after `onAbort` runs; `release` stops listening. It
80
+ * never counts as an unhandled rejection.
81
+ */
82
+ export declare function whenAborted(signal: AbortSignal, onAbort?: () => void): {
83
+ promise: Promise<never>;
84
+ release: () => void;
85
+ };
86
+ /**
87
+ * A body read up to `cap` bytes: its bytes, or null past the cap with the rest of the stream cancelled. Given a signal,
88
+ * its abort cancels the stream and throws the signal's reason at once, also for a stream that answers neither.
89
+ */
90
+ export declare function readCapped(response: Response, cap: number, signal?: AbortSignal): Promise<Uint8Array | null>;
91
+ /**
92
+ * Requests through a `fetch`, the body streamed up to the cap, up to five redirects followed one at a time and a sixth
93
+ * the failure `redirects`; given rules, each target is checked before its request. A target that is not http or https
94
+ * is refused (`scheme`). Node's fetch hands back a redirect asked for with `redirect: 'manual'` as it came
95
+ * (https://github.com/nodejs/undici/issues/1193); with no rules, a `fetch` that hides it, as a browser's does, is asked
96
+ * again to follow its redirects itself.
97
+ */
56
98
  export declare function fetchTransport(fetchFn: typeof globalThis.fetch, userAgent?: string): Transport;
57
99
  /**
58
100
  * Requests through AgentOS's checked egress (`@framers/agentos/safety/egress`, from 0.13.120): each address, and each
59
101
  * address a redirect names, is checked against the policy before a byte is sent, its connection pinned to the address
60
- * checked, and the answer bounded by the deadline and the size cap. Up to five redirects are followed.
102
+ * checked, and the answer bounded by the deadline and the size cap. Up to five redirects are followed, a target that is
103
+ * not http or https refused (`scheme`); given rules, each address is checked against them before its request too.
61
104
  *
62
105
  * @throws {Error} When the installed AgentOS has no checked egress.
63
106
  */
@@ -65,11 +108,19 @@ export declare function egressTransport(profile: EgressProfile, bounds: {
65
108
  timeoutMs: number;
66
109
  maxBytes: number;
67
110
  }, userAgent?: string): Promise<Transport>;
111
+ /** A page's text as blocks, the characters a reader never sees removed. */
112
+ export declare function pageBlocks(kind: 'html' | 'text', text: string, budget: number): string[];
113
+ /** Reads a page the reader's own fetch found no text in, such as one a script draws. Answers the text as blocks, or null. */
114
+ export interface PageFallbackReader {
115
+ read(url: string, options: {
116
+ signal: AbortSignal;
117
+ }): Promise<string[] | null>;
118
+ }
68
119
  /** How a page is read. */
69
120
  export interface PageReaderOptions {
70
121
  /** The request each attempt goes through. */
71
122
  transport: Transport;
72
- /** The fallback for a failure it may mend, or null for none. */
123
+ /** The fallback for a failure it may mend, or null for none; not used when `allowUrl` or `robots` is given. */
73
124
  scraper: PageScraper | null;
74
125
  /** The deadline of one request, in milliseconds. */
75
126
  timeoutMs: number;
@@ -83,8 +134,26 @@ export interface PageReaderOptions {
83
134
  now: () => number;
84
135
  /** Where the reader writes its events. */
85
136
  logger?: CitationLogger;
137
+ /**
138
+ * The host's rule for an address, run on the first address and on every redirect target before its request: false
139
+ * ends the page as `refused` with `detail: 'host'`, the address never requested.
140
+ */
141
+ allowUrl?: (url: string) => boolean;
142
+ /**
143
+ * robots.txt by RFC 9309, read before the first address and before every redirect target: a disallow ends the page as
144
+ * `refused` with `detail: 'robots'`, the address never requested; a robots.txt the egress policy refuses ends it with
145
+ * the policy's reason, as the same read with no robots rule does. None is read when left out.
146
+ */
147
+ robots?: RobotsRule;
148
+ /**
149
+ * The reader for a page that answered 200 with no text, called once for the address the reader's own request reached
150
+ * under the rules (every hop checked), with the caller's signal, its blocks held to `textBudget`. With none, or when it
151
+ * answers null, the page is `unreadable` with `no_text`. A fallback that requests the page itself does so outside
152
+ * these rules: the reader asks it only for an address they cleared.
153
+ */
154
+ fallback?: PageFallbackReader;
86
155
  }
87
- /** A page as the check reads it: its status, and when it was read, its blocks and its sentences as passages. */
156
+ /** A page as the check reads it: its status, and when it was read, its document, its blocks and its sentences as passages. */
88
157
  export interface PageRead {
89
158
  /** How the page fared. */
90
159
  status: PageStatus;
@@ -92,11 +161,20 @@ export interface PageRead {
92
161
  blocks: string[];
93
162
  /** Its sentences, shaped as AgentOS's `VerificationSource`, in page order; empty when it was not read. */
94
163
  passages: CitationSource[];
164
+ /**
165
+ * The page as a document: its address after redirects, its `<title>` cut to 255 characters (the host when it has
166
+ * none), the licence it states (`not read` when the reader finds none), the publisher's name and its own date where
167
+ * it states them, and its blocks; null when it was not read.
168
+ */
169
+ document: SourceDocument | null;
170
+ /** The page's own date in milliseconds, as on the document; null when it states none or was not read. */
171
+ pageDate: number | null;
95
172
  }
96
173
  /**
97
- * Reads one page: one request, then for a failure the fallback may mend (a deadline, no connection, an error status
98
- * other than 404 and 410, a page with no text) one try through the fallback scraper. Answers a status for a page that
99
- * is not read; throws only the caller's own cancellation.
174
+ * Reads one page: the rules checked when given, one request, then for a 200 with no text one read of its address after
175
+ * redirects through the fallback reader when given, and, when no rules are given, for a failure the scraper may mend (a
176
+ * deadline, no connection, an error status other than 404 and 410, a page with no text) one try through the fallback
177
+ * scraper. Answers a status for a page that is not read; throws only the caller's own cancellation.
100
178
  *
101
179
  * @param url - The address to fetch, from {@link pageKey}.
102
180
  * @param options - How to read it.
@@ -1 +1 @@
1
- {"version":3,"file":"page.d.ts","sourceRoot":"","sources":["../src/page.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAAiC,KAAK,cAAc,EAAuB,MAAM,qCAAqC,CAAC;AAC9H,OAAO,EAAkB,KAAK,WAAW,EAAE,KAAK,YAAY,EAAE,MAAM,kCAAkC,CAAC;AAGvG,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1E;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CACnD;AAED,8HAA8H;AAC9H,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE/C,yCAAyC;AACzC,MAAM,WAAW,eAAe;IAC9B,gCAAgC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,0CAA0C;IAC1C,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACpC,wEAAwE;IACxE,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAC9C,6CAA6C;IAC7C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,4CAA4C;AAC5C,MAAM,MAAM,SAAS,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,eAAe,CAAC,CAAC;AAEvF,+EAA+E;AAC/E,qBAAa,WAAY,SAAQ,KAAK;IAMlC,QAAQ,CAAC,OAAO,EAAE,WAAW;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IANhC;;;OAGG;gBAEQ,OAAO,EAAE,WAAW,EACpB,MAAM,GAAE,MAAM,GAAG,IAAW;CAKxC;AAWD,8GAA8G;AAC9G,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAI3C;AAED,yGAAyG;AACzG,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAM5C;AA6BD,uFAAuF;AACvF,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,UAAU,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAa9F;AAwCD;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAiCrJ;AAoFD,0BAA0B;AAC1B,MAAM,WAAW,iBAAiB;IAChC,6CAA6C;IAC7C,SAAS,EAAE,SAAS,CAAC;IACrB,gEAAgE;IAChE,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;IAC5B,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,+CAA+C;IAC/C,GAAG,EAAE,MAAM,MAAM,CAAC;IAClB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB;AAED,gHAAgH;AAChH,MAAM,WAAW,QAAQ;IACvB,0BAA0B;IAC1B,MAAM,EAAE,UAAU,CAAC;IACnB,kEAAkE;IAClE,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,0GAA0G;IAC1G,QAAQ,EAAE,cAAc,EAAE,CAAC;CAC5B;AAoGD;;;;;;;GAOG;AACH,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAkBzF"}
1
+ {"version":3,"file":"page.d.ts","sourceRoot":"","sources":["../src/page.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,EAAiC,KAAK,cAAc,EAAE,KAAK,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,qCAAqC,CAAC;AAClJ,OAAO,EAAkB,KAAK,WAAW,EAAE,KAAK,YAAY,EAAE,MAAM,kCAAkC,CAAC;AAKvG,OAAO,EAA4B,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AACxE,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1E;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CACnD;AAED,8HAA8H;AAC9H,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE/C,yCAAyC;AACzC,MAAM,WAAW,eAAe;IAC9B,gCAAgC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,0CAA0C;IAC1C,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACpC,wEAAwE;IACxE,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAC9C,oHAAoH;IACpH,UAAU,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAC9C,6CAA6C;IAC7C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,mGAAmG;IACnG,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IACpC,2HAA2H;IAC3H,MAAM,CAAC,EAAE,UAAU,CAAC;CACrB;AAED;;;;GAIG;AACH,MAAM,MAAM,SAAS,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,CAAC,EAAE,cAAc,KAAK,OAAO,CAAC,eAAe,CAAC,CAAC;AAE/G,+EAA+E;AAC/E,qBAAa,WAAY,SAAQ,KAAK;IAMlC,QAAQ,CAAC,OAAO,EAAE,WAAW;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IANhC;;;OAGG;gBAEQ,OAAO,EAAE,WAAW,EACpB,MAAM,GAAE,MAAM,GAAG,IAAW;CAKxC;AAQD,+FAA+F;AAC/F,eAAO,MAAM,cAAc,EAAE,QAAQ,CAAC,aAAa,CAAgD,CAAC;AAEpG,8GAA8G;AAC9G,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAI3C;AAED,yGAAyG;AACzG,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAM5C;AAyCD;;;GAGG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,GAAE,MAAM,IAAsB,GAAG;IAAE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAAC,OAAO,EAAE,MAAM,IAAI,CAAA;CAAE,CAYxI;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAyBlH;AA4CD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,UAAU,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAwB9F;AAwCD;;;;;;;GAOG;AACH,wBAAsB,eAAe,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAwCrJ;AAqFD,2EAA2E;AAC3E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAGxF;AAED,6HAA6H;AAC7H,MAAM,WAAW,kBAAkB;IACjC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE;QAAE,MAAM,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC;CAC/E;AAED,0BAA0B;AAC1B,MAAM,WAAW,iBAAiB;IAChC,6CAA6C;IAC7C,SAAS,EAAE,SAAS,CAAC;IACrB,+GAA+G;IAC/G,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;IAC5B,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,+CAA+C;IAC/C,GAAG,EAAE,MAAM,MAAM,CAAC;IAClB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB;;;OAGG;IACH,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IACpC;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,kBAAkB,CAAC;CAC/B;AAED,8HAA8H;AAC9H,MAAM,WAAW,QAAQ;IACvB,0BAA0B;IAC1B,MAAM,EAAE,UAAU,CAAC;IACnB,kEAAkE;IAClE,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,0GAA0G;IAC1G,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B;;;;OAIG;IACH,QAAQ,EAAE,cAAc,GAAG,IAAI,CAAC;IAChC,yGAAyG;IACzG,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAoLD;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAgCzF"}