@framers/agentos-ext-citation-check 0.1.4 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +98 -5
- package/dist/firecrawl.d.ts +77 -0
- package/dist/firecrawl.d.ts.map +1 -0
- package/dist/firecrawl.js +184 -0
- package/dist/firecrawl.js.map +1 -0
- package/dist/index.d.ts +9 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -2
- package/dist/index.js.map +1 -1
- package/dist/licence.d.ts +31 -0
- package/dist/licence.d.ts.map +1 -0
- package/dist/licence.js +122 -0
- package/dist/licence.js.map +1 -0
- package/dist/match.d.ts +2 -1
- package/dist/match.d.ts.map +1 -1
- package/dist/match.js +11 -2
- package/dist/match.js.map +1 -1
- package/dist/meta.d.ts +66 -0
- package/dist/meta.d.ts.map +1 -0
- package/dist/meta.js +283 -0
- package/dist/meta.js.map +1 -0
- package/dist/page.d.ts +95 -17
- package/dist/page.d.ts.map +1 -1
- package/dist/page.js +291 -50
- package/dist/page.js.map +1 -1
- package/dist/parse.d.ts +2 -1
- package/dist/parse.d.ts.map +1 -1
- package/dist/parse.js +54 -3
- package/dist/parse.js.map +1 -1
- package/dist/robots.d.ts +90 -0
- package/dist/robots.d.ts.map +1 -0
- package/dist/robots.js +249 -0
- package/dist/robots.js.map +1 -0
- package/dist/types.d.ts +9 -6
- package/dist/types.d.ts.map +1 -1
- package/dist/web-source.d.ts +95 -0
- package/dist/web-source.d.ts.map +1 -0
- package/dist/web-source.js +176 -0
- package/dist/web-source.js.map +1 -0
- package/manifest.json +4 -3
- package/package.json +10 -8
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file web-source.ts
|
|
3
|
+
* @description The open web as a public source. The host gives the search: a phrase in, the addresses of pages that
|
|
4
|
+
* may hold it out, best first, from whatever search service it uses. The provider answers each address as a hit and
|
|
5
|
+
* reads a hit's page through the page reader: under the host's rule for every address and redirect target, given the
|
|
6
|
+
* phrase whose search found the page, and under robots.txt by RFC 9309, through AgentOS's checked egress unless the
|
|
7
|
+
* host gives a transport. The page answers as a document with the host's name without "www." as its site, its own
|
|
8
|
+
* title, the licence it states, the publisher's name and its own date; a page that answered 200 with no text is read
|
|
9
|
+
* once through the host's fallback reader when one is given. A page that is not read is a skip, never a throw; only
|
|
10
|
+
* the caller's own cancellation throws.
|
|
11
|
+
*
|
|
12
|
+
* @module agentos/extensions/research/citation-check/web-source
|
|
13
|
+
*/
|
|
14
|
+
import { type CallOptions, type PublicSourceProvider, type ReadOutcome, type SearchOutcome, type SourceHit } from '@framers/agentos-ext-public-sources';
|
|
15
|
+
import { type EgressProfile, type PageFallbackReader, type Transport } from './page.js';
|
|
16
|
+
import type { RobotsRule } from './robots.js';
|
|
17
|
+
import type { CitationLogger } from './types.js';
|
|
18
|
+
/** robots.txt as the provider reads it when the host names no rule: the `*` group, 2,000 ms, 512,000 bytes, ten minutes kept. */
|
|
19
|
+
export declare const DEFAULT_ROBOTS: Readonly<RobotsRule>;
|
|
20
|
+
/** How the provider searches and reads. */
|
|
21
|
+
export interface WebPageSourceOptions {
|
|
22
|
+
/** The host's search: a short phrase in, page addresses out, best first. A throw answers the search as failed. */
|
|
23
|
+
search: (phrase: string, signal?: AbortSignal) => Promise<readonly string[]>;
|
|
24
|
+
/** The request pages go through; AgentOS's checked egress with `egress` when left out. */
|
|
25
|
+
transport?: Transport;
|
|
26
|
+
/** The checked egress policy when no transport is given: `hosted` (the default), public https addresses alone, or `local`. */
|
|
27
|
+
egress?: EgressProfile;
|
|
28
|
+
/** The User-Agent the checked egress sends when no transport is given; a browser's when left out. */
|
|
29
|
+
userAgent?: string;
|
|
30
|
+
/**
|
|
31
|
+
* The host's rule for each address and each redirect target, with the phrase whose search answered the hit read (the
|
|
32
|
+
* empty string for a hit object this provider's search did not answer, a copy of one included): false refuses it
|
|
33
|
+
* before it is requested.
|
|
34
|
+
*/
|
|
35
|
+
allowUrl?: (url: string, phrase: string) => boolean;
|
|
36
|
+
/** robots.txt as read before each address; {@link DEFAULT_ROBOTS} when left out, none with null. */
|
|
37
|
+
robots?: RobotsRule | null;
|
|
38
|
+
/** The reader for a page that answered 200 with no text. */
|
|
39
|
+
fallback?: PageFallbackReader;
|
|
40
|
+
/** The deadline of one request, the most bytes of a body, the most characters of text: 15,000, 5,000,000 and 300,000. */
|
|
41
|
+
timeoutMs?: number;
|
|
42
|
+
maxBytes?: number;
|
|
43
|
+
textBudget?: number;
|
|
44
|
+
/** The source's name as a reader sees it; `the open web` when left out. */
|
|
45
|
+
site?: string;
|
|
46
|
+
/** The clock of each document's `readAt`; `Date.now` when left out. */
|
|
47
|
+
now?: () => number;
|
|
48
|
+
/** Where the reader writes its events. */
|
|
49
|
+
logger?: CitationLogger;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* A `PublicSourceProvider` over the open web: `search` asks the host's search for addresses, `read` reads a page. It
|
|
53
|
+
* queues nothing: the host reads at most as many pages at once as it calls `read` for.
|
|
54
|
+
*/
|
|
55
|
+
export declare class WebPageSourceProvider implements PublicSourceProvider {
|
|
56
|
+
readonly site: string;
|
|
57
|
+
private readonly searchFn;
|
|
58
|
+
private readonly options;
|
|
59
|
+
private readonly timeoutMs;
|
|
60
|
+
private readonly maxBytes;
|
|
61
|
+
private readonly textBudget;
|
|
62
|
+
private readonly robots;
|
|
63
|
+
private readonly now;
|
|
64
|
+
private transport;
|
|
65
|
+
/** The phrase each hit this provider's search answered was found by, held as long as the hit is. */
|
|
66
|
+
private readonly phrases;
|
|
67
|
+
/**
|
|
68
|
+
* @param options - The host's search and how pages are read.
|
|
69
|
+
* @throws {TypeError} For a search that is not a function, or a bound that is not a whole number of at least 1.
|
|
70
|
+
*/
|
|
71
|
+
constructor(options: WebPageSourceOptions);
|
|
72
|
+
/** The transport, built once: the host's, or AgentOS's checked egress. */
|
|
73
|
+
private transportOf;
|
|
74
|
+
/**
|
|
75
|
+
* Searches for a short phrase: a cancellation that came first is thrown before anything runs, then `onSend` is
|
|
76
|
+
* awaited once and the host's search runs, and its addresses answer
|
|
77
|
+
* as hits (`key` the address without its fragment, `title` the address), each page once, at most `limit` (3 when
|
|
78
|
+
* left out), an address that is not http or https passed over. `limited` when `onSend` throws a limiter's refusal
|
|
79
|
+
* (nothing was sent), `failed` when it throws anything else or the search throws. The caller's own cancellation is
|
|
80
|
+
* thrown.
|
|
81
|
+
*/
|
|
82
|
+
search(phrase: string, options?: CallOptions & {
|
|
83
|
+
limit?: number;
|
|
84
|
+
}): Promise<SearchOutcome>;
|
|
85
|
+
/**
|
|
86
|
+
* Reads a hit's page: `document` with `site` the host without "www.", `revision` null, `readAt` the clock's, and the
|
|
87
|
+
* page's title, licence, publisher and date as read, or `skipped` with the status when one came. The caller's own
|
|
88
|
+
* cancellation is thrown.
|
|
89
|
+
*
|
|
90
|
+
* @throws The caller's cancellation reason; an `Error` when no transport is given and the installed AgentOS has no
|
|
91
|
+
* checked egress.
|
|
92
|
+
*/
|
|
93
|
+
read(hit: SourceHit, options?: Omit<CallOptions, 'onSend'>): Promise<ReadOutcome>;
|
|
94
|
+
}
|
|
95
|
+
//# sourceMappingURL=web-source.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"web-source.d.ts","sourceRoot":"","sources":["../src/web-source.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAEL,KAAK,WAAW,EAChB,KAAK,oBAAoB,EACzB,KAAK,WAAW,EAChB,KAAK,aAAa,EAClB,KAAK,SAAS,EACf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAsC,KAAK,aAAa,EAAE,KAAK,kBAAkB,EAAE,KAAK,SAAS,EAAE,MAAM,WAAW,CAAC;AAC5H,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjD,iIAAiI;AACjI,eAAO,MAAM,cAAc,EAAE,QAAQ,CAAC,UAAU,CAAwF,CAAC;AAEzI,2CAA2C;AAC3C,MAAM,WAAW,oBAAoB;IACnC,kHAAkH;IAClH,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC,CAAC;IAC7E,0FAA0F;IAC1F,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,8HAA8H;IAC9H,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,qGAAqG;IACrG,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;IACpD,oGAAoG;IACpG,MAAM,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;IAC3B,4DAA4D;IAC5D,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAC9B,yHAAyH;IACzH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,2EAA2E;IAC3E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,uEAAuE;IACvE,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB;AAsCD;;;GAGG;AACH,qBAAa,qBAAsB,YAAW,oBAAoB;IAChE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiC;IAC1D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuB;IAC/C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,SAAS,CAAmC;IACpD,oGAAoG;IACpG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoC;IAE5D;;;OAGG;gBACS,OAAO,EAAE,oBAAoB;IAYzC,0EAA0E;IAC1E,OAAO,CAAC,WAAW;IAMnB;;;;;;;OAOG;IACG,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,WAAW,GAAG;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,aAAa,CAAC;IAmCpG;;;;;;;OAOG;IACG,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,GAAE,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAM,GAAG,OAAO,CAAC,WAAW,CAAC;CAuB5F"}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file web-source.ts
|
|
3
|
+
* @description The open web as a public source. The host gives the search: a phrase in, the addresses of pages that
|
|
4
|
+
* may hold it out, best first, from whatever search service it uses. The provider answers each address as a hit and
|
|
5
|
+
* reads a hit's page through the page reader: under the host's rule for every address and redirect target, given the
|
|
6
|
+
* phrase whose search found the page, and under robots.txt by RFC 9309, through AgentOS's checked egress unless the
|
|
7
|
+
* host gives a transport. The page answers as a document with the host's name without "www." as its site, its own
|
|
8
|
+
* title, the licence it states, the publisher's name and its own date; a page that answered 200 with no text is read
|
|
9
|
+
* once through the host's fallback reader when one is given. A page that is not read is a skip, never a throw; only
|
|
10
|
+
* the caller's own cancellation throws.
|
|
11
|
+
*
|
|
12
|
+
* @module agentos/extensions/research/citation-check/web-source
|
|
13
|
+
*/
|
|
14
|
+
import { LimiterRefused, } from '@framers/agentos-ext-public-sources';
|
|
15
|
+
import { egressTransport, pageKey, readPage } from './page.js';
|
|
16
|
+
/** robots.txt as the provider reads it when the host names no rule: the `*` group, 2,000 ms, 512,000 bytes, ten minutes kept. */
|
|
17
|
+
export const DEFAULT_ROBOTS = Object.freeze({ token: '*', timeoutMs: 2_000, maxBytes: 512_000, cacheMs: 600_000 });
|
|
18
|
+
/** The most hits a search answers when the call names no limit. */
|
|
19
|
+
const DEFAULT_LIMIT = 3;
|
|
20
|
+
/** A whole number of at least 1, or the default when left out. */
|
|
21
|
+
function whole(name, value, fallback) {
|
|
22
|
+
if (value === undefined)
|
|
23
|
+
return fallback;
|
|
24
|
+
if (!Number.isInteger(value) || value < 1)
|
|
25
|
+
throw new TypeError(`${name} must be a whole number of at least 1`);
|
|
26
|
+
return value;
|
|
27
|
+
}
|
|
28
|
+
/** Whether an error is a limiter's refusal, from any copy of public-sources. */
|
|
29
|
+
function isLimiterRefusal(error) {
|
|
30
|
+
return error instanceof LimiterRefused || (error instanceof Error && error.name === 'LimiterRefused');
|
|
31
|
+
}
|
|
32
|
+
/** An address as a hit's key: http or https, its fragment removed; null for any other. */
|
|
33
|
+
function keyOf(address) {
|
|
34
|
+
if (typeof address !== 'string')
|
|
35
|
+
return null;
|
|
36
|
+
try {
|
|
37
|
+
const key = pageKey(address.trim());
|
|
38
|
+
const { protocol } = new URL(key);
|
|
39
|
+
return protocol === 'http:' || protocol === 'https:' ? key : null;
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/** A page's host without a leading "www.", in lower case. */
|
|
46
|
+
function siteOf(url) {
|
|
47
|
+
try {
|
|
48
|
+
return new URL(url).hostname.toLowerCase().replace(/^www\./u, '');
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
return url;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* A `PublicSourceProvider` over the open web: `search` asks the host's search for addresses, `read` reads a page. It
|
|
56
|
+
* queues nothing: the host reads at most as many pages at once as it calls `read` for.
|
|
57
|
+
*/
|
|
58
|
+
export class WebPageSourceProvider {
|
|
59
|
+
site;
|
|
60
|
+
searchFn;
|
|
61
|
+
options;
|
|
62
|
+
timeoutMs;
|
|
63
|
+
maxBytes;
|
|
64
|
+
textBudget;
|
|
65
|
+
robots;
|
|
66
|
+
now;
|
|
67
|
+
transport = null;
|
|
68
|
+
/** The phrase each hit this provider's search answered was found by, held as long as the hit is. */
|
|
69
|
+
phrases = new WeakMap();
|
|
70
|
+
/**
|
|
71
|
+
* @param options - The host's search and how pages are read.
|
|
72
|
+
* @throws {TypeError} For a search that is not a function, or a bound that is not a whole number of at least 1.
|
|
73
|
+
*/
|
|
74
|
+
constructor(options) {
|
|
75
|
+
if (typeof options.search !== 'function')
|
|
76
|
+
throw new TypeError('a web page source needs a search function');
|
|
77
|
+
this.options = options;
|
|
78
|
+
this.searchFn = options.search;
|
|
79
|
+
this.site = options.site ?? 'the open web';
|
|
80
|
+
this.timeoutMs = whole('timeoutMs', options.timeoutMs, 15_000);
|
|
81
|
+
this.maxBytes = whole('maxBytes', options.maxBytes, 5_000_000);
|
|
82
|
+
this.textBudget = whole('textBudget', options.textBudget, 300_000);
|
|
83
|
+
this.robots = options.robots === undefined ? DEFAULT_ROBOTS : options.robots;
|
|
84
|
+
this.now = options.now ?? Date.now;
|
|
85
|
+
}
|
|
86
|
+
/** The transport, built once: the host's, or AgentOS's checked egress. */
|
|
87
|
+
transportOf() {
|
|
88
|
+
if (this.options.transport !== undefined)
|
|
89
|
+
return Promise.resolve(this.options.transport);
|
|
90
|
+
this.transport ??= egressTransport(this.options.egress ?? 'hosted', { timeoutMs: this.timeoutMs, maxBytes: this.maxBytes }, this.options.userAgent);
|
|
91
|
+
return this.transport;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Searches for a short phrase: a cancellation that came first is thrown before anything runs, then `onSend` is
|
|
95
|
+
* awaited once and the host's search runs, and its addresses answer
|
|
96
|
+
* as hits (`key` the address without its fragment, `title` the address), each page once, at most `limit` (3 when
|
|
97
|
+
* left out), an address that is not http or https passed over. `limited` when `onSend` throws a limiter's refusal
|
|
98
|
+
* (nothing was sent), `failed` when it throws anything else or the search throws. The caller's own cancellation is
|
|
99
|
+
* thrown.
|
|
100
|
+
*/
|
|
101
|
+
async search(phrase, options = {}) {
|
|
102
|
+
const limit = options.limit ?? DEFAULT_LIMIT;
|
|
103
|
+
// A search cancelled before it starts sends nothing, so nothing is charged for it.
|
|
104
|
+
options.signal?.throwIfAborted();
|
|
105
|
+
if (options.onSend !== undefined) {
|
|
106
|
+
try {
|
|
107
|
+
await options.onSend();
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
if (options.signal?.aborted)
|
|
111
|
+
throw options.signal.reason;
|
|
112
|
+
return isLimiterRefusal(error) ? { kind: 'limited' } : { kind: 'failed', status: null };
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
options.signal?.throwIfAborted();
|
|
116
|
+
let addresses;
|
|
117
|
+
try {
|
|
118
|
+
addresses = await this.searchFn(phrase, options.signal);
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
if (options.signal?.aborted)
|
|
122
|
+
throw options.signal.reason;
|
|
123
|
+
return { kind: 'failed', status: null };
|
|
124
|
+
}
|
|
125
|
+
if (options.signal?.aborted)
|
|
126
|
+
throw options.signal.reason;
|
|
127
|
+
const hits = [];
|
|
128
|
+
const seen = new Set();
|
|
129
|
+
for (const address of Array.isArray(addresses) ? addresses : []) {
|
|
130
|
+
if (hits.length >= limit)
|
|
131
|
+
break;
|
|
132
|
+
const key = keyOf(address);
|
|
133
|
+
if (key === null || seen.has(key))
|
|
134
|
+
continue;
|
|
135
|
+
seen.add(key);
|
|
136
|
+
const hit = { key, title: address.trim() };
|
|
137
|
+
this.phrases.set(hit, phrase);
|
|
138
|
+
hits.push(hit);
|
|
139
|
+
}
|
|
140
|
+
return { kind: 'hits', hits };
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Reads a hit's page: `document` with `site` the host without "www.", `revision` null, `readAt` the clock's, and the
|
|
144
|
+
* page's title, licence, publisher and date as read, or `skipped` with the status when one came. The caller's own
|
|
145
|
+
* cancellation is thrown.
|
|
146
|
+
*
|
|
147
|
+
* @throws The caller's cancellation reason; an `Error` when no transport is given and the installed AgentOS has no
|
|
148
|
+
* checked egress.
|
|
149
|
+
*/
|
|
150
|
+
async read(hit, options = {}) {
|
|
151
|
+
options.signal?.throwIfAborted();
|
|
152
|
+
const url = keyOf(hit.key);
|
|
153
|
+
if (url === null)
|
|
154
|
+
return { kind: 'skipped', status: null };
|
|
155
|
+
const phrase = this.phrases.get(hit) ?? '';
|
|
156
|
+
const { allowUrl, fallback, logger } = this.options;
|
|
157
|
+
const transport = await this.transportOf();
|
|
158
|
+
const page = await readPage(url, {
|
|
159
|
+
transport,
|
|
160
|
+
scraper: null,
|
|
161
|
+
timeoutMs: this.timeoutMs,
|
|
162
|
+
maxBytes: this.maxBytes,
|
|
163
|
+
textBudget: this.textBudget,
|
|
164
|
+
now: this.now,
|
|
165
|
+
...(options.signal !== undefined ? { signal: options.signal } : {}),
|
|
166
|
+
...(logger !== undefined ? { logger } : {}),
|
|
167
|
+
...(allowUrl !== undefined ? { allowUrl: (address) => allowUrl(address, phrase) } : {}),
|
|
168
|
+
...(this.robots !== null ? { robots: this.robots } : {}),
|
|
169
|
+
...(fallback !== undefined ? { fallback } : {}),
|
|
170
|
+
});
|
|
171
|
+
if (page.document === null)
|
|
172
|
+
return { kind: 'skipped', status: page.status.status };
|
|
173
|
+
return { kind: 'document', document: { ...page.document, site: siteOf(page.document.url), revision: null } };
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
//# sourceMappingURL=web-source.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"web-source.js","sourceRoot":"","sources":["../src/web-source.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EACL,cAAc,GAMf,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,QAAQ,EAA+D,MAAM,WAAW,CAAC;AAI5H,iIAAiI;AACjI,MAAM,CAAC,MAAM,cAAc,GAAyB,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;AAkCzI,mEAAmE;AACnE,MAAM,aAAa,GAAG,CAAC,CAAC;AAExB,kEAAkE;AAClE,SAAS,KAAK,CAAC,IAAY,EAAE,KAAyB,EAAE,QAAgB;IACtE,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,uCAAuC,CAAC,CAAC;IAC/G,OAAO,KAAK,CAAC;AACf,CAAC;AAED,gFAAgF;AAChF,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,KAAK,YAAY,cAAc,IAAI,CAAC,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,CAAC,CAAC;AACxG,CAAC;AAED,0FAA0F;AAC1F,SAAS,KAAK,CAAC,OAAgB;IAC7B,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACpC,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,OAAO,QAAQ,KAAK,OAAO,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IACpE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,6DAA6D;AAC7D,SAAS,MAAM,CAAC,GAAW;IACzB,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACpE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,qBAAqB;IACvB,IAAI,CAAS;IACL,QAAQ,CAAiC;IACzC,OAAO,CAAuB;IAC9B,SAAS,CAAS;IAClB,QAAQ,CAAS;IACjB,UAAU,CAAS;IACnB,MAAM,CAAoB;IAC1B,GAAG,CAAe;IAC3B,SAAS,GAA8B,IAAI,CAAC;IACpD,oGAAoG;IACnF,OAAO,GAAG,IAAI,OAAO,EAAqB,CAAC;IAE5D;;;OAGG;IACH,YAAY,OAA6B;QACvC,IAAI,OAAO,OAAO,CAAC,MAAM,KAAK,UAAU;YAAE,MAAM,IAAI,SAAS,CAAC,2CAA2C,CAAC,CAAC;QAC3G,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;QAC/B,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,cAAc,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,WAAW,EAAE,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC/D,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC/D,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACnE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QAC7E,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACrC,CAAC;IAED,0EAA0E;IAClE,WAAW;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACzF,IAAI,CAAC,SAAS,KAAK,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACpJ,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,MAAM,CAAC,MAAc,EAAE,UAA4C,EAAE;QACzE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,aAAa,CAAC;QAC7C,mFAAmF;QACnF,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC;YACzB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;oBAAE,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;gBACzD,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAC1F,CAAC;QACH,CAAC;QACD,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,IAAI,SAA6B,CAAC;QAClC,IAAI,CAAC;YACH,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC1D,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;gBAAE,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;YACzD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC1C,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QACzD,MAAM,IAAI,GAAgB,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YAChE,IAAI,IAAI,CAAC,MAAM,IAAI,KAAK;gBAAE,MAAM;YAChC,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;YAC3B,IAAI,GAAG,KAAK,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAC5C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,MAAM,GAAG,GAAc,EAAE,GAAG,EAAE,KAAK,EAAG,OAAkB,CAAC,IAAI,EAAE,EAAE,CAAC;YAClE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAChC,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,IAAI,CAAC,GAAc,EAAE,UAAuC,EAAE;QAClE,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,GAAG,KAAK,IAAI;YAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC3D,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAC3C,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACpD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE;YAC/B,SAAS;YACT,OAAO,EAAE,IAAI;YACb,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3C,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,OAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/F,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChD,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,QAAQ,KAAK,IAAI;YAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnF,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;IAC/G,CAAC;CACF"}
|
package/manifest.json
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
"$schema": "https://agentos.sh/schemas/extension-manifest-v1.json",
|
|
3
3
|
"id": "com.framers.research.citation-check",
|
|
4
4
|
"name": "Citation Check",
|
|
5
|
-
"version": "0.1
|
|
6
|
-
"description": "Checks the quoted claims of a document against the pages they cite, with AgentOS's citation tooling: word for word first, then by embeddings when a model is configured",
|
|
5
|
+
"version": "0.2.1",
|
|
6
|
+
"description": "Checks the quoted claims of a document against the pages they cite, with AgentOS's citation tooling: word for word first, then by embeddings when a model is configured. Reads public pages under robots.txt and a host's rules, and serves the open web as a public source",
|
|
7
7
|
"author": {
|
|
8
8
|
"name": "Frame",
|
|
9
9
|
"email": "team@frame.dev",
|
|
@@ -14,7 +14,8 @@
|
|
|
14
14
|
"citations",
|
|
15
15
|
"fact-checking",
|
|
16
16
|
"grounding",
|
|
17
|
-
"research"
|
|
17
|
+
"research",
|
|
18
|
+
"robots-txt"
|
|
18
19
|
],
|
|
19
20
|
"agentosVersion": ">=0.13.120",
|
|
20
21
|
"categories": ["research"],
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@framers/agentos-ext-citation-check",
|
|
3
|
-
"version": "0.1
|
|
4
|
-
"description": "Checks the quoted claims of a document against the pages they cite, with AgentOS's citation tooling: word for word first, then by embeddings when a model is configured",
|
|
3
|
+
"version": "0.2.1",
|
|
4
|
+
"description": "Checks the quoted claims of a document against the pages they cite, with AgentOS's citation tooling: word for word first, then by embeddings when a model is configured. Reads public pages under robots.txt and a host's rules, and serves the open web as a public source",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
7
7
|
"type": "module",
|
|
@@ -22,7 +22,8 @@
|
|
|
22
22
|
"citations",
|
|
23
23
|
"fact-checking",
|
|
24
24
|
"grounding",
|
|
25
|
-
"research"
|
|
25
|
+
"research",
|
|
26
|
+
"robots-txt"
|
|
26
27
|
],
|
|
27
28
|
"author": {
|
|
28
29
|
"name": "Frame",
|
|
@@ -53,15 +54,16 @@
|
|
|
53
54
|
},
|
|
54
55
|
"dependencies": {
|
|
55
56
|
"@framers/agentos-ext-grounding-guard": "^0.11.0",
|
|
56
|
-
"@framers/agentos-ext-public-sources": "^0.2.
|
|
57
|
-
"@framers/agentos-ext-web-scraper": "^0.1.4"
|
|
57
|
+
"@framers/agentos-ext-public-sources": "^0.2.3",
|
|
58
|
+
"@framers/agentos-ext-web-scraper": "^0.1.4",
|
|
59
|
+
"entities": "^8.1.0"
|
|
58
60
|
},
|
|
59
61
|
"devDependencies": {
|
|
60
62
|
"@types/node": "^20.12.12",
|
|
61
63
|
"rimraf": "^6.1.3",
|
|
62
64
|
"typescript": "^6.0.3",
|
|
63
|
-
"vitest": "^
|
|
64
|
-
"@framers/agentos": "^0.
|
|
65
|
+
"vitest": "^5.1.0",
|
|
66
|
+
"@framers/agentos": "^0.17.0"
|
|
65
67
|
},
|
|
66
68
|
"homepage": "https://github.com/framerslab/agentos-extensions/tree/master/registry/curated/research/citation-check#readme",
|
|
67
69
|
"bugs": {
|
|
@@ -72,7 +74,7 @@
|
|
|
72
74
|
"manifest.json",
|
|
73
75
|
"README.md"
|
|
74
76
|
],
|
|
75
|
-
"gitHead": "
|
|
77
|
+
"gitHead": "de5818a684eaff8563d424923b4120d877f703fa",
|
|
76
78
|
"scripts": {
|
|
77
79
|
"build": "tsc",
|
|
78
80
|
"test": "vitest run",
|