@framers/agentos-ext-citation-check 0.0.0-stage → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +98 -0
- package/README.md +167 -2
- package/dist/bin.d.ts +10 -0
- package/dist/bin.d.ts.map +1 -0
- package/dist/bin.js +11 -0
- package/dist/bin.js.map +1 -0
- package/dist/cli.d.ts +36 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +145 -0
- package/dist/cli.js.map +1 -0
- package/dist/embeddings.d.ts +54 -0
- package/dist/embeddings.d.ts.map +1 -0
- package/dist/embeddings.js +52 -0
- package/dist/embeddings.js.map +1 -0
- package/dist/index.d.ts +55 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +63 -0
- package/dist/index.js.map +1 -0
- package/dist/match.d.ts +44 -0
- package/dist/match.d.ts.map +1 -0
- package/dist/match.js +108 -0
- package/dist/match.js.map +1 -0
- package/dist/page.d.ts +105 -0
- package/dist/page.d.ts.map +1 -0
- package/dist/page.js +400 -0
- package/dist/page.js.map +1 -0
- package/dist/parse.d.ts +36 -0
- package/dist/parse.d.ts.map +1 -0
- package/dist/parse.js +314 -0
- package/dist/parse.js.map +1 -0
- package/dist/report.d.ts +28 -0
- package/dist/report.d.ts.map +1 -0
- package/dist/report.js +128 -0
- package/dist/report.js.map +1 -0
- package/dist/runner.d.ts +89 -0
- package/dist/runner.d.ts.map +1 -0
- package/dist/runner.js +298 -0
- package/dist/runner.js.map +1 -0
- package/dist/tools/citationVerify.d.ts +59 -0
- package/dist/tools/citationVerify.d.ts.map +1 -0
- package/dist/tools/citationVerify.js +98 -0
- package/dist/tools/citationVerify.js.map +1 -0
- package/dist/types.d.ts +150 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +20 -0
- package/dist/types.js.map +1 -0
- package/manifest.json +66 -0
- package/package.json +79 -3
package/LICENSE
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
|
|
10
|
+
|
|
11
|
+
"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
|
|
12
|
+
|
|
13
|
+
"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
|
|
14
|
+
|
|
15
|
+
"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
|
|
16
|
+
|
|
17
|
+
"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
|
|
18
|
+
|
|
19
|
+
"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
|
|
20
|
+
|
|
21
|
+
"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
|
|
22
|
+
|
|
23
|
+
"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
|
|
24
|
+
|
|
25
|
+
"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
|
|
26
|
+
|
|
27
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
|
|
28
|
+
|
|
29
|
+
2. Grant of Copyright License.
|
|
30
|
+
|
|
31
|
+
Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
|
|
32
|
+
|
|
33
|
+
3. Grant of Patent License.
|
|
34
|
+
|
|
35
|
+
Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
|
|
36
|
+
|
|
37
|
+
4. Redistribution.
|
|
38
|
+
|
|
39
|
+
You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
|
|
40
|
+
|
|
41
|
+
(a) You must give any other recipients of the Work or Derivative Works a copy of this License; and
|
|
42
|
+
|
|
43
|
+
(b) You must cause any modified files to carry prominent notices stating that You changed the files; and
|
|
44
|
+
|
|
45
|
+
(c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
|
|
46
|
+
|
|
47
|
+
(d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.
|
|
48
|
+
|
|
49
|
+
You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
|
|
50
|
+
|
|
51
|
+
5. Submission of Contributions.
|
|
52
|
+
|
|
53
|
+
Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
|
|
54
|
+
|
|
55
|
+
6. Trademarks.
|
|
56
|
+
|
|
57
|
+
This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
|
|
58
|
+
|
|
59
|
+
7. Disclaimer of Warranty.
|
|
60
|
+
|
|
61
|
+
Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
|
|
62
|
+
|
|
63
|
+
8. Limitation of Liability.
|
|
64
|
+
|
|
65
|
+
In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
|
|
66
|
+
|
|
67
|
+
9. Accepting Warranty or Additional Liability.
|
|
68
|
+
|
|
69
|
+
While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
|
|
70
|
+
|
|
71
|
+
END OF TERMS AND CONDITIONS
|
|
72
|
+
|
|
73
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
74
|
+
|
|
75
|
+
To apply the Apache License to your work, attach the following
|
|
76
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
77
|
+
replaced with your own identifying information. (Don't include
|
|
78
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
79
|
+
comment syntax for the file format. We also recommend that a
|
|
80
|
+
file or class name and description of purpose be included on the
|
|
81
|
+
same "printed page" as the copyright notice for easier
|
|
82
|
+
identification within third-party archives.
|
|
83
|
+
|
|
84
|
+
Copyright (c) 2025 Framers
|
|
85
|
+
|
|
86
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
87
|
+
you may not use this file except in compliance with the License.
|
|
88
|
+
You may obtain a copy of the License at
|
|
89
|
+
|
|
90
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
91
|
+
|
|
92
|
+
Unless required by applicable law or agreed to in writing, software
|
|
93
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
94
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
95
|
+
See the License for the specific language governing permissions and
|
|
96
|
+
limitations under the License.
|
|
97
|
+
|
|
98
|
+
|
package/README.md
CHANGED
|
@@ -1,3 +1,168 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @framers/agentos-ext-citation-check
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Checks the quoted claims of a document against the pages they cite, with AgentOS's own citation tooling. Each page is fetched once and no search service is called. A claim whose words stand on its page is `exact`. A claim that is not is graded by AgentOS's `CitationVerifier` against the page's sentences when an embedding function is configured, and is `not_found` otherwise, shown with the sentences nearest it. Nothing that costs money runs unless you pass it in.
|
|
4
|
+
|
|
5
|
+
With `research.md`:
|
|
6
|
+
|
|
7
|
+
```markdown
|
|
8
|
+
The report says "The harbour opened in 1887" (https://example.org/harbour, read 1 Mar 2026).
|
|
9
|
+
|
|
10
|
+
A guide notes that "the lighthouse stands 40 metres tall" (<https://example.org/lighthouse>).
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
npx --package @framers/agentos-ext-citation-check agentos-citations verify research.md
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
prints, when the harbour page holds the first quotation word for word and the guide's page says the lighthouse stands 31 metres tall:
|
|
18
|
+
|
|
19
|
+
```text
|
|
20
|
+
Checked 2 claims against 2 pages on 2026-03-01T09:00:00.000Z: each claim's words were looked for on its page, and each claim not found there is shown with the page's sentences nearest it by shared words.
|
|
21
|
+
Verdicts: 1 exact, 1 not found. Result: fails.
|
|
22
|
+
|
|
23
|
+
| Line | Verdict | Claim | Source | What the page says |
|
|
24
|
+
| --- | --- | --- | --- | --- |
|
|
25
|
+
| 1 | exact | The harbour opened in 1887 | <https://example.org/harbour> | The harbour opened in 1887 and handled 4,200 ships in its first year. |
|
|
26
|
+
| 3 | not found | the lighthouse stands 40 metres tall | <https://example.org/lighthouse> | nearest (0.67): Its lighthouse stands 31 metres tall. |
|
|
27
|
+
|
|
28
|
+
Nearest passages for the claims not found:
|
|
29
|
+
|
|
30
|
+
- Line 3, "the lighthouse stands 40 metres tall":
|
|
31
|
+
1. (0.67) Its lighthouse stands 31 metres tall.
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
and exits 1. The output is the markdown for a document's "Citation check log" section; it carries no heading of its own.
|
|
35
|
+
|
|
36
|
+
## Installation
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
npm install @framers/agentos-ext-citation-check
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
It needs Node.js 22 or later, as `@framers/agentos` does. It reads pages with [public-sources](https://www.npmjs.com/package/@framers/agentos-ext-public-sources) (`htmlToBlocks` in page mode and `citationSources`), the [web scraper](https://www.npmjs.com/package/@framers/agentos-ext-web-scraper) (a browser's request headers and the fallback tiers) and the `values` entry of [grounding-guard](https://www.npmjs.com/package/@framers/agentos-ext-grounding-guard) (the sentence rule and the word rules). `@framers/agentos` 0.13.120 or later, the first release with the checked egress the tool reads pages through, is a peer, loaded only to grade with embeddings or to read pages through checked egress. Grounding-guard lists `@huggingface/transformers` as an optional dependency for its NLI model; `npm install --omit=optional` leaves it out when you do not use NLI.
|
|
43
|
+
|
|
44
|
+
## Input
|
|
45
|
+
|
|
46
|
+
A markdown document: each quotation (`"…"`, `“…”` or `„…“`) and the first http or https address after it in its sentence, in parentheses, in a markdown link, between angle brackets or bare, on the same line or a later line of the paragraph.
|
|
47
|
+
|
|
48
|
+
```markdown
|
|
49
|
+
The report says "The harbour opened in 1887" (https://example.org/harbour, read 1 Mar 2026).
|
|
50
|
+
|
|
51
|
+
A guide notes that "the lighthouse stands 40 metres tall" (<https://example.org/lighthouse>).
|
|
52
|
+
|
|
53
|
+
The page lists "Five berths", and "Two cranes lift the containers" (https://example.org/port).
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Several quotations waiting in a sentence share the address that follows them, as the last line's two do. An address ends at white space or a quotation mark, without the punctuation after it and without a closing bracket it did not open, so `(https://example.org/wiki/River_(Delta), read 1 Mar 2026)` gives `https://example.org/wiki/River_(Delta)`. A quotation of four words or more that no address follows before its sentence ends is reported as uncited; a shorter one reads as a term or a title and is passed over. Quotations in fenced code and code spans, and escaped quotation marks (`\"`), are not claims.
|
|
57
|
+
|
|
58
|
+
A JSON document: an array of claims.
|
|
59
|
+
|
|
60
|
+
```json
|
|
61
|
+
[
|
|
62
|
+
{ "claim": "The harbour opened in 1887", "url": "https://example.org/harbour", "exact": true },
|
|
63
|
+
{ "claim": "Roughly four thousand vessels called in the first year", "url": "https://example.org/harbour", "exact": false },
|
|
64
|
+
{ "claim": "The tide rises twice each day" }
|
|
65
|
+
]
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
`url` left out or empty makes the claim uncited, and `exact`, true when left out, records whether the claim gives itself as the page's own words; the check reads every claim the same way, word for word first. An entry with no claim, a `url` that is not http or https, or an `exact` that is not a boolean is refused with the entry's number and line. `parseCitedClaims(text)` reads a document that is a JSON array as JSON and any other as markdown; `{ format: 'markdown' }` or `{ format: 'json' }` names the form.
|
|
69
|
+
|
|
70
|
+
## Verdicts
|
|
71
|
+
|
|
72
|
+
| Verdict | When |
|
|
73
|
+
| --- | --- |
|
|
74
|
+
| `exact` | The claim's words stand on the page |
|
|
75
|
+
| `supported` | `CitationVerifier` found a sentence at or above the support threshold (0.6 by default) |
|
|
76
|
+
| `weak` | The best sentence lies between the two thresholds |
|
|
77
|
+
| `unverifiable` | No sentence reaches the unverifiable threshold (0.3 by default) |
|
|
78
|
+
| `contradicted` | A `supported` claim the NLI function reads as contradicted by its sentence, with a score over 0.7 |
|
|
79
|
+
| `not_found` | The words are not on the page and no embedding function graded the claim |
|
|
80
|
+
| `unreachable` | The page could not be fetched: no answer by the deadline, no connection, a status that is not 2xx, an address the egress policy refuses, or too many redirects |
|
|
81
|
+
| `unreadable` | The page came but holds no text this check reads: a PDF, another binary type, a body past the size cap, or no text a reader sees |
|
|
82
|
+
| `uncited` | The claim cites no address |
|
|
83
|
+
|
|
84
|
+
Words are compared after both sides are read the same way: compatibility forms (NFKC), typographic quotation marks as plain ones, every dash a hyphen with no space around it, case, white space, and soft hyphens and zero-width characters. An ellipsis or words in square brackets (`[…]`, `[It]`) in a quotation are gaps: the pieces between them must stand on the page in order, all in one block of its text (a paragraph, a list item, a table cell, a caption or a quotation), so a sentence of one paragraph and a cell further down the page never pass as one quotation. A quotation without gaps may run on from one block into the next, as across a line break. A match never cuts a word or a number, so "rose to 5" does not stand in "rose to 50 percent". Each check carries the sentence its verdict rests on (for an exact quotation that runs past one sentence, its block), and a `not_found` claim the three sentences nearest it by the share of content words and numbers the two have in common.
|
|
85
|
+
|
|
86
|
+
`citationReportFails(report)` is true when a claim is `contradicted`, `not_found` or `unverifiable`, or when its page is `unreachable` or `unreadable`; `{ allowUnreachable: true }` leaves the last two out.
|
|
87
|
+
|
|
88
|
+
## The command
|
|
89
|
+
|
|
90
|
+
```text
|
|
91
|
+
agentos-citations verify <file> [--json] [--embeddings none|openai] [--format auto|markdown|json]
|
|
92
|
+
[--allow-unreachable] [--no-fallback] [--timeout <ms>]
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
With the pack installed, `npx agentos-citations verify <file>` runs the command; without an install, `npx --package @framers/agentos-ext-citation-check agentos-citations verify <file>` runs it, the package named so that npx does not look for a package called `agentos-citations`. `<file>` is a path, or `-` for standard input. The command prints the markdown log, or the report as JSON with `--json`. It exits 0 when the check passes, 1 when it fails, and 2 for a usage or input error. `--embeddings openai` grades the claims not found word for word with OpenAI embeddings through AgentOS's `embedText`, with `OPENAI_API_KEY` from the environment and `OPENAI_BASE_URL` when it is set; without the key it exits 2 before fetching anything. `--no-fallback` leaves out the web scraper's retry.
|
|
96
|
+
|
|
97
|
+
## In code
|
|
98
|
+
|
|
99
|
+
```ts
|
|
100
|
+
import { checkCitations, citationReportFails, parseCitedClaims, renderCitationLog } from '@framers/agentos-ext-citation-check';
|
|
101
|
+
|
|
102
|
+
const claims = parseCitedClaims(markdown);
|
|
103
|
+
const report = await checkCitations(claims, { timeoutMs: 10_000 });
|
|
104
|
+
const log = renderCitationLog(report); // the "Citation check log" section's body
|
|
105
|
+
if (citationReportFails(report)) process.exitCode = 1;
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
`checkCitationDocument(text, options)` parses and checks in one call. The report holds one check per claim, in the document's order (`line`, `claim`, `url`, `exact`, `verdict`, `grader`, `score`, `passage`, `nearest`, `note`, and `page`, the status of the page it cites, null for an uncited claim), one status per distinct page (`outcome`, `finalUrl`, `status`, `failure`, `detail`, `via`, `passages`), and a summary with the count of each verdict, the uncited claims and the pages that were not read.
|
|
109
|
+
|
|
110
|
+
### Reading pages
|
|
111
|
+
|
|
112
|
+
Each distinct address is fetched once, its fragment removed, four at a time. A request sends a browser's headers from the web scraper pack, follows redirects, and stops at the deadline (`timeoutMs`, 15,000) or past the size cap (`maxBytes`, 5,000,000). HTML and XML are read with public-sources' `htmlToBlocks` in page mode, which keeps the tables, figures, quotations and code a page shows; other text by paragraphs; a PDF or another binary type not at all. The charset comes from a byte order mark, the content type or a `<meta>` tag. A page that times out, does not connect, answers an error status other than 404 and 410, or holds no text is retried once through the web scraper's `WebScraperService`, whose tiers run a headless browser when `playwright-core` is installed and a second plain request otherwise. A page that fails is a verdict, never an exception.
|
|
113
|
+
|
|
114
|
+
| Option | Default | What it does |
|
|
115
|
+
| --- | --- | --- |
|
|
116
|
+
| `fetch` | the global `fetch` | The requests go through it; pass one that applies your egress rules |
|
|
117
|
+
| `egress` | none | `hosted` or `local`: read pages through AgentOS's checked egress instead (`@framers/agentos/safety/egress`, from 0.13.120) |
|
|
118
|
+
| `fallback` | on unless `fetch` or `egress` is given | `false`, `true`, or a scraper with `scrape(input)` |
|
|
119
|
+
| `timeoutMs`, `maxBytes`, `textBudget` | 15,000, 5,000,000, 300,000 | One request's deadline, a body's bytes, a page's characters of text |
|
|
120
|
+
| `concurrency`, `maxPages` | 4, 100 | Pages read at once; distinct pages per check, more refused before any is fetched |
|
|
121
|
+
| `logger` | none | AgentOS's `ILogger`: the events `citation_check.page_unreachable`, `citation_check.page_unreadable`, `citation_check.page_read_by_fallback` and `citation_check.embedding_failed`, with the page's origin and never a claim |
|
|
122
|
+
| `signal` | none | Cancels the check, which throws the signal's reason |
|
|
123
|
+
|
|
124
|
+
The global `fetch` reaches any address a document names. A host that checks documents from people it does not trust passes `egress: 'hosted'`, which reaches public https addresses on port 443 alone and checks every redirect, or a `fetch` of its own. The fallback's requests go through neither, so it is off once either is given unless `fallback: true` turns it on.
|
|
125
|
+
|
|
126
|
+
### Grading with embeddings
|
|
127
|
+
|
|
128
|
+
```ts
|
|
129
|
+
import { checkCitations, managerEmbeddings, openAIEmbeddings } from '@framers/agentos-ext-citation-check';
|
|
130
|
+
|
|
131
|
+
// OpenAI through AgentOS's embedText; billed by OpenAI.
|
|
132
|
+
await checkCitations(claims, { embedFn: openAIEmbeddings({ apiKey: process.env.OPENAI_API_KEY }) });
|
|
133
|
+
|
|
134
|
+
// An embedding manager the host already runs, such as AgentOS's EmbeddingManager.
|
|
135
|
+
await checkCitations(claims, { embedFn: managerEmbeddings(embeddingManager, { modelId: 'text-embedding-3-small' }) });
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
The claims of one page that are not on it word for word are graded in one `CitationVerifier.verify` call against the page's sentences (`maxPassages`, 2,000, in page order), which embeds them in one batch. `supportThreshold` and `unverifiableThreshold` pass through to the verifier. An `nliFn`, such as grounding-guard's `createNliFunction()`, lets the verifier turn a `supported` claim its sentence contradicts into `contradicted`; it downloads and runs its model locally. When the embedding function fails, the page's claims are graded by shared words, and each check's `note` says why.
|
|
139
|
+
|
|
140
|
+
## The tool
|
|
141
|
+
|
|
142
|
+
The pack's one tool, `citation_verify`, takes `{ document, format?, allowUnreachable? }` and answers `{ fails, log, report }`. A document the parser cannot read, or arguments of the wrong type, answer `success: false` with the reason.
|
|
143
|
+
|
|
144
|
+
```ts
|
|
145
|
+
import { createExtensionPack } from '@framers/agentos-ext-citation-check';
|
|
146
|
+
|
|
147
|
+
const pack = createExtensionPack({ options: { embeddings: 'openai' }, getSecret: (id) => secrets[id] });
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
The tool reads pages through AgentOS's checked egress with the hosted policy, so a document cannot point it at a private address; it needs AgentOS 0.13.120 or later. Pack options: `embeddings` (`none` or `openai`, the key from the `openai.apiKey` secret or `OPENAI_API_KEY`), `embeddingModel`, `egress` (`hosted` or `local`), and from code `fetch`, `embedFn` and `nliFn`, with `timeoutMs`, `maxBytes`, `maxPages` and `priority`.
|
|
151
|
+
|
|
152
|
+
## API
|
|
153
|
+
|
|
154
|
+
| Export | What it is |
|
|
155
|
+
| --- | --- |
|
|
156
|
+
| `parseCitedClaims(text, { format })`, `CitationInputError` | The claims of a markdown or JSON document, with their lines; the error for a document it cannot read |
|
|
157
|
+
| `checkCitations(claims, options)`, `checkCitationDocument(text, options)`, `CitationSetupError` | The check, and the error for settings it cannot run with |
|
|
158
|
+
| `renderCitationLog(report, { allowUnreachable })`, `citationReportFails(report, { allowUnreachable })`, `FAILING_VERDICTS` | The markdown log and the result |
|
|
159
|
+
| `openAIEmbeddings(options)`, `managerEmbeddings(manager, request)` | Embedding functions from AgentOS's `embedText` and from an embedding manager |
|
|
160
|
+
| `normalizeForMatch`, `quoteFragments`, `locateQuote`, `lexicalNearest`, `withoutInvisible` | The classical checks the runner uses |
|
|
161
|
+
| `CitationVerifyTool`, `createExtensionPack` | The `citation_verify` tool and the pack |
|
|
162
|
+
| `runCitationCli(argv, io)`, `CITATION_CLI_USAGE` | The command, for a host that runs it in its own process |
|
|
163
|
+
| `CITATION_VERDICTS`, `pageKey`, `isHttpUrl` | The verdicts in order, a page's address without its fragment, the address check |
|
|
164
|
+
| `CitedClaim`, `CitationCheck`, `CitationVerdict`, `CitationPassage`, `PageStatus`, `PageFailure`, `CitationReport`, `CitationSummary`, `CitationCheckOptions`, `CitationLogger`, `EmbedFn`, `NliFn` and the rest | The types |
|
|
165
|
+
|
|
166
|
+
## License
|
|
167
|
+
|
|
168
|
+
Apache-2.0
|
package/dist/bin.d.ts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* @file bin.ts
|
|
4
|
+
* @description The `agentos-citations` executable: runs the command with the process's arguments, input, output and
|
|
5
|
+
* environment, and exits with its status.
|
|
6
|
+
*
|
|
7
|
+
* @module agentos/extensions/research/citation-check/bin
|
|
8
|
+
*/
|
|
9
|
+
export {};
|
|
10
|
+
//# sourceMappingURL=bin.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bin.d.ts","sourceRoot":"","sources":["../src/bin.ts"],"names":[],"mappings":";AACA;;;;;;GAMG"}
|
package/dist/bin.js
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* @file bin.ts
|
|
4
|
+
* @description The `agentos-citations` executable: runs the command with the process's arguments, input, output and
|
|
5
|
+
* environment, and exits with its status.
|
|
6
|
+
*
|
|
7
|
+
* @module agentos/extensions/research/citation-check/bin
|
|
8
|
+
*/
|
|
9
|
+
import { runCitationCli } from './cli.js';
|
|
10
|
+
process.exitCode = await runCitationCli(process.argv.slice(2));
|
|
11
|
+
//# sourceMappingURL=bin.js.map
|
package/dist/bin.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bin.js","sourceRoot":"","sources":["../src/bin.ts"],"names":[],"mappings":";AACA;;;;;;GAMG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE1C,OAAO,CAAC,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC"}
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file cli.ts
|
|
3
|
+
* @description The `agentos-citations` command: `verify <file>` checks a document's cited claims and prints the
|
|
4
|
+
* markdown log, or the report as JSON with `--json`; its exit status is 0 when the check passes, 1 when it fails and 2
|
|
5
|
+
* for a usage or input error. `--embeddings openai` grades the claims not found word for word with OpenAI embeddings
|
|
6
|
+
* through AgentOS, with `OPENAI_API_KEY` (and `OPENAI_BASE_URL` when set) from the environment.
|
|
7
|
+
*
|
|
8
|
+
* @module agentos/extensions/research/citation-check/cli
|
|
9
|
+
*/
|
|
10
|
+
import { type CitationCheckOptions } from './runner.js';
|
|
11
|
+
/** The command's help. */
|
|
12
|
+
export declare const CITATION_CLI_USAGE = "Usage: agentos-citations verify <file> [options]\n\nChecks each quoted claim in <file> against the page its address names, and prints the\nmarkdown for a \"Citation check log\" section. <file> is markdown, in which each quotation\nis followed by its address (\"the quoted words\" (https://example.org/page)), or a JSON\narray of {\"claim\", \"url\", \"exact\"}; - reads standard input.\n\nOptions:\n --json print the report as JSON instead of the markdown log\n --embeddings <name> none (the default) or openai: grade the claims not found word for\n word with OpenAI embeddings through AgentOS (OPENAI_API_KEY, and\n OPENAI_BASE_URL when set)\n --format <format> auto (the default), markdown or json\n --allow-unreachable do not fail on a page that could not be fetched or read\n --no-fallback do not retry a failed page through the web scraper's tiers\n --timeout <ms> the deadline for each page, in milliseconds (default 15000)\n -h, --help print this help\n\nExit status: 0 when every claim passes; 1 when a claim is contradicted, not found or\nunverifiable, or its page unreachable or unreadable; 2 for a usage or input error.";
|
|
13
|
+
/** Where the command reads and writes, which a test or a host replaces. */
|
|
14
|
+
export interface CitationCliIO {
|
|
15
|
+
/** Standard output; the process's when left out. */
|
|
16
|
+
stdout?: (text: string) => void;
|
|
17
|
+
/** Standard error; the process's when left out. */
|
|
18
|
+
stderr?: (text: string) => void;
|
|
19
|
+
/** The environment the keys are read from; the process's when left out. */
|
|
20
|
+
env?: Record<string, string | undefined>;
|
|
21
|
+
/** Reads a file; `fs.readFile` as UTF-8 when left out. */
|
|
22
|
+
readFile?: (path: string) => Promise<string>;
|
|
23
|
+
/** Reads standard input; the process's when left out. */
|
|
24
|
+
readStdin?: () => Promise<string>;
|
|
25
|
+
/** Options the check starts from; the command's own flags override them. */
|
|
26
|
+
options?: CitationCheckOptions;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Runs the command.
|
|
30
|
+
*
|
|
31
|
+
* @param argv - The arguments after the command's name.
|
|
32
|
+
* @param io - Where it reads and writes.
|
|
33
|
+
* @returns The exit status: 0 when the check passes, 1 when it fails, 2 for a usage or input error.
|
|
34
|
+
*/
|
|
35
|
+
export declare function runCitationCli(argv: readonly string[], io?: CitationCliIO): Promise<number>;
|
|
36
|
+
//# sourceMappingURL=cli.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAOH,OAAO,EAA6C,KAAK,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAGnG,0BAA0B;AAC1B,eAAO,MAAM,kBAAkB,qtCAmBoD,CAAC;AAEpF,2EAA2E;AAC3E,MAAM,WAAW,aAAa;IAC5B,oDAAoD;IACpD,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,mDAAmD;IACnD,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,2EAA2E;IAC3E,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC;IACzC,0DAA0D;IAC1D,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAClC,4EAA4E;IAC5E,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAcD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,EAAE,GAAE,aAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,CAmErG"}
|
package/dist/cli.js
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file cli.ts
|
|
3
|
+
* @description The `agentos-citations` command: `verify <file>` checks a document's cited claims and prints the
|
|
4
|
+
* markdown log, or the report as JSON with `--json`; its exit status is 0 when the check passes, 1 when it fails and 2
|
|
5
|
+
* for a usage or input error. `--embeddings openai` grades the claims not found word for word with OpenAI embeddings
|
|
6
|
+
* through AgentOS, with `OPENAI_API_KEY` (and `OPENAI_BASE_URL` when set) from the environment.
|
|
7
|
+
*
|
|
8
|
+
* @module agentos/extensions/research/citation-check/cli
|
|
9
|
+
*/
|
|
10
|
+
import { readFile } from 'node:fs/promises';
|
|
11
|
+
import { parseArgs } from 'node:util';
|
|
12
|
+
import { openAIEmbeddings } from './embeddings.js';
|
|
13
|
+
import { CitationInputError } from './parse.js';
|
|
14
|
+
import { citationReportFails, renderCitationLog } from './report.js';
|
|
15
|
+
import { checkCitationDocument, CitationSetupError } from './runner.js';
|
|
16
|
+
/** The command's help. */
|
|
17
|
+
export const CITATION_CLI_USAGE = `Usage: agentos-citations verify <file> [options]
|
|
18
|
+
|
|
19
|
+
Checks each quoted claim in <file> against the page its address names, and prints the
|
|
20
|
+
markdown for a "Citation check log" section. <file> is markdown, in which each quotation
|
|
21
|
+
is followed by its address ("the quoted words" (https://example.org/page)), or a JSON
|
|
22
|
+
array of {"claim", "url", "exact"}; - reads standard input.
|
|
23
|
+
|
|
24
|
+
Options:
|
|
25
|
+
--json print the report as JSON instead of the markdown log
|
|
26
|
+
--embeddings <name> none (the default) or openai: grade the claims not found word for
|
|
27
|
+
word with OpenAI embeddings through AgentOS (OPENAI_API_KEY, and
|
|
28
|
+
OPENAI_BASE_URL when set)
|
|
29
|
+
--format <format> auto (the default), markdown or json
|
|
30
|
+
--allow-unreachable do not fail on a page that could not be fetched or read
|
|
31
|
+
--no-fallback do not retry a failed page through the web scraper's tiers
|
|
32
|
+
--timeout <ms> the deadline for each page, in milliseconds (default 15000)
|
|
33
|
+
-h, --help print this help
|
|
34
|
+
|
|
35
|
+
Exit status: 0 when every claim passes; 1 when a claim is contradicted, not found or
|
|
36
|
+
unverifiable, or its page unreachable or unreadable; 2 for a usage or input error.`;
|
|
37
|
+
/** The process's standard input, read to its end as UTF-8. */
|
|
38
|
+
async function processStdin() {
|
|
39
|
+
const chunks = [];
|
|
40
|
+
for await (const chunk of process.stdin)
|
|
41
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
42
|
+
return Buffer.concat(chunks).toString('utf8');
|
|
43
|
+
}
|
|
44
|
+
/** An error's message, its first line. */
|
|
45
|
+
function messageOf(error) {
|
|
46
|
+
return ((error instanceof Error ? error.message : String(error)).split('\n')[0] ?? '').trim();
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Runs the command.
|
|
50
|
+
*
|
|
51
|
+
* @param argv - The arguments after the command's name.
|
|
52
|
+
* @param io - Where it reads and writes.
|
|
53
|
+
* @returns The exit status: 0 when the check passes, 1 when it fails, 2 for a usage or input error.
|
|
54
|
+
*/
|
|
55
|
+
export async function runCitationCli(argv, io = {}) {
|
|
56
|
+
const stdout = io.stdout ?? ((text) => void process.stdout.write(text));
|
|
57
|
+
const stderr = io.stderr ?? ((text) => void process.stderr.write(text));
|
|
58
|
+
const env = io.env ?? process.env;
|
|
59
|
+
const usage = (problem) => {
|
|
60
|
+
stderr(`${problem}\n\n${CITATION_CLI_USAGE}\n`);
|
|
61
|
+
return 2;
|
|
62
|
+
};
|
|
63
|
+
let parsed;
|
|
64
|
+
try {
|
|
65
|
+
parsed = parseCommand(argv);
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
return usage(messageOf(error));
|
|
69
|
+
}
|
|
70
|
+
const { values, positionals } = parsed;
|
|
71
|
+
if (values.help === true) {
|
|
72
|
+
stdout(`${CITATION_CLI_USAGE}\n`);
|
|
73
|
+
return 0;
|
|
74
|
+
}
|
|
75
|
+
const [command, file, ...extra] = positionals;
|
|
76
|
+
if (command !== 'verify' || file === undefined || extra.length > 0)
|
|
77
|
+
return usage('agentos-citations takes one command, verify, and one file');
|
|
78
|
+
const embeddings = values.embeddings ?? 'none';
|
|
79
|
+
if (embeddings !== 'none' && embeddings !== 'openai')
|
|
80
|
+
return usage(`--embeddings takes none or openai, not ${embeddings}`);
|
|
81
|
+
const format = values.format ?? 'auto';
|
|
82
|
+
if (format !== 'auto' && format !== 'markdown' && format !== 'json')
|
|
83
|
+
return usage(`--format takes auto, markdown or json, not ${format}`);
|
|
84
|
+
let timeoutMs;
|
|
85
|
+
if (values.timeout !== undefined) {
|
|
86
|
+
timeoutMs = Number(values.timeout);
|
|
87
|
+
if (!Number.isInteger(timeoutMs) || timeoutMs < 1)
|
|
88
|
+
return usage(`--timeout takes a whole number of milliseconds, not ${values.timeout}`);
|
|
89
|
+
}
|
|
90
|
+
const options = { ...io.options };
|
|
91
|
+
if (timeoutMs !== undefined)
|
|
92
|
+
options.timeoutMs = timeoutMs;
|
|
93
|
+
if (values['no-fallback'] === true)
|
|
94
|
+
options.fallback = false;
|
|
95
|
+
if (embeddings === 'openai') {
|
|
96
|
+
const apiKey = env.OPENAI_API_KEY;
|
|
97
|
+
if (apiKey === undefined || apiKey === '') {
|
|
98
|
+
stderr('--embeddings openai needs OPENAI_API_KEY in the environment\n');
|
|
99
|
+
return 2;
|
|
100
|
+
}
|
|
101
|
+
const baseUrl = env.OPENAI_BASE_URL;
|
|
102
|
+
options.embedFn = openAIEmbeddings({ apiKey, ...(baseUrl ? { baseUrl } : {}) });
|
|
103
|
+
}
|
|
104
|
+
let text;
|
|
105
|
+
try {
|
|
106
|
+
text = file === '-' ? await (io.readStdin ?? processStdin)() : await (io.readFile ?? ((path) => readFile(path, 'utf8')))(file);
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
stderr(`cannot read ${file}: ${messageOf(error)}\n`);
|
|
110
|
+
return 2;
|
|
111
|
+
}
|
|
112
|
+
let report;
|
|
113
|
+
try {
|
|
114
|
+
report = await checkCitationDocument(text, { ...options, format: format });
|
|
115
|
+
}
|
|
116
|
+
catch (error) {
|
|
117
|
+
if (error instanceof CitationInputError || error instanceof CitationSetupError) {
|
|
118
|
+
stderr(`${messageOf(error)}\n`);
|
|
119
|
+
return 2;
|
|
120
|
+
}
|
|
121
|
+
stderr(`the check stopped: ${messageOf(error)}\n`);
|
|
122
|
+
return 2;
|
|
123
|
+
}
|
|
124
|
+
const allowUnreachable = values['allow-unreachable'] === true;
|
|
125
|
+
stdout(values.json === true ? `${JSON.stringify(report, null, 2)}\n` : renderCitationLog(report, { allowUnreachable }));
|
|
126
|
+
return citationReportFails(report, { allowUnreachable }) ? 1 : 0;
|
|
127
|
+
}
|
|
128
|
+
/** The arguments, read strictly: an unknown option is an error. */
|
|
129
|
+
function parseCommand(argv) {
|
|
130
|
+
return parseArgs({
|
|
131
|
+
args: [...argv],
|
|
132
|
+
allowPositionals: true,
|
|
133
|
+
strict: true,
|
|
134
|
+
options: {
|
|
135
|
+
json: { type: 'boolean' },
|
|
136
|
+
embeddings: { type: 'string' },
|
|
137
|
+
format: { type: 'string' },
|
|
138
|
+
'allow-unreachable': { type: 'boolean' },
|
|
139
|
+
'no-fallback': { type: 'boolean' },
|
|
140
|
+
timeout: { type: 'string' },
|
|
141
|
+
help: { type: 'boolean', short: 'h' },
|
|
142
|
+
},
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
//# sourceMappingURL=cli.js.map
|
package/dist/cli.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAyB,MAAM,YAAY,CAAC;AACvE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAA6B,MAAM,aAAa,CAAC;AAGnG,0BAA0B;AAC1B,MAAM,CAAC,MAAM,kBAAkB,GAAG;;;;;;;;;;;;;;;;;;;mFAmBiD,CAAC;AAkBpF,8DAA8D;AAC9D,KAAK,UAAU,YAAY;IACzB,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK;QAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAClH,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAChD,CAAC;AAED,0CAA0C;AAC1C,SAAS,SAAS,CAAC,KAAc;IAC/B,OAAO,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AAChG,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAuB,EAAE,KAAoB,EAAE;IAClF,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM,IAAI,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAChF,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM,IAAI,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAChF,MAAM,GAAG,GAAG,EAAE,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IAClC,MAAM,KAAK,GAAG,CAAC,OAAe,EAAU,EAAE;QACxC,MAAM,CAAC,GAAG,OAAO,OAAO,kBAAkB,IAAI,CAAC,CAAC;QAChD,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;IAEF,IAAI,MAAuC,CAAC;IAC5C,IAAI,CAAC;QACH,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IACvC,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,kBAAkB,IAAI,CAAC,CAAC;QAClC,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,GAAG,WAAW,CAAC;IAC9C,IAAI,OAAO,KAAK,QAAQ,IAAI,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC9I,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC;IAC/C,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC,0CAA0C,UAAU,EAAE,CAAC,CAAC;IAC3H,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC;IACvC,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,UAAU,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC,8CAA8C,MAAM,EAAE,CAAC,CAAC;IAC1I,IAAI,SAA6B,CAAC;IAClC,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QACjC,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC,uDAAuD,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3I,CAAC;IAED,MAAM,OAAO,GAAyB,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;IACxD,IAAI,SAAS,KAAK,SAAS;QAAE,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;IAC3D,IAAI,MAAM,CAAC,aAAa,CAAC,KAAK,IAAI;QAAE,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;IAC7D,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,GAAG,CAAC,cAAc,CAAC;QAClC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAC1C,MAAM,CAAC,+DAA+D,CAAC,CAAC;YACxE,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,CAAC,eAAe,CAAC;QACpC,OAAO,CAAC,OAAO,GAAG,gBAAgB,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC;IAED,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS,IAAI,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACzI,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,eAAe,IAAI,KAAK,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACrD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,MAAsB,CAAC;IAC3B,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,qBAAqB,CAAC,IAAI,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,MAA0B,EAAE,CAAC,CAAC;IACjG,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,kBAAkB,IAAI,KAAK,YAAY,kBAAkB,EAAE,CAAC;YAC/E,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAChC,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,CAAC,sBAAsB,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnD,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,gBAAgB,GAAG,MAAM,CAAC,mBAAmB,CAAC,KAAK,IAAI,CAAC;IAC9D,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC;IACxH,OAAO,mBAAmB,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,mEAAmE;AACnE,SAAS,YAAY,CAAC,IAAuB;IAC3C,OAAO,SAAS,CAAC;QACf,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC;QACf,gBAAgB,EAAE,IAAI;QACtB,MAAM,EAAE,IAAI;QACZ,OAAO,EAAE;YACP,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YACzB,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC9B,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,mBAAmB,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YACxC,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;YAClC,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC3B,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE;SACtC;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file embeddings.ts
|
|
3
|
+
* @description Embedding functions from AgentOS's own surface, for `CitationVerifier` to grade with: one over
|
|
4
|
+
* `embedText` and its OpenAI provider (a key and, when wanted, a base address of an OpenAI-compatible endpoint), and one
|
|
5
|
+
* over an `IEmbeddingManager` a host has already configured. AgentOS is loaded when the first texts are embedded, so a
|
|
6
|
+
* check that grades by words alone never loads it.
|
|
7
|
+
*
|
|
8
|
+
* @module agentos/extensions/research/citation-check/embeddings
|
|
9
|
+
*/
|
|
10
|
+
import type { EmbedFn } from './types.js';
|
|
11
|
+
/** How the OpenAI embedding function calls AgentOS's `embedText`. */
|
|
12
|
+
export interface OpenAIEmbeddingOptions {
|
|
13
|
+
/** The API key; `OPENAI_API_KEY` from the environment when left out. */
|
|
14
|
+
apiKey?: string;
|
|
15
|
+
/** An OpenAI-compatible base address, such as `https://api.openai.com/v1`; `OPENAI_BASE_URL` or OpenAI's own when left out. */
|
|
16
|
+
baseUrl?: string;
|
|
17
|
+
/** The embedding model; AgentOS's default for OpenAI when left out. */
|
|
18
|
+
model?: string;
|
|
19
|
+
/** The dimensions of each vector, for a model that can reduce them. */
|
|
20
|
+
dimensions?: number;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* An embedding function over AgentOS's `embedText` with the OpenAI provider; from AgentOS 0.13.84 a long list goes in the
|
|
24
|
+
* batches the provider takes. Each call is billed by the provider.
|
|
25
|
+
*
|
|
26
|
+
* @param options - The key, the base address, the model and the dimensions.
|
|
27
|
+
*/
|
|
28
|
+
export declare function openAIEmbeddings(options?: OpenAIEmbeddingOptions): EmbedFn;
|
|
29
|
+
/** The part of AgentOS's `IEmbeddingManager` the embedding function calls. */
|
|
30
|
+
export interface EmbeddingManagerLike {
|
|
31
|
+
generateEmbeddings(request: {
|
|
32
|
+
texts: string[];
|
|
33
|
+
modelId?: string;
|
|
34
|
+
providerId?: string;
|
|
35
|
+
}): Promise<{
|
|
36
|
+
embeddings: number[][];
|
|
37
|
+
errors?: Array<{
|
|
38
|
+
textIndex: number;
|
|
39
|
+
message: string;
|
|
40
|
+
}>;
|
|
41
|
+
}>;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* An embedding function over an embedding manager the host has configured, such as AgentOS's `EmbeddingManager`. A
|
|
45
|
+
* batch in which any text failed is refused as a whole.
|
|
46
|
+
*
|
|
47
|
+
* @param manager - The manager.
|
|
48
|
+
* @param request - The model and the provider each request names; the manager's own choice when left out.
|
|
49
|
+
*/
|
|
50
|
+
export declare function managerEmbeddings(manager: EmbeddingManagerLike, request?: {
|
|
51
|
+
modelId?: string;
|
|
52
|
+
providerId?: string;
|
|
53
|
+
}): EmbedFn;
|
|
54
|
+
//# sourceMappingURL=embeddings.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,qEAAqE;AACrE,MAAM,WAAW,sBAAsB;IACrC,wEAAwE;IACxE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+HAA+H;IAC/H,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uEAAuE;IACvE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAgB9E;AAED,8EAA8E;AAC9E,MAAM,WAAW,oBAAoB;IACnC,kBAAkB,CAAC,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;QAC/F,UAAU,EAAE,MAAM,EAAE,EAAE,CAAC;QACvB,MAAM,CAAC,EAAE,KAAK,CAAC;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACxD,CAAC,CAAC;CACJ;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,oBAAoB,EAAE,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,OAAO,CAUjI"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file embeddings.ts
|
|
3
|
+
* @description Embedding functions from AgentOS's own surface, for `CitationVerifier` to grade with: one over
|
|
4
|
+
* `embedText` and its OpenAI provider (a key and, when wanted, a base address of an OpenAI-compatible endpoint), and one
|
|
5
|
+
* over an `IEmbeddingManager` a host has already configured. AgentOS is loaded when the first texts are embedded, so a
|
|
6
|
+
* check that grades by words alone never loads it.
|
|
7
|
+
*
|
|
8
|
+
* @module agentos/extensions/research/citation-check/embeddings
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* An embedding function over AgentOS's `embedText` with the OpenAI provider; from AgentOS 0.13.84 a long list goes in the
|
|
12
|
+
* batches the provider takes. Each call is billed by the provider.
|
|
13
|
+
*
|
|
14
|
+
* @param options - The key, the base address, the model and the dimensions.
|
|
15
|
+
*/
|
|
16
|
+
export function openAIEmbeddings(options = {}) {
|
|
17
|
+
return async (texts) => {
|
|
18
|
+
const { embedText } = await import('@framers/agentos');
|
|
19
|
+
const result = await embedText({
|
|
20
|
+
provider: 'openai',
|
|
21
|
+
input: texts,
|
|
22
|
+
...(options.model ? { model: options.model } : {}),
|
|
23
|
+
...(options.apiKey ? { apiKey: options.apiKey } : {}),
|
|
24
|
+
...(options.baseUrl ? { baseUrl: options.baseUrl } : {}),
|
|
25
|
+
...(options.dimensions ? { dimensions: options.dimensions } : {}),
|
|
26
|
+
});
|
|
27
|
+
if (result.embeddings.length !== texts.length) {
|
|
28
|
+
throw new Error(`the embedding provider answered ${result.embeddings.length} vectors for ${texts.length} texts`);
|
|
29
|
+
}
|
|
30
|
+
return result.embeddings;
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* An embedding function over an embedding manager the host has configured, such as AgentOS's `EmbeddingManager`. A
|
|
35
|
+
* batch in which any text failed is refused as a whole.
|
|
36
|
+
*
|
|
37
|
+
* @param manager - The manager.
|
|
38
|
+
* @param request - The model and the provider each request names; the manager's own choice when left out.
|
|
39
|
+
*/
|
|
40
|
+
export function managerEmbeddings(manager, request = {}) {
|
|
41
|
+
return async (texts) => {
|
|
42
|
+
const response = await manager.generateEmbeddings({ texts, ...request });
|
|
43
|
+
const failed = response.errors?.[0];
|
|
44
|
+
if (failed !== undefined)
|
|
45
|
+
throw new Error(`the embedding manager failed text ${failed.textIndex + 1} of ${texts.length}: ${failed.message}`);
|
|
46
|
+
if (response.embeddings.length !== texts.length) {
|
|
47
|
+
throw new Error(`the embedding manager answered ${response.embeddings.length} vectors for ${texts.length} texts`);
|
|
48
|
+
}
|
|
49
|
+
return response.embeddings;
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=embeddings.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../src/embeddings.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAgBH;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,UAAkC,EAAE;IACnE,OAAO,KAAK,EAAE,KAAK,EAAE,EAAE;QACrB,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC;YAC7B,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,KAAK;YACZ,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClD,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClE,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,mCAAmC,MAAM,CAAC,UAAU,CAAC,MAAM,gBAAgB,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC;QACnH,CAAC;QACD,OAAO,MAAM,CAAC,UAAU,CAAC;IAC3B,CAAC,CAAC;AACJ,CAAC;AAUD;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAA6B,EAAE,UAAqD,EAAE;IACtH,OAAO,KAAK,EAAE,KAAK,EAAE,EAAE;QACrB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QACzE,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;QACpC,IAAI,MAAM,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,MAAM,CAAC,SAAS,GAAG,CAAC,OAAO,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7I,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,kCAAkC,QAAQ,CAAC,UAAU,CAAC,MAAM,gBAAgB,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC;QACpH,CAAC;QACD,OAAO,QAAQ,CAAC,UAAU,CAAC;IAC7B,CAAC,CAAC;AACJ,CAAC"}
|