@framers/agentos-ext-citation-check 0.0.0-stage → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/LICENSE +98 -0
  2. package/README.md +167 -2
  3. package/dist/bin.d.ts +10 -0
  4. package/dist/bin.d.ts.map +1 -0
  5. package/dist/bin.js +11 -0
  6. package/dist/bin.js.map +1 -0
  7. package/dist/cli.d.ts +36 -0
  8. package/dist/cli.d.ts.map +1 -0
  9. package/dist/cli.js +145 -0
  10. package/dist/cli.js.map +1 -0
  11. package/dist/embeddings.d.ts +54 -0
  12. package/dist/embeddings.d.ts.map +1 -0
  13. package/dist/embeddings.js +52 -0
  14. package/dist/embeddings.js.map +1 -0
  15. package/dist/index.d.ts +55 -0
  16. package/dist/index.d.ts.map +1 -0
  17. package/dist/index.js +63 -0
  18. package/dist/index.js.map +1 -0
  19. package/dist/match.d.ts +44 -0
  20. package/dist/match.d.ts.map +1 -0
  21. package/dist/match.js +108 -0
  22. package/dist/match.js.map +1 -0
  23. package/dist/page.d.ts +105 -0
  24. package/dist/page.d.ts.map +1 -0
  25. package/dist/page.js +400 -0
  26. package/dist/page.js.map +1 -0
  27. package/dist/parse.d.ts +36 -0
  28. package/dist/parse.d.ts.map +1 -0
  29. package/dist/parse.js +314 -0
  30. package/dist/parse.js.map +1 -0
  31. package/dist/report.d.ts +28 -0
  32. package/dist/report.d.ts.map +1 -0
  33. package/dist/report.js +128 -0
  34. package/dist/report.js.map +1 -0
  35. package/dist/runner.d.ts +89 -0
  36. package/dist/runner.d.ts.map +1 -0
  37. package/dist/runner.js +298 -0
  38. package/dist/runner.js.map +1 -0
  39. package/dist/tools/citationVerify.d.ts +59 -0
  40. package/dist/tools/citationVerify.d.ts.map +1 -0
  41. package/dist/tools/citationVerify.js +98 -0
  42. package/dist/tools/citationVerify.js.map +1 -0
  43. package/dist/types.d.ts +150 -0
  44. package/dist/types.d.ts.map +1 -0
  45. package/dist/types.js +20 -0
  46. package/dist/types.js.map +1 -0
  47. package/manifest.json +66 -0
  48. package/package.json +79 -3
@@ -0,0 +1,55 @@
1
+ /**
2
+ * @file index.ts
3
+ * @description Checks the quoted claims of a document against the pages they cite, with AgentOS's citation tooling and
4
+ * no search service: `parseCitedClaims` finds the claims and their addresses, `checkCitations` reads each page once and
5
+ * grades each claim (word for word first, then through AgentOS's `CitationVerifier` when an embedding function is
6
+ * given), `renderCitationLog` writes the markdown for a "Citation check log" section, and `citationReportFails` decides
7
+ * the result. The pack's one tool is `citation_verify`; the `agentos-citations` command runs the same check.
8
+ *
9
+ * @module @framers/agentos-ext-citation-check
10
+ */
11
+ import type { ExtensionPack, ExtensionPackContext } from '@framers/agentos';
12
+ import type { EgressProfile } from './page.js';
13
+ import type { EmbedFn, NliFn } from './types.js';
14
+ export * from './types.js';
15
+ export { parseCitedClaims, CitationInputError, isHttpUrl, type CitedClaimFormat } from './parse.js';
16
+ export { normalizeForMatch, quoteFragments, locateQuote, lexicalNearest, withoutInvisible } from './match.js';
17
+ export { pageKey, type EgressProfile, type PageScraper } from './page.js';
18
+ export { checkCitations, checkCitationDocument, CitationSetupError, type CitationCheckOptions } from './runner.js';
19
+ export { renderCitationLog, citationReportFails, FAILING_VERDICTS, type CitationReportOptions } from './report.js';
20
+ export { openAIEmbeddings, managerEmbeddings, type OpenAIEmbeddingOptions, type EmbeddingManagerLike } from './embeddings.js';
21
+ export { CitationVerifyTool, type CitationVerifyInput, type CitationVerifyOutput } from './tools/citationVerify.js';
22
+ export { runCitationCli, CITATION_CLI_USAGE, type CitationCliIO } from './cli.js';
23
+ /** The pack's options, from its manifest entry or from the host's code. */
24
+ export interface CitationCheckPackOptions {
25
+ /** `none` (the default) or `openai`: grade a claim not found word for word with OpenAI embeddings through AgentOS. */
26
+ embeddings?: 'none' | 'openai';
27
+ /** The OpenAI embedding model; AgentOS's default when left out. */
28
+ embeddingModel?: string;
29
+ /** The checked egress policy pages are read through: `hosted` (the default) or `local`. Ignored when `fetch` is given. */
30
+ egress?: EgressProfile;
31
+ /** A `fetch` that applies the host's own egress rules, used instead of checked egress. */
32
+ fetch?: typeof globalThis.fetch;
33
+ /** An embedding function of the host's, used instead of `embeddings`. */
34
+ embedFn?: EmbedFn;
35
+ /** An NLI function, such as grounding-guard's `createNliFunction()`, that turns a contradicted `supported` claim. */
36
+ nliFn?: NliFn;
37
+ /** The deadline for each page, in milliseconds; 15,000 when left out. */
38
+ timeoutMs?: number;
39
+ /** The most bytes read of a page; 5,000,000 when left out. */
40
+ maxBytes?: number;
41
+ /** The most distinct pages one call reads; 100 when left out. */
42
+ maxPages?: number;
43
+ /** The tool descriptor's priority; 50 when left out. */
44
+ priority?: number;
45
+ }
46
+ /**
47
+ * The pack: one tool, `citation_verify`. With `embeddings: 'openai'` the OpenAI key comes from the `openai.apiKey`
48
+ * secret, or `OPENAI_API_KEY` when the secret is not set.
49
+ *
50
+ * @param context - The pack's options, secrets and logger.
51
+ * @throws {TypeError} For an `embeddings` or an `egress` the pack does not know.
52
+ */
53
+ export declare function createExtensionPack(context?: ExtensionPackContext): ExtensionPack;
54
+ export default createExtensionPack;
55
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAE5E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEjD,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,SAAS,EAAE,KAAK,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACpG,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,WAAW,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9G,OAAO,EAAE,OAAO,EAAE,KAAK,aAAa,EAAE,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AAC1E,OAAO,EAAE,cAAc,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,KAAK,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnH,OAAO,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,KAAK,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACnH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,KAAK,sBAAsB,EAAE,KAAK,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAC9H,OAAO,EAAE,kBAAkB,EAAE,KAAK,mBAAmB,EAAE,KAAK,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACpH,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,KAAK,aAAa,EAAE,MAAM,UAAU,CAAC;AAElF,2EAA2E;AAC3E,MAAM,WAAW,wBAAwB;IACvC,sHAAsH;IACtH,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC/B,mEAAmE;IACnE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0HAA0H;IAC1H,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,0FAA0F;IAC1F,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAChC,yEAAyE;IACzE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,qHAAqH;IACrH,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,yEAAyE;IACzE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,GAAE,oBAAyB,GAAG,aAAa,CAgCrF;AAED,eAAe,mBAAmB,CAAC"}
package/dist/index.js ADDED
@@ -0,0 +1,63 @@
1
+ /**
2
+ * @file index.ts
3
+ * @description Checks the quoted claims of a document against the pages they cite, with AgentOS's citation tooling and
4
+ * no search service: `parseCitedClaims` finds the claims and their addresses, `checkCitations` reads each page once and
5
+ * grades each claim (word for word first, then through AgentOS's `CitationVerifier` when an embedding function is
6
+ * given), `renderCitationLog` writes the markdown for a "Citation check log" section, and `citationReportFails` decides
7
+ * the result. The pack's one tool is `citation_verify`; the `agentos-citations` command runs the same check.
8
+ *
9
+ * @module @framers/agentos-ext-citation-check
10
+ */
11
+ import { openAIEmbeddings } from './embeddings.js';
12
+ import { CitationVerifyTool } from './tools/citationVerify.js';
13
+ export * from './types.js';
14
+ export { parseCitedClaims, CitationInputError, isHttpUrl } from './parse.js';
15
+ export { normalizeForMatch, quoteFragments, locateQuote, lexicalNearest, withoutInvisible } from './match.js';
16
+ export { pageKey } from './page.js';
17
+ export { checkCitations, checkCitationDocument, CitationSetupError } from './runner.js';
18
+ export { renderCitationLog, citationReportFails, FAILING_VERDICTS } from './report.js';
19
+ export { openAIEmbeddings, managerEmbeddings } from './embeddings.js';
20
+ export { CitationVerifyTool } from './tools/citationVerify.js';
21
+ export { runCitationCli, CITATION_CLI_USAGE } from './cli.js';
22
+ /**
23
+ * The pack: one tool, `citation_verify`. With `embeddings: 'openai'` the OpenAI key comes from the `openai.apiKey`
24
+ * secret, or `OPENAI_API_KEY` when the secret is not set.
25
+ *
26
+ * @param context - The pack's options, secrets and logger.
27
+ * @throws {TypeError} For an `embeddings` or an `egress` the pack does not know.
28
+ */
29
+ export function createExtensionPack(context = {}) {
30
+ const options = (context.options ?? {});
31
+ const embeddings = options.embeddings ?? 'none';
32
+ if (embeddings !== 'none' && embeddings !== 'openai')
33
+ throw new TypeError(`embeddings must be none or openai, not ${JSON.stringify(embeddings)}`);
34
+ if (options.egress !== undefined && options.egress !== 'hosted' && options.egress !== 'local') {
35
+ throw new TypeError(`egress must be hosted or local, not ${JSON.stringify(options.egress)}`);
36
+ }
37
+ const apiKey = context.getSecret?.('openai.apiKey');
38
+ const embedFn = options.embedFn ??
39
+ (embeddings === 'openai' ? openAIEmbeddings({ ...(apiKey ? { apiKey } : {}), ...(options.embeddingModel ? { model: options.embeddingModel } : {}) }) : undefined);
40
+ const tool = new CitationVerifyTool({
41
+ ...(options.fetch !== undefined ? { fetch: options.fetch } : { egress: options.egress ?? 'hosted' }),
42
+ ...(embedFn !== undefined ? { embedFn } : {}),
43
+ ...(options.nliFn !== undefined ? { nliFn: options.nliFn } : {}),
44
+ ...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
45
+ ...(options.maxBytes !== undefined ? { maxBytes: options.maxBytes } : {}),
46
+ ...(options.maxPages !== undefined ? { maxPages: options.maxPages } : {}),
47
+ ...(context.logger !== undefined ? { logger: context.logger } : {}),
48
+ });
49
+ return {
50
+ name: '@framers/agentos-ext-citation-check',
51
+ descriptors: [
52
+ {
53
+ id: tool.name,
54
+ kind: 'tool',
55
+ priority: options.priority ?? 50,
56
+ payload: tool,
57
+ requiredSecrets: [{ id: 'openai.apiKey', optional: true }],
58
+ },
59
+ ],
60
+ };
61
+ }
62
+ export default createExtensionPack;
63
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAEnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAG/D,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,SAAS,EAAyB,MAAM,YAAY,CAAC;AACpG,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,WAAW,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9G,OAAO,EAAE,OAAO,EAAwC,MAAM,WAAW,CAAC;AAC1E,OAAO,EAAE,cAAc,EAAE,qBAAqB,EAAE,kBAAkB,EAA6B,MAAM,aAAa,CAAC;AACnH,OAAO,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,gBAAgB,EAA8B,MAAM,aAAa,CAAC;AACnH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAA0D,MAAM,iBAAiB,CAAC;AAC9H,OAAO,EAAE,kBAAkB,EAAuD,MAAM,2BAA2B,CAAC;AACpH,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAsB,MAAM,UAAU,CAAC;AA0BlF;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAAC,UAAgC,EAAE;IACpE,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAA6B,CAAC;IACpE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,MAAM,CAAC;IAChD,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,QAAQ;QAAE,MAAM,IAAI,SAAS,CAAC,0CAA0C,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAClJ,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QAC9F,MAAM,IAAI,SAAS,CAAC,uCAAuC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC/F,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,eAAe,CAAC,CAAC;IACpD,MAAM,OAAO,GACX,OAAO,CAAC,OAAO;QACf,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACpK,MAAM,IAAI,GAAG,IAAI,kBAAkB,CAAC;QAClC,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;QACpG,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpE,CAAC,CAAC;IACH,OAAO;QACL,IAAI,EAAE,qCAAqC;QAC3C,WAAW,EAAE;YACX;gBACE,EAAE,EAAE,IAAI,CAAC,IAAI;gBACb,IAAI,EAAE,MAAM;gBACZ,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,EAAE;gBAChC,OAAO,EAAE,IAAI;gBACb,eAAe,EAAE,CAAC,EAAE,EAAE,EAAE,eAAe,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;aAC3D;SACF;KACF,CAAC;AACJ,CAAC;AAED,eAAe,mBAAmB,CAAC"}
@@ -0,0 +1,44 @@
1
+ /**
2
+ * @file match.ts
3
+ * @description The classical checks, run before any model: whether a quotation's words stand on a page once both are
4
+ * read the same way (compatibility forms, typographic quotation marks, dashes, case, white space and invisible
5
+ * characters set aside, an ellipsis or a bracketed insertion read as a gap), never taking a match that cuts a word or a
6
+ * number; and which of a page's sentences share the most content words with a claim.
7
+ *
8
+ * @module agentos/extensions/research/citation-check/match
9
+ */
10
+ import type { CitationPassage } from './types.js';
11
+ /** A text without the characters a reader never sees. */
12
+ export declare function withoutInvisible(text: string): string;
13
+ /**
14
+ * A text as the checks read it: NFKC, the invisible characters removed, typographic and angle quotation marks plain,
15
+ * every dash a hyphen-minus with no space around it, lower case, white space collapsed.
16
+ */
17
+ export declare function normalizeForMatch(text: string): string;
18
+ /**
19
+ * A quotation's pieces as {@link normalizeForMatch} reads them: the words between its gaps (an ellipsis, a bracketed
20
+ * insertion), each without the punctuation and quotation marks at its ends; pieces with no word are left out.
21
+ */
22
+ export declare function quoteFragments(quotation: string): string[];
23
+ /**
24
+ * Where a quotation's pieces stand in a normalized text, each after the one before and none cutting a word or a number:
25
+ * the offsets from the first piece's start to the last piece's end, or null when a piece does not stand there.
26
+ *
27
+ * @param text - The page's text, read with {@link normalizeForMatch}.
28
+ * @param fragments - The quotation's pieces, from {@link quoteFragments}.
29
+ */
30
+ export declare function locateQuote(text: string, fragments: readonly string[]): {
31
+ start: number;
32
+ end: number;
33
+ } | null;
34
+ /**
35
+ * The passages nearest a claim by shared words: each passage's Jaccard index over the content words and numbers of the
36
+ * two (read with {@link normalizeForMatch}), the passages that share none left out and repeated passages counted once,
37
+ * nearest first, ties in the passages' order.
38
+ *
39
+ * @param claim - The claim.
40
+ * @param passages - The page's sentences, in page order.
41
+ * @param count - The most passages answered.
42
+ */
43
+ export declare function lexicalNearest(claim: string, passages: readonly string[], count: number): CitationPassage[];
44
+ //# sourceMappingURL=match.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"match.d.ts","sourceRoot":"","sources":["../src/match.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAWlD,yDAAyD;AACzD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAErD;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKtD;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAK1D;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,MAAM,EAAE,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAe7G;AAWD;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,eAAe,EAAE,CAgB3G"}
package/dist/match.js ADDED
@@ -0,0 +1,108 @@
1
+ /**
2
+ * @file match.ts
3
+ * @description The classical checks, run before any model: whether a quotation's words stand on a page once both are
4
+ * read the same way (compatibility forms, typographic quotation marks, dashes, case, white space and invisible
5
+ * characters set aside, an ellipsis or a bracketed insertion read as a gap), never taking a match that cuts a word or a
6
+ * number; and which of a page's sentences share the most content words with a claim.
7
+ *
8
+ * @module agentos/extensions/research/citation-check/match
9
+ */
10
+ import { collapse, contentWords, exactSpan, fold, words } from '@framers/agentos-ext-grounding-guard/values';
11
+ /** Characters a reader never sees: soft hyphens, zero-width spaces and joiners, word joiners, byte order marks. */
12
+ const INVISIBLE = /[\u00AD\u200B-\u200D\u2060\uFEFF]/gu;
13
+ /** Dashes and minus signs, with the white space around them, read as one hyphen-minus. */
14
+ const DASHES = /\s*[\u2010-\u2015\u2212\uFE58\uFE63\uFF0D-]+\s*/gu;
15
+ /** Where a quotation leaves words out or puts its own in: an ellipsis, or words in square brackets. */
16
+ const GAP = /\s*(?:\.{3,}|\[[^\]]*\])\s*/u;
17
+ /** Punctuation and quotation marks at the ends of a quotation's piece, which the page may set differently. */
18
+ const EDGES = /^[\s"',.;:!?]+|[\s"',.;:!?]+$/gu;
19
+ /** A text without the characters a reader never sees. */
20
+ export function withoutInvisible(text) {
21
+ return text.replace(INVISIBLE, '');
22
+ }
23
+ /**
24
+ * A text as the checks read it: NFKC, the invisible characters removed, typographic and angle quotation marks plain,
25
+ * every dash a hyphen-minus with no space around it, lower case, white space collapsed.
26
+ */
27
+ export function normalizeForMatch(text) {
28
+ const folded = fold(withoutInvisible(text.normalize('NFKC')))
29
+ .replace(/[«»]/gu, '"')
30
+ .replace(/[‹›]/gu, "'");
31
+ return collapse(folded.replace(DASHES, '-').toLowerCase());
32
+ }
33
+ /**
34
+ * A quotation's pieces as {@link normalizeForMatch} reads them: the words between its gaps (an ellipsis, a bracketed
35
+ * insertion), each without the punctuation and quotation marks at its ends; pieces with no word are left out.
36
+ */
37
+ export function quoteFragments(quotation) {
38
+ return normalizeForMatch(quotation)
39
+ .split(GAP)
40
+ .map((piece) => piece.replace(EDGES, ''))
41
+ .filter((piece) => words(piece).length > 0);
42
+ }
43
+ /**
44
+ * Where a quotation's pieces stand in a normalized text, each after the one before and none cutting a word or a number:
45
+ * the offsets from the first piece's start to the last piece's end, or null when a piece does not stand there.
46
+ *
47
+ * @param text - The page's text, read with {@link normalizeForMatch}.
48
+ * @param fragments - The quotation's pieces, from {@link quoteFragments}.
49
+ */
50
+ export function locateQuote(text, fragments) {
51
+ let offset = 0;
52
+ let start = -1;
53
+ let end = -1;
54
+ for (const fragment of fragments) {
55
+ // The rest of the text starts after the last match and its space, so the offsets `exactSpan` gives (into its own
56
+ // collapsed copy) are offsets into this slice.
57
+ while (text[offset] === ' ')
58
+ offset += 1;
59
+ const span = exactSpan(text.slice(offset), fragment);
60
+ if (span === null)
61
+ return null;
62
+ if (start === -1)
63
+ start = offset + span.start;
64
+ end = offset + span.end;
65
+ offset = end;
66
+ }
67
+ return start === -1 ? null : { start, end };
68
+ }
69
+ /**
70
+ * The words a passage is compared by: its content words (three letters or more, no stop word) and every number, so "40
71
+ * metres" and "31 metres" differ.
72
+ */
73
+ function overlapTerms(text) {
74
+ const normalized = normalizeForMatch(text);
75
+ return new Set([...contentWords(normalized), ...words(normalized).filter((word) => /\p{N}/u.test(word))]);
76
+ }
77
+ /**
78
+ * The passages nearest a claim by shared words: each passage's Jaccard index over the content words and numbers of the
79
+ * two (read with {@link normalizeForMatch}), the passages that share none left out and repeated passages counted once,
80
+ * nearest first, ties in the passages' order.
81
+ *
82
+ * @param claim - The claim.
83
+ * @param passages - The page's sentences, in page order.
84
+ * @param count - The most passages answered.
85
+ */
86
+ export function lexicalNearest(claim, passages, count) {
87
+ const wanted = overlapTerms(claim);
88
+ if (wanted.size === 0)
89
+ return [];
90
+ const seen = new Set();
91
+ const scored = [];
92
+ passages.forEach((text, order) => {
93
+ if (seen.has(text))
94
+ return;
95
+ seen.add(text);
96
+ const have = overlapTerms(text);
97
+ let shared = 0;
98
+ for (const word of wanted)
99
+ if (have.has(word))
100
+ shared += 1;
101
+ if (shared === 0)
102
+ return;
103
+ scored.push({ text, score: shared / (wanted.size + have.size - shared), order });
104
+ });
105
+ scored.sort((a, b) => b.score - a.score || a.order - b.order);
106
+ return scored.slice(0, Math.max(0, count)).map(({ text, score }) => ({ text, score }));
107
+ }
108
+ //# sourceMappingURL=match.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"match.js","sourceRoot":"","sources":["../src/match.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,6CAA6C,CAAC;AAG7G,mHAAmH;AACnH,MAAM,SAAS,GAAG,qCAAqC,CAAC;AACxD,0FAA0F;AAC1F,MAAM,MAAM,GAAG,mDAAmD,CAAC;AACnE,uGAAuG;AACvG,MAAM,GAAG,GAAG,8BAA8B,CAAC;AAC3C,8GAA8G;AAC9G,MAAM,KAAK,GAAG,iCAAiC,CAAC;AAEhD,yDAAyD;AACzD,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;AACrC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;SAC1D,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC1B,OAAO,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;AAC7D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,SAAiB;IAC9C,OAAO,iBAAiB,CAAC,SAAS,CAAC;SAChC,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;SACxC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAChD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,SAA4B;IACpE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;IACf,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC;IACb,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,iHAAiH;QACjH,+CAA+C;QAC/C,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG;YAAE,MAAM,IAAI,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;QACrD,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC/B,IAAI,KAAK,KAAK,CAAC,CAAC;YAAE,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC;QAC9C,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC;QACxB,MAAM,GAAG,GAAG,CAAC;IACf,CAAC;IACD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC3C,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,UAAU,CAAC,EAAE,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5G,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa,EAAE,QAA2B,EAAE,KAAa;IACtF,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,MAAM,GAA+C,EAAE,CAAC;IAC9D,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC/B,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO;QAC3B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACf,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,IAAI,IAAI,MAAM;YAAE,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,MAAM,IAAI,CAAC,CAAC;QAC3D,IAAI,MAAM,KAAK,CAAC;YAAE,OAAO;QACzB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IACnF,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;AACzF,CAAC"}
package/dist/page.d.ts ADDED
@@ -0,0 +1,105 @@
1
+ /**
2
+ * @file page.ts
3
+ * @description One cited page as passages. A request goes out once per page, with a browser-like User-Agent from the
4
+ * web scraper pack, a deadline and a size cap, through Node's fetch (redirects followed) or through AgentOS's checked
5
+ * egress (every address and every redirect checked against a policy); a page that fails in a way a browser may mend is
6
+ * retried once through the web scraper pack's tiers. The answer is read by its content type: HTML and XML through
7
+ * public-sources' `htmlToBlocks` in page mode, other text by paragraphs, a PDF or another binary not at all; its
8
+ * charset from a byte order mark, the content type or a `<meta>` tag. Its sentences become passages through
9
+ * public-sources' `citationSources`, cut by grounding-guard's `sentenceSpans`. A page that fails is a status, never a
10
+ * throw; only the caller's own cancellation throws.
11
+ *
12
+ * @module agentos/extensions/research/citation-check/page
13
+ */
14
+ import { type CitationSource } from '@framers/agentos-ext-public-sources';
15
+ import { type ScrapeInput, type ScrapeResult } from '@framers/agentos-ext-web-scraper';
16
+ import type { CitationLogger, PageFailure, PageStatus } from './types.js';
17
+ /**
18
+ * The fallback a failed page is retried through: the web scraper pack's `WebScraperService`, whose tiers go from a
19
+ * headless browser to a plain request, or any object with its `scrape`.
20
+ */
21
+ export interface PageScraper {
22
+ scrape(input: ScrapeInput): Promise<ScrapeResult>;
23
+ }
24
+ /** The policies of AgentOS's checked egress: `hosted` (public https addresses on port 443 alone) or `local` (any address). */
25
+ export type EgressProfile = 'hosted' | 'local';
26
+ /** An answer as a transport gives it. */
27
+ export interface TransportAnswer {
28
+ /** The answer's HTTP status. */
29
+ status: number;
30
+ /** The address the answer came from, after redirects. */
31
+ finalUrl: string;
32
+ /** A response header's value, or null. */
33
+ header(name: string): string | null;
34
+ /** The body read up to `cap` bytes: its bytes, or null past the cap. */
35
+ read(cap: number): Promise<Uint8Array | null>;
36
+ /** Releases a body that will not be read. */
37
+ discard(): Promise<void>;
38
+ }
39
+ /** One GET of an address under a signal. */
40
+ export type Transport = (url: string, signal: AbortSignal) => Promise<TransportAnswer>;
41
+ /** A failure a transport names itself, such as the egress policy's refusal. */
42
+ export declare class PageRefusal extends Error {
43
+ readonly failure: PageFailure;
44
+ readonly detail: string | null;
45
+ /**
46
+ * @param failure - The failure it stands for.
47
+ * @param detail - More about it, such as the egress policy's reason.
48
+ */
49
+ constructor(failure: PageFailure, detail?: string | null);
50
+ }
51
+ /** The address a page is fetched at: the cited address without its fragment, which never reaches a server. */
52
+ export declare function pageKey(url: string): string;
53
+ /** An address's origin for a log line, which leaves out the path and the query where secrets can sit. */
54
+ export declare function originOf(url: string): string;
55
+ /** Requests through a `fetch`: redirects followed, the body streamed up to the cap. */
56
+ export declare function fetchTransport(fetchFn: typeof globalThis.fetch, userAgent?: string): Transport;
57
+ /**
58
+ * Requests through AgentOS's checked egress (`@framers/agentos/safety/egress`, from 0.13.120): each address, and each
59
+ * address a redirect names, is checked against the policy before a byte is sent, its connection pinned to the address
60
+ * checked, and the answer bounded by the deadline and the size cap. Up to five redirects are followed.
61
+ *
62
+ * @throws {Error} When the installed AgentOS has no checked egress.
63
+ */
64
+ export declare function egressTransport(profile: EgressProfile, bounds: {
65
+ timeoutMs: number;
66
+ maxBytes: number;
67
+ }, userAgent?: string): Promise<Transport>;
68
+ /** How a page is read. */
69
+ export interface PageReaderOptions {
70
+ /** The request each attempt goes through. */
71
+ transport: Transport;
72
+ /** The fallback for a failure it may mend, or null for none. */
73
+ scraper: PageScraper | null;
74
+ /** The deadline of one request, in milliseconds. */
75
+ timeoutMs: number;
76
+ /** The most bytes of a body read. */
77
+ maxBytes: number;
78
+ /** The most characters of text kept. */
79
+ textBudget: number;
80
+ /** The caller's cancellation. */
81
+ signal?: AbortSignal;
82
+ /** The clock that stamps a page's `readAt`. */
83
+ now: () => number;
84
+ /** Where the reader writes its events. */
85
+ logger?: CitationLogger;
86
+ }
87
+ /** A page as the check reads it: its status, and when it was read, its blocks and its sentences as passages. */
88
+ export interface PageRead {
89
+ /** How the page fared. */
90
+ status: PageStatus;
91
+ /** Its text blocks, in page order; empty when it was not read. */
92
+ blocks: string[];
93
+ /** Its sentences, shaped as AgentOS's `VerificationSource`, in page order; empty when it was not read. */
94
+ passages: CitationSource[];
95
+ }
96
+ /**
97
+ * Reads one page: one request, then for a failure the fallback may mend (a deadline, no connection, an error status
98
+ * other than 404 and 410, a page with no text) one try through the fallback scraper. Answers a status for a page that
99
+ * is not read; throws only the caller's own cancellation.
100
+ *
101
+ * @param url - The address to fetch, from {@link pageKey}.
102
+ * @param options - How to read it.
103
+ */
104
+ export declare function readPage(url: string, options: PageReaderOptions): Promise<PageRead>;
105
+ //# sourceMappingURL=page.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"page.d.ts","sourceRoot":"","sources":["../src/page.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAAiC,KAAK,cAAc,EAAuB,MAAM,qCAAqC,CAAC;AAC9H,OAAO,EAAkB,KAAK,WAAW,EAAE,KAAK,YAAY,EAAE,MAAM,kCAAkC,CAAC;AAGvG,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1E;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CACnD;AAED,8HAA8H;AAC9H,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE/C,yCAAyC;AACzC,MAAM,WAAW,eAAe;IAC9B,gCAAgC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,0CAA0C;IAC1C,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACpC,wEAAwE;IACxE,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAC9C,6CAA6C;IAC7C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,4CAA4C;AAC5C,MAAM,MAAM,SAAS,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,eAAe,CAAC,CAAC;AAEvF,+EAA+E;AAC/E,qBAAa,WAAY,SAAQ,KAAK;IAMlC,QAAQ,CAAC,OAAO,EAAE,WAAW;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IANhC;;;OAGG;gBAEQ,OAAO,EAAE,WAAW,EACpB,MAAM,GAAE,MAAM,GAAG,IAAW;CAKxC;AAWD,8GAA8G;AAC9G,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAI3C;AAED,yGAAyG;AACzG,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAM5C;AA6BD,uFAAuF;AACvF,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,UAAU,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAa9F;AAwCD;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAiCrJ;AAoFD,0BAA0B;AAC1B,MAAM,WAAW,iBAAiB;IAChC,6CAA6C;IAC7C,SAAS,EAAE,SAAS,CAAC;IACrB,gEAAgE;IAChE,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;IAC5B,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,+CAA+C;IAC/C,GAAG,EAAE,MAAM,MAAM,CAAC;IAClB,0CAA0C;IAC1C,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB;AAED,gHAAgH;AAChH,MAAM,WAAW,QAAQ;IACvB,0BAA0B;IAC1B,MAAM,EAAE,UAAU,CAAC;IACnB,kEAAkE;IAClE,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,0GAA0G;IAC1G,QAAQ,EAAE,cAAc,EAAE,CAAC;CAC5B;AAoGD;;;;;;;GAOG;AACH,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAkBzF"}