@ganju/utils 0.0.14 → 0.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/channelCustomer.d.ts +99 -0
- package/dist/channelCustomer.d.ts.map +1 -0
- package/dist/channelCustomer.js +533 -0
- package/dist/channelDebounce.d.ts +7 -0
- package/dist/channelDebounce.d.ts.map +1 -1
- package/dist/chunkOverlap.d.ts +2 -0
- package/dist/chunkOverlap.d.ts.map +1 -0
- package/dist/chunkOverlap.js +9 -0
- package/dist/chunking.d.ts.map +1 -1
- package/dist/chunking.js +24 -4
- package/dist/citations.d.ts +28 -0
- package/dist/citations.d.ts.map +1 -0
- package/dist/citations.js +111 -0
- package/dist/constants.d.ts +49 -0
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +202 -17
- package/dist/data.d.ts +9 -0
- package/dist/data.d.ts.map +1 -0
- package/dist/data.js +78 -0
- package/dist/dataApi.d.ts +511 -0
- package/dist/dataApi.d.ts.map +1 -0
- package/dist/dataApi.js +355 -0
- package/dist/dataCompiler.d.ts +68 -0
- package/dist/dataCompiler.d.ts.map +1 -0
- package/dist/dataCompiler.js +726 -0
- package/dist/dataDdl.d.ts +29 -0
- package/dist/dataDdl.d.ts.map +1 -0
- package/dist/dataDdl.js +354 -0
- package/dist/dataEngine.d.ts +143 -0
- package/dist/dataEngine.d.ts.map +1 -0
- package/dist/dataEngine.js +2022 -0
- package/dist/dataIdentifiers.d.ts +8 -0
- package/dist/dataIdentifiers.d.ts.map +1 -0
- package/dist/dataIdentifiers.js +55 -0
- package/dist/dataImport.d.ts +77 -0
- package/dist/dataImport.d.ts.map +1 -0
- package/dist/dataImport.js +450 -0
- package/dist/dataRpc.d.ts +131 -0
- package/dist/dataRpc.d.ts.map +1 -0
- package/dist/dataRpc.js +2 -0
- package/dist/dataTypes.d.ts +157 -0
- package/dist/dataTypes.d.ts.map +1 -0
- package/dist/dataTypes.js +160 -0
- package/dist/index.d.ts +570 -36
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +72 -0
- package/dist/jsonSchemaToZodShape.d.ts +5 -1
- package/dist/jsonSchemaToZodShape.d.ts.map +1 -1
- package/dist/jsonSchemaToZodShape.js +21 -9
- package/dist/oauth.d.ts +14 -0
- package/dist/oauth.d.ts.map +1 -1
- package/dist/oauth.js +42 -2
- package/dist/pagination.d.ts +47 -0
- package/dist/pagination.d.ts.map +1 -0
- package/dist/pagination.js +160 -0
- package/dist/processQueueBatch.d.ts +6 -1
- package/dist/processQueueBatch.d.ts.map +1 -1
- package/dist/processQueueBatch.js +23 -10
- package/dist/queueFailure.d.ts +14 -0
- package/dist/queueFailure.d.ts.map +1 -0
- package/dist/queueFailure.js +43 -0
- package/dist/resourceRerank.d.ts +3 -2
- package/dist/resourceRerank.d.ts.map +1 -1
- package/dist/resourceRerank.js +8 -3
- package/dist/schema.d.ts +374 -29
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +290 -6
- package/dist/ssrf.d.ts +1 -0
- package/dist/ssrf.d.ts.map +1 -1
- package/dist/ssrf.js +48 -12
- package/dist/toolConfirmation.d.ts +48 -0
- package/dist/toolConfirmation.d.ts.map +1 -1
- package/dist/toolConfirmation.js +75 -1
- package/dist/toolCount.d.ts +13 -0
- package/dist/toolCount.d.ts.map +1 -0
- package/dist/toolCount.js +45 -0
- package/dist/toolLint.d.ts +1 -1
- package/dist/toolLint.d.ts.map +1 -1
- package/dist/toolLint.js +20 -0
- package/dist/untrustedContent.d.ts +135 -0
- package/dist/untrustedContent.d.ts.map +1 -0
- package/dist/untrustedContent.js +402 -0
- package/package.json +17 -1
- package/src/channelCustomer.ts +604 -0
- package/src/channelDebounce.ts +17 -0
- package/src/chunkOverlap.ts +6 -0
- package/src/chunking.ts +25 -4
- package/src/citations.ts +127 -0
- package/src/constants.ts +219 -17
- package/src/data.ts +74 -0
- package/src/dataApi.ts +428 -0
- package/src/dataCompiler.ts +948 -0
- package/src/dataDdl.ts +419 -0
- package/src/dataEngine.ts +2753 -0
- package/src/dataIdentifiers.ts +63 -0
- package/src/dataImport.ts +554 -0
- package/src/dataRpc.ts +121 -0
- package/src/dataTypes.ts +312 -0
- package/src/index.ts +156 -2
- package/src/jsonSchemaToZodShape.ts +31 -13
- package/src/oauth.ts +42 -1
- package/src/pagination.ts +193 -0
- package/src/processQueueBatch.ts +42 -12
- package/src/queueFailure.ts +63 -0
- package/src/resourceRerank.ts +10 -3
- package/src/schema.ts +346 -4
- package/src/ssrf.ts +40 -11
- package/src/toolConfirmation.ts +123 -0
- package/src/toolCount.ts +53 -0
- package/src/toolLint.ts +27 -0
- package/src/untrustedContent.ts +476 -0
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.overlapTail = void 0;
|
|
4
|
+
// The tail of the previous chunk a chunk opens with, so a sentence cut at the
|
|
5
|
+
// boundary still reads whole on one side of it. `overlap` of zero (or less)
|
|
6
|
+
// borrows nothing — `slice(-0)` is `slice(0)`, the whole previous chunk, which
|
|
7
|
+
// would double every chunk's text.
|
|
8
|
+
const overlapTail = (previous, overlap) => overlap > 0 ? previous.slice(-overlap) : '';
|
|
9
|
+
exports.overlapTail = overlapTail;
|
package/dist/chunking.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAIxC,OAAO,KAAK,EACV,iBAAiB,EACjB,yBAAyB,EAC1B,MAAM,qBAAqB,CAAC;AAE7B,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,SAAS,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC;AASpE,eAAO,MAAM,cAAc,GACzB,MAAM,MAAM,EACZ,QAAQ,MAAM,EACd,aAAY,SAAS,SAAS,EAA+B,KAC5D,MAAM,EA8BR,CAAC;AA4BF,eAAO,MAAM,SAAS,GACpB,MAAM,MAAM,EACZ,UAAU;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,KACxD,MAAM,EAAoD,CAAC;AAE9D,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,MAAM,CAAC;AAqBhD;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,GAAI,OAAO;IACtC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,KAAG,aAAa,GAAG,IAUnB,CAAC;AAqFF,eAAO,MAAM,WAAW,GAAI,OAAO;IACjC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,KAAG,MASW,CAAC;AAIhB,MAAM,WAAW,aAAc,SAAQ,OAAO,CAAC,yBAAyB,CAAC;IACvE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;CAChC;AAID,eAAO,MAAM,aAAa,GACxB,QAAQ,MAAM,EACd,WAAW,iBAAiB,EAAE,GAAG,IAAI,EACrC,iBAAiB,MAAM,GAAG,IAAI,EAC9B,UAAU;IACR,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,KACA,aAAa,EA4Cf,CAAC"}
|
package/dist/chunking.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.prepareChunks = exports.buildHeader = exports.headingSyntaxFor = exports.chunkText = exports.splitRecursive = void 0;
|
|
4
4
|
const constants_1 = require("./constants");
|
|
5
|
+
const chunkOverlap_1 = require("./chunkOverlap");
|
|
5
6
|
const sanitize_1 = require("./sanitize");
|
|
6
7
|
const trySplit = (text, sep) => {
|
|
7
8
|
if (typeof sep === 'string') {
|
|
@@ -55,7 +56,7 @@ const chunkPieces = (text, options) => {
|
|
|
55
56
|
const rawChunks = (0, exports.splitRecursive)(trimmed, target);
|
|
56
57
|
const pieces = [];
|
|
57
58
|
for (let i = 0; i < rawChunks.length; i++) {
|
|
58
|
-
const prev = i > 0 ? rawChunks[i - 1]
|
|
59
|
+
const prev = i > 0 ? (0, chunkOverlap_1.overlapTail)(rawChunks[i - 1], overlap) : '';
|
|
59
60
|
pieces.push({
|
|
60
61
|
raw: rawChunks[i],
|
|
61
62
|
content: (prev ? prev + '\n' : '') + rawChunks[i]
|
|
@@ -68,6 +69,12 @@ exports.chunkText = chunkText;
|
|
|
68
69
|
const MARKDOWN_HEADING = /^ {0,3}(#{1,6})[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$/;
|
|
69
70
|
const MARKDOWN_FENCE = /^ {0,3}(```|~~~)/;
|
|
70
71
|
const HTML_HEADING = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1\s*>/gi;
|
|
72
|
+
// A heading tag still open at the end of a piece, from its opening tag on: the
|
|
73
|
+
// chunker split inside it, and the rest of it opens the next piece.
|
|
74
|
+
const OPEN_HTML_HEADING = /<h([1-6])\b[^>]*>(?![\s\S]*<\/h\1\s*>)[\s\S]*$/i;
|
|
75
|
+
// Longer than any heading worth carrying: an opening tag this far back with no
|
|
76
|
+
// close is broken markup, not a heading that happened to be split.
|
|
77
|
+
const MAX_OPEN_HEADING_CHARS = 1000;
|
|
71
78
|
const MAX_HEADING_CHARS = 120;
|
|
72
79
|
const MARKDOWN_FILE_NAME = /\.(md|markdown|mdx)$/i;
|
|
73
80
|
/**
|
|
@@ -128,14 +135,27 @@ const scanHtmlHeadings = (text) => [...text.matchAll(HTML_HEADING)]
|
|
|
128
135
|
const createHeadingTracker = (syntax) => {
|
|
129
136
|
let stack = [];
|
|
130
137
|
const fence = { inFence: false };
|
|
138
|
+
// The unfinished HTML heading the previous piece ended in, if any.
|
|
139
|
+
let carry = '';
|
|
131
140
|
const apply = (event) => {
|
|
132
141
|
stack = stack.filter(h => h.level < event.level);
|
|
133
142
|
stack.push({ level: event.level, text: event.text });
|
|
134
143
|
};
|
|
144
|
+
// An HTML heading split between two pieces is read whole from the second,
|
|
145
|
+
// where it ends, as if that piece opened with it: it does, mid-heading. The
|
|
146
|
+
// separator the chunker dropped at the split is restored as a space.
|
|
147
|
+
const scanHtml = (raw) => {
|
|
148
|
+
const text = carry ? `${carry} ${raw}` : raw;
|
|
149
|
+
const shift = text.length - raw.length;
|
|
150
|
+
const open = OPEN_HTML_HEADING.exec(text)?.[0] ?? '';
|
|
151
|
+
carry = open.length <= MAX_OPEN_HEADING_CHARS ? open : '';
|
|
152
|
+
return scanHtmlHeadings(text).map(event => ({
|
|
153
|
+
...event,
|
|
154
|
+
offset: Math.max(0, event.offset - shift)
|
|
155
|
+
}));
|
|
156
|
+
};
|
|
135
157
|
return (raw) => {
|
|
136
|
-
const events = syntax === 'markdown'
|
|
137
|
-
? scanMarkdownHeadings(raw, fence)
|
|
138
|
-
: scanHtmlHeadings(raw);
|
|
158
|
+
const events = syntax === 'markdown' ? scanMarkdownHeadings(raw, fence) : scanHtml(raw);
|
|
139
159
|
const [first, ...rest] = events;
|
|
140
160
|
// Before a document's first heading there is no section to continue — only
|
|
141
161
|
// the metadata header, or the markup around an HTML body — so a chunk that
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Source } from './sources';
|
|
2
|
+
/**
|
|
3
|
+
* Whether the answer cites this exact URL, not merely one that starts with it:
|
|
4
|
+
* on a crawled docs site the index page is a prefix of every page beneath it,
|
|
5
|
+
* so a plain substring test kept "/docs" and the site root in the footer of an
|
|
6
|
+
* answer that cited "/docs/api". A trailing slash and a #fragment still count
|
|
7
|
+
* as the same page.
|
|
8
|
+
*/
|
|
9
|
+
export declare const citesUrl: (answer: string, url: string) => boolean;
|
|
10
|
+
/** Every page number the answer cites, ranges expanded. */
|
|
11
|
+
export declare const citedPages: (answer: string) => Set<number>;
|
|
12
|
+
/**
|
|
13
|
+
* Narrow a turn's sources to the ones its answer actually cites.
|
|
14
|
+
*
|
|
15
|
+
* Every search hit used to become a footer line, so an answer drawn from one
|
|
16
|
+
* handbook section was followed by eight unrelated pages the search had also
|
|
17
|
+
* returned. A search hit is kept when the answer names its resource — by title,
|
|
18
|
+
* file name or URL — and, for a paged file, only the pages the answer cites
|
|
19
|
+
* (or the best-ranked one, when it cites none). Resources the model read whole
|
|
20
|
+
* are always kept: reading one is itself the citation.
|
|
21
|
+
*
|
|
22
|
+
* An answer that names nothing gets no search sources at all. The model is told
|
|
23
|
+
* to cite what it uses, so a reply that names nothing is a greeting or "I
|
|
24
|
+
* couldn't find that" — and a footer under it would claim sources it never
|
|
25
|
+
* used, just because search runs on every message.
|
|
26
|
+
*/
|
|
27
|
+
export declare const selectCitedSources: (sources: Source[], answerText: string) => Source[];
|
|
28
|
+
//# sourceMappingURL=citations.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"citations.d.ts","sourceRoot":"","sources":["../src/citations.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAyCxC;;;;;;GAMG;AACH,eAAO,MAAM,QAAQ,GAAI,QAAQ,MAAM,EAAE,KAAK,MAAM,KAAG,OAetD,CAAC;AAEF,2DAA2D;AAC3D,eAAO,MAAM,UAAU,GAAI,QAAQ,MAAM,KAAG,GAAG,CAAC,MAAM,CAYrD,CAAC;AAEF;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,kBAAkB,GAC7B,SAAS,MAAM,EAAE,EACjB,YAAY,MAAM,KACjB,MAAM,EA4BR,CAAC"}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.selectCitedSources = exports.citedPages = exports.citesUrl = void 0;
|
|
4
|
+
// "p. 125", "pp. 123–126", "page 12", "págs. 4-5", "páginas 3 a 5". An
|
|
5
|
+
// abbreviation counts only with its dot and a number has to stand alone, so
|
|
6
|
+
// "p2p", "mp3" and "pp 2026" are not page references. The lookarounds, not \b,
|
|
7
|
+
// mark the edges: \b sees "á" as a word break.
|
|
8
|
+
const PAGE_REFERENCE = /(?<![\p{L}\p{N}])(?:(?:pp?|p[aá]gs?)\.|pages?|p[aá]ginas?)\s*(\d{1,5})(?![\p{L}\p{N}])(?:(?:\s*[-–—]\s*|\s+(?:to|a)\s+)(\d{1,5})(?![\p{L}\p{N}]))?/giu;
|
|
9
|
+
const MAX_PAGE_RANGE = 50;
|
|
10
|
+
// Lowercased words only, so "ETag header — HTTP | MDN" in an answer matches the
|
|
11
|
+
// title "ETag header - HTTP | MDN", and "mml-book.pdf" matches "mml-book".
|
|
12
|
+
const normalize = (text) => ` ${text
|
|
13
|
+
.toLowerCase()
|
|
14
|
+
.normalize('NFKD')
|
|
15
|
+
.replace(/[̀-ͯ]/g, '')
|
|
16
|
+
.replace(/[^a-z0-9]+/g, ' ')
|
|
17
|
+
.trim()} `;
|
|
18
|
+
// The ways an answer refers to a source: its whole title, the title without a
|
|
19
|
+
// parenthetical or a site suffix ("Northwind Handbook (draft)", "ETag header
|
|
20
|
+
// - HTTP | MDN"), its file name with and without the extension, and its URL.
|
|
21
|
+
const namesFor = (source) => {
|
|
22
|
+
const names = [source.title, source.title.replace(/\s*\([^)]*\)\s*/g, ' ')];
|
|
23
|
+
const lead = source.title.split(/\s+[|\-–—:]\s+/)[0];
|
|
24
|
+
// A one-word lead ("Introduction", "Pricing") names half the corpus.
|
|
25
|
+
if (lead.trim().split(/\s+/).length >= 2)
|
|
26
|
+
names.push(lead);
|
|
27
|
+
if (source.fileName) {
|
|
28
|
+
names.push(source.fileName, source.fileName.replace(/\.[a-z0-9]+$/i, ''));
|
|
29
|
+
}
|
|
30
|
+
return names.map(normalize).filter(name => name.trim().length >= 4);
|
|
31
|
+
};
|
|
32
|
+
// The characters that can carry a URL on past the point we matched. Brackets,
|
|
33
|
+
// quotes and whitespace end it — that is how a URL sits inside Markdown links,
|
|
34
|
+
// parentheses and prose.
|
|
35
|
+
const URL_CONTINUATION = /^[^\s<>()[\]{}"'`|]*/;
|
|
36
|
+
// Punctuation that closes a sentence around a URL rather than belonging to it.
|
|
37
|
+
const TRAILING_PUNCTUATION = /[.,;:!?*_]+$/;
|
|
38
|
+
/**
|
|
39
|
+
* Whether the answer cites this exact URL, not merely one that starts with it:
|
|
40
|
+
* on a crawled docs site the index page is a prefix of every page beneath it,
|
|
41
|
+
* so a plain substring test kept "/docs" and the site root in the footer of an
|
|
42
|
+
* answer that cited "/docs/api". A trailing slash and a #fragment still count
|
|
43
|
+
* as the same page.
|
|
44
|
+
*/
|
|
45
|
+
const citesUrl = (answer, url) => {
|
|
46
|
+
const target = url.replace(/\/+$/, '');
|
|
47
|
+
if (!target)
|
|
48
|
+
return false;
|
|
49
|
+
for (let at = answer.indexOf(target); at !== -1; at = answer.indexOf(target, at + 1)) {
|
|
50
|
+
const rest = answer
|
|
51
|
+
.slice(at + target.length)
|
|
52
|
+
.match(URL_CONTINUATION)[0]
|
|
53
|
+
.replace(TRAILING_PUNCTUATION, '');
|
|
54
|
+
if (/^\/*(#.*)?$/.test(rest))
|
|
55
|
+
return true;
|
|
56
|
+
}
|
|
57
|
+
return false;
|
|
58
|
+
};
|
|
59
|
+
exports.citesUrl = citesUrl;
|
|
60
|
+
/** Every page number the answer cites, ranges expanded. */
|
|
61
|
+
const citedPages = (answer) => {
|
|
62
|
+
const pages = new Set();
|
|
63
|
+
for (const match of answer.matchAll(PAGE_REFERENCE)) {
|
|
64
|
+
const from = Number(match[1]);
|
|
65
|
+
const to = match[2] ? Number(match[2]) : from;
|
|
66
|
+
if (to < from || to - from > MAX_PAGE_RANGE) {
|
|
67
|
+
pages.add(from);
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
for (let page = from; page <= to; page++)
|
|
71
|
+
pages.add(page);
|
|
72
|
+
}
|
|
73
|
+
return pages;
|
|
74
|
+
};
|
|
75
|
+
exports.citedPages = citedPages;
|
|
76
|
+
/**
|
|
77
|
+
* Narrow a turn's sources to the ones its answer actually cites.
|
|
78
|
+
*
|
|
79
|
+
* Every search hit used to become a footer line, so an answer drawn from one
|
|
80
|
+
* handbook section was followed by eight unrelated pages the search had also
|
|
81
|
+
* returned. A search hit is kept when the answer names its resource — by title,
|
|
82
|
+
* file name or URL — and, for a paged file, only the pages the answer cites
|
|
83
|
+
* (or the best-ranked one, when it cites none). Resources the model read whole
|
|
84
|
+
* are always kept: reading one is itself the citation.
|
|
85
|
+
*
|
|
86
|
+
* An answer that names nothing gets no search sources at all. The model is told
|
|
87
|
+
* to cite what it uses, so a reply that names nothing is a greeting or "I
|
|
88
|
+
* couldn't find that" — and a footer under it would claim sources it never
|
|
89
|
+
* used, just because search runs on every message.
|
|
90
|
+
*/
|
|
91
|
+
const selectCitedSources = (sources, answerText) => {
|
|
92
|
+
const searchEntries = sources.filter(s => s.score !== undefined);
|
|
93
|
+
const readEntries = sources.filter(s => s.score === undefined);
|
|
94
|
+
if (searchEntries.length === 0)
|
|
95
|
+
return sources;
|
|
96
|
+
const answer = normalize(answerText);
|
|
97
|
+
const pages = (0, exports.citedPages)(answerText);
|
|
98
|
+
const citedResources = new Set(searchEntries
|
|
99
|
+
.filter(s => (0, exports.citesUrl)(answerText, s.uri) ||
|
|
100
|
+
namesFor(s).some(name => answer.includes(name)))
|
|
101
|
+
.map(s => s.resourceId));
|
|
102
|
+
const kept = [];
|
|
103
|
+
for (const resourceId of citedResources) {
|
|
104
|
+
const entries = searchEntries.filter(s => s.resourceId === resourceId);
|
|
105
|
+
const onCitedPage = entries.filter(s => s.pageNumber !== undefined && pages.has(s.pageNumber));
|
|
106
|
+
kept.push(...(onCitedPage.length > 0 ? onCitedPage : entries.slice(0, 1)));
|
|
107
|
+
}
|
|
108
|
+
const cited = searchEntries.filter(s => kept.includes(s));
|
|
109
|
+
return [...cited, ...readEntries];
|
|
110
|
+
};
|
|
111
|
+
exports.selectCitedSources = selectCitedSources;
|
package/dist/constants.d.ts
CHANGED
|
@@ -62,6 +62,10 @@ interface PlanLimits {
|
|
|
62
62
|
includedMessages: number;
|
|
63
63
|
includedEmbeddedBytes: number;
|
|
64
64
|
autoSyncIntervals: ReadonlyArray<typeof RESOURCE_SYNC_INTERVAL_DAILY | typeof RESOURCE_SYNC_INTERVAL_WEEKLY>;
|
|
65
|
+
canUseData: boolean;
|
|
66
|
+
maxDataBytes: number | null;
|
|
67
|
+
maxDataTablesPerDatabase: number | null;
|
|
68
|
+
triggerRunHardCap: number | null;
|
|
65
69
|
}
|
|
66
70
|
export type { PlanLimits };
|
|
67
71
|
export declare const constants: {
|
|
@@ -94,6 +98,20 @@ export declare const constants: {
|
|
|
94
98
|
PRICING_TOOL_CALL_PER_M_USD: number;
|
|
95
99
|
PRICING_TOOL_CALL_HARD_CAP: number;
|
|
96
100
|
PRICING_CUSTOM_DOMAIN_USD: number;
|
|
101
|
+
DATA_MAX_DATABASE_BYTES: number;
|
|
102
|
+
DATA_MAX_TABLES_PER_DATABASE: number;
|
|
103
|
+
DATA_TRIGGER_RUN_HARD_CAP: number;
|
|
104
|
+
DATA_DELETE_GRACE_DAYS: number;
|
|
105
|
+
DATA_PURGE_BATCH: number;
|
|
106
|
+
DATA_ACTIVITY_THROTTLE_MS: number;
|
|
107
|
+
ARTIFACT_DATABASE_STATUS_CREATING: "CREATING";
|
|
108
|
+
ARTIFACT_DATABASE_STATUS_READY: "READY";
|
|
109
|
+
ARTIFACT_DATABASE_STATUS_FAILED: "FAILED";
|
|
110
|
+
ARTIFACT_DATABASE_STATUS_DELETING: "DELETING";
|
|
111
|
+
ARTIFACT_DATABASE_STATUS_PURGING: "PURGING";
|
|
112
|
+
ARTIFACT_DATABASE_STATUS_DELETED: "DELETED";
|
|
113
|
+
DATA_PURGE_STALE_MS: number;
|
|
114
|
+
ARTIFACT_DATABASE_STATUSES: ("FAILED" | "CREATING" | "READY" | "DELETING" | "PURGING" | "DELETED")[];
|
|
97
115
|
PLAN_FEATURE_ORGANIZATION: "organization";
|
|
98
116
|
PLAN_FEATURE_PROJECT: "project";
|
|
99
117
|
PLAN_FEATURE_TOOL: "tool";
|
|
@@ -108,6 +126,7 @@ export declare const constants: {
|
|
|
108
126
|
PLAN_FEATURE_MESSAGE: "message";
|
|
109
127
|
PLAN_FEATURE_TOOL_CALL: "toolCall";
|
|
110
128
|
PLAN_FEATURE_RESOURCE_SYNC: "resourceSync";
|
|
129
|
+
PLAN_FEATURE_DATA: "data";
|
|
111
130
|
PLAN_LIMIT_ERROR_CODE: string;
|
|
112
131
|
BILLING_METER_MESSAGES: string;
|
|
113
132
|
BILLING_METER_SHARED_MESSAGES: string;
|
|
@@ -201,6 +220,7 @@ export declare const constants: {
|
|
|
201
220
|
SLACK_OAUTH_TOKEN_URL: string;
|
|
202
221
|
OAUTH_TOKEN_URLS: Record<string, string>;
|
|
203
222
|
CREDENTIAL_REFRESH_BUFFER_MS: number;
|
|
223
|
+
CREDENTIAL_REFRESH_GIVE_UP_MS: number;
|
|
204
224
|
GOOGLE_CALENDAR_API_BASE: string;
|
|
205
225
|
GOOGLE_DRIVE_API_BASE: string;
|
|
206
226
|
GOOGLE_DRIVE_DEFAULT_PAGE_SIZE: number;
|
|
@@ -242,6 +262,7 @@ export declare const constants: {
|
|
|
242
262
|
label: string;
|
|
243
263
|
}[];
|
|
244
264
|
REAUTH_ERROR_CODES: string[];
|
|
265
|
+
OAUTH_CLIENT_ERROR_CODES: string[];
|
|
245
266
|
RESOURCE_TYPE_TOOLTIP_MIN_WIDTH: number;
|
|
246
267
|
SEARCH_DEBOUNCE_MS: number;
|
|
247
268
|
GOOGLE_DRIVE_EXPORT_MIME_TYPES: Record<string, string>;
|
|
@@ -333,6 +354,7 @@ export declare const constants: {
|
|
|
333
354
|
CRAWL_DEFAULT_MAX_DEPTH: number;
|
|
334
355
|
CRAWL_MAX_DEPTH_LIMIT: number;
|
|
335
356
|
CRAWL_PAGE_FETCH_TIMEOUT_MS: number;
|
|
357
|
+
CRAWL_MAX_REDIRECTS: number;
|
|
336
358
|
CRAWL_PAGE_QUEUE_BATCH_SIZE: number;
|
|
337
359
|
CRAWL_USER_AGENT: string;
|
|
338
360
|
RESOURCE_ICON_THEME_DARK: string;
|
|
@@ -350,6 +372,7 @@ export declare const constants: {
|
|
|
350
372
|
TELEGRAM_MAX_FILE_BYTES: number;
|
|
351
373
|
RATE_LIMIT_BACKOFF_SECONDS: number;
|
|
352
374
|
TRANSIENT_BACKOFF_SECONDS: number;
|
|
375
|
+
QUEUE_MAX_RETRIES: number;
|
|
353
376
|
ENCODINGS: string[];
|
|
354
377
|
ENCODING_UTF8: "utf-8";
|
|
355
378
|
CHANNEL_PLATFORM_TELEGRAM: "telegram";
|
|
@@ -395,6 +418,16 @@ export declare const constants: {
|
|
|
395
418
|
MCP_TOOL_CONFIRMATION_TTL_MS: number;
|
|
396
419
|
MCP_TOOL_CONFIRMATION_ARG: string;
|
|
397
420
|
MCP_TOOL_CONFIRMATION_ARG_DESCRIPTION: string;
|
|
421
|
+
MCP_TOOL_CONFIRMATION_ARG_DESCRIPTION_CONDITIONAL: string;
|
|
422
|
+
TOOL_APPROVAL_PATH: string;
|
|
423
|
+
TOOL_APPROVAL_DECISION_APPROVE: string;
|
|
424
|
+
TOOL_APPROVAL_DECISION_DENY: string;
|
|
425
|
+
TOOL_APPROVAL_DECISIONS: readonly ["approve", "deny"];
|
|
426
|
+
TOOL_APPROVAL_STATUS_PENDING: string;
|
|
427
|
+
TOOL_APPROVAL_STATUS_APPROVED: string;
|
|
428
|
+
TOOL_APPROVAL_STATUS_DENIED: string;
|
|
429
|
+
TOOL_APPROVAL_STATUS_USED: string;
|
|
430
|
+
TOOL_APPROVAL_STATUS_EXPIRED: string;
|
|
398
431
|
CHANNEL_DEBOUNCE_DEFAULT_MS: number;
|
|
399
432
|
CHANNEL_DEBOUNCE_MIN_MS: number;
|
|
400
433
|
CHANNEL_DEBOUNCE_MAX_MS: number;
|
|
@@ -465,8 +498,12 @@ export declare const constants: {
|
|
|
465
498
|
RESOURCE_SEARCH_RRF_K: number;
|
|
466
499
|
RESOURCE_SEARCH_COMMON_TERM_DIVISOR: number;
|
|
467
500
|
RESOURCE_SEARCH_COMMON_TERM_MIN: number;
|
|
501
|
+
RESOURCE_SEARCH_RERANK_ENABLED: boolean;
|
|
468
502
|
RESOURCE_SEARCH_RERANK_CANDIDATES: number;
|
|
469
503
|
RESOURCE_SEARCH_RERANK_MODEL: string;
|
|
504
|
+
RESOURCE_SEARCH_EXACT_MAX_RESOURCES: number;
|
|
505
|
+
RESOURCE_SEARCH_EXACT_MAX_CHUNKS: number;
|
|
506
|
+
RESOURCE_SEARCH_SUBSTRING_MAX_CHUNKS: number;
|
|
470
507
|
RESOURCE_HANDLER_SLEEP_AFTER: string;
|
|
471
508
|
DOCS_URL: string;
|
|
472
509
|
BASE64_DATA_URI_RE: RegExp;
|
|
@@ -477,6 +514,8 @@ export declare const constants: {
|
|
|
477
514
|
}[];
|
|
478
515
|
DEFAULT_MAX_TOKENS: number;
|
|
479
516
|
RESOURCE_TOOL_KEY_LIST_RESOURCES: string;
|
|
517
|
+
LIST_RESOURCES_DEFAULT_LIMIT: number;
|
|
518
|
+
LIST_RESOURCES_MAX_LIMIT: number;
|
|
480
519
|
RESOURCE_TOOL_KEY_SEARCH_RESOURCES: string;
|
|
481
520
|
RESOURCE_TOOL_KEY_READ_RESOURCE: string;
|
|
482
521
|
RESOURCE_TOOL_KEY_SEND_RESOURCE: string;
|
|
@@ -648,8 +687,10 @@ export declare const constants: {
|
|
|
648
687
|
CUSTOM_CODE_RESOURCE_FILE_TOO_LARGE_MESSAGE: string;
|
|
649
688
|
RESERVED_SLUGS: string[];
|
|
650
689
|
MCP_INTERNAL_HEADER: string;
|
|
690
|
+
MCP_EVAL_RERANK_HEADER: string;
|
|
651
691
|
MCP_CHANNEL_ID_HEADER: string;
|
|
652
692
|
MCP_CHANNEL_PLATFORM_HEADER: string;
|
|
693
|
+
MCP_CHANNEL_CONVERSATION_HEADER: string;
|
|
653
694
|
MCP_CHANNEL_CLIENT_USER_AGENT: string;
|
|
654
695
|
MCP_SESSION_HEADER: string;
|
|
655
696
|
MCP_REQUEST_METHOD_INITIALIZE: "initialize";
|
|
@@ -687,6 +728,14 @@ export declare const constants: {
|
|
|
687
728
|
ACCESS_TOKEN_SCOPE_MESSAGE: string;
|
|
688
729
|
CUSTOM_CODE_LOGS_DEFAULT_LIMIT: number;
|
|
689
730
|
CUSTOM_CODE_LOGS_MAX_LIMIT: number;
|
|
731
|
+
OBSERVABILITY_RANGE_DAYS: number[];
|
|
732
|
+
OBSERVABILITY_UNUSED_DAYS: number;
|
|
733
|
+
OBSERVABILITY_FOLLOW_UP_SECONDS: number;
|
|
734
|
+
OBSERVABILITY_SCHEMA_REJECTION_PREFIX: string;
|
|
735
|
+
OBSERVABILITY_TOP_ERRORS: number;
|
|
736
|
+
OBSERVABILITY_CALLS_DEFAULT_LIMIT: number;
|
|
737
|
+
OBSERVABILITY_CALLS_MAX_LIMIT: number;
|
|
738
|
+
OBSERVABILITY_SESSION_MAX_CALLS: number;
|
|
690
739
|
BOT_GRANT_TYPE: string;
|
|
691
740
|
EXTERNAL_LINK_VERIFICATION_PREFIX: string;
|
|
692
741
|
EXTERNAL_LINK_TTL_SECONDS: number;
|
package/dist/constants.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAkKA,QAAA,MAAM,yBAAyB,EAAiB,UAAU,CAAC;AAC3D,QAAA,MAAM,+BAA+B,EAAuB,gBAAgB,CAAC;AAC7E,QAAA,MAAM,8BAA8B,EAAsB,eAAe,CAAC;AAC1E,QAAA,MAAM,wBAAwB,EAAgB,SAAS,CAAC;AAuCxD,QAAA,MAAM,sBAAsB,EAAiB,UAAU,CAAC;AACxD,QAAA,MAAM,4BAA4B,EAAuB,gBAAgB,CAAC;AAC1E,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AAClD,QAAA,MAAM,oBAAoB,EAAe,QAAQ,CAAC;AA+FlD,QAAA,MAAM,4BAA4B,EAAc,OAAO,CAAC;AACxD,QAAA,MAAM,6BAA6B,EAAe,QAAQ,CAAC;AA2V3D,QAAA,MAAM,sBAAsB,EAAkB,WAAW,CAAC;AAC1D,QAAA,MAAM,mBAAmB,EAAe,QAAQ,CAAC;AAGjD,QAAA,MAAM,mBAAmB,EAAe,QAAQ,CAAC;AAocjD,MAAM,MAAM,mBAAmB,GAC3B;IACE,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,SAAS,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC9D;IACE,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAuiCpE,UAAU,UAAU;IAElB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAGhC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IAMjC,sBAAsB,EAAE,MAAM,CAAC;IAM/B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,SAAS,EAAE,OAAO,CAAC;IAKnB,eAAe,EAAE,OAAO,CAAC;IAOzB,gBAAgB,EAAE,OAAO,CAAC;IAK1B,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAQ3C,iBAAiB,EAAE,MAAM,CAAC;IAM1B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAG/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAI9B,iBAAiB,EAAE,aAAa,CAC9B,OAAO,4BAA4B,GAAG,OAAO,6BAA6B,CAC3E,CAAC;IAIF,UAAU,EAAE,OAAO,CAAC;IAGpB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,wBAAwB,EAAE,MAAM,GAAG,IAAI,CAAC;IAExC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC;AAwQD,YAAY,EAAE,UAAU,EAAE,CAAC;AAE3B,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eA9yEhB,OAAO,yBAAyB,GAChC,OAAO,+BAA+B,GACtC,OAAO,8BAA8B,GACrC,OAAO,wBAAwB;eAC5B,MAAM;;;;;;;;;;;;;;;;;;eAsCT,OAAO,sBAAsB,GAC7B,OAAO,4BAA4B,GACnC,OAAO,oBAAoB,GAC3B,OAAO,oBAAoB;eACxB,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAqcT,OAAO,mBAAmB,GAC1B,OAAO,mBAAmB,GAC1B,OAAO,sBAAsB;eAC1B,MAAM;eACN,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAyWwB,MAAM,GAAG,MAAM;cAAQ,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6lEnE,CAAC"}
|