@mastra/rag 0.0.0-error-handler-fix-20251020202607 → 0.0.0-execa-dynamic-import-20260304221256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +907 -3
- package/LICENSE.md +15 -0
- package/dist/docs/SKILL.md +38 -0
- package/dist/docs/assets/SOURCE_MAP.json +6 -0
- package/dist/docs/references/docs-rag-chunking-and-embedding.md +183 -0
- package/dist/docs/references/docs-rag-graph-rag.md +215 -0
- package/dist/docs/references/docs-rag-overview.md +72 -0
- package/dist/docs/references/docs-rag-retrieval.md +515 -0
- package/dist/docs/references/reference-rag-chunk.md +221 -0
- package/dist/docs/references/reference-rag-database-config.md +261 -0
- package/dist/docs/references/reference-rag-document.md +114 -0
- package/dist/docs/references/reference-rag-extract-params.md +168 -0
- package/dist/docs/references/reference-rag-graph-rag.md +111 -0
- package/dist/docs/references/reference-rag-rerank.md +75 -0
- package/dist/docs/references/reference-rag-rerankWithScorer.md +80 -0
- package/dist/docs/references/reference-tools-document-chunker-tool.md +89 -0
- package/dist/docs/references/reference-tools-graph-rag-tool.md +182 -0
- package/dist/docs/references/reference-tools-vector-query-tool.md +459 -0
- package/dist/document/document.d.ts +1 -1
- package/dist/document/document.d.ts.map +1 -1
- package/dist/document/extractors/index.d.ts +2 -1
- package/dist/document/extractors/index.d.ts.map +1 -1
- package/dist/document/extractors/keywords.d.ts +2 -2
- package/dist/document/extractors/keywords.d.ts.map +1 -1
- package/dist/document/extractors/questions.d.ts +2 -2
- package/dist/document/extractors/questions.d.ts.map +1 -1
- package/dist/document/extractors/schema.d.ts +13 -0
- package/dist/document/extractors/schema.d.ts.map +1 -0
- package/dist/document/extractors/summary.d.ts.map +1 -1
- package/dist/document/extractors/title.d.ts +2 -2
- package/dist/document/extractors/title.d.ts.map +1 -1
- package/dist/document/extractors/types.d.ts +13 -6
- package/dist/document/extractors/types.d.ts.map +1 -1
- package/dist/document/transformers/character.d.ts.map +1 -1
- package/dist/document/transformers/html.d.ts +1 -0
- package/dist/document/transformers/html.d.ts.map +1 -1
- package/dist/document/transformers/markdown.d.ts.map +1 -1
- package/dist/document/transformers/semantic-markdown.d.ts +6 -4
- package/dist/document/transformers/semantic-markdown.d.ts.map +1 -1
- package/dist/document/transformers/sentence.d.ts +0 -1
- package/dist/document/transformers/sentence.d.ts.map +1 -1
- package/dist/document/transformers/text.d.ts +2 -2
- package/dist/document/transformers/text.d.ts.map +1 -1
- package/dist/document/transformers/token.d.ts +5 -4
- package/dist/document/transformers/token.d.ts.map +1 -1
- package/dist/document/types.d.ts +3 -6
- package/dist/document/types.d.ts.map +1 -1
- package/dist/document/validation.d.ts.map +1 -1
- package/dist/graph-rag/index.d.ts +13 -2
- package/dist/graph-rag/index.d.ts.map +1 -1
- package/dist/index.cjs +718 -1002
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +721 -996
- package/dist/index.js.map +1 -1
- package/dist/rerank/relevance/mastra-agent/index.d.ts +2 -2
- package/dist/rerank/relevance/mastra-agent/index.d.ts.map +1 -1
- package/dist/tools/document-chunker.d.ts +3 -2
- package/dist/tools/document-chunker.d.ts.map +1 -1
- package/dist/tools/graph-rag.d.ts +2 -2
- package/dist/tools/graph-rag.d.ts.map +1 -1
- package/dist/tools/index.d.ts +1 -0
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/types.d.ts +171 -14
- package/dist/tools/types.d.ts.map +1 -1
- package/dist/tools/vector-query.d.ts +2 -2
- package/dist/tools/vector-query.d.ts.map +1 -1
- package/dist/utils/convert-sources.d.ts +3 -1
- package/dist/utils/convert-sources.d.ts.map +1 -1
- package/dist/utils/index.d.ts +1 -0
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/tool-helpers.d.ts +54 -0
- package/dist/utils/tool-helpers.d.ts.map +1 -0
- package/dist/utils/tool-schemas.d.ts +2 -2
- package/dist/utils/tool-schemas.d.ts.map +1 -1
- package/dist/utils/vector-search.d.ts.map +1 -1
- package/package.json +21 -17
- package/dist/utils/vector-prompts.d.ts +0 -32
- package/dist/utils/vector-prompts.d.ts.map +0 -1
package/dist/index.cjs
CHANGED
|
@@ -21,7 +21,7 @@ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
|
21
21
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
22
22
|
var __getProtoOf = Object.getPrototypeOf;
|
|
23
23
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
24
|
-
var __knownSymbol = (name14, symbol15) => (symbol15 = Symbol[name14]) ? symbol15 : Symbol.for("Symbol." + name14);
|
|
24
|
+
var __knownSymbol = (name14, symbol15) => (symbol15 = Symbol[name14]) ? symbol15 : /* @__PURE__ */ Symbol.for("Symbol." + name14);
|
|
25
25
|
var __typeError = (msg) => {
|
|
26
26
|
throw TypeError(msg);
|
|
27
27
|
};
|
|
@@ -85,7 +85,7 @@ var __privateSet = (obj, member, value, setter) => (__accessCheck(obj, member, "
|
|
|
85
85
|
|
|
86
86
|
// ../../node_modules/.pnpm/secure-json-parse@2.7.0/node_modules/secure-json-parse/index.js
|
|
87
87
|
var require_secure_json_parse = __commonJS({
|
|
88
|
-
"../../node_modules/.pnpm/secure-json-parse@2.7.0/node_modules/secure-json-parse/index.js"(exports, module) {
|
|
88
|
+
"../../node_modules/.pnpm/secure-json-parse@2.7.0/node_modules/secure-json-parse/index.js"(exports$1, module) {
|
|
89
89
|
var hasBuffer = typeof Buffer !== "undefined";
|
|
90
90
|
var suspectProtoRx = /"(?:_|\\u005[Ff])(?:_|\\u005[Ff])(?:p|\\u0070)(?:r|\\u0072)(?:o|\\u006[Ff])(?:t|\\u0074)(?:o|\\u006[Ff])(?:_|\\u005[Ff])(?:_|\\u005[Ff])"\s*:/;
|
|
91
91
|
var suspectConstructorRx = /"(?:c|\\u0063)(?:o|\\u006[Ff])(?:n|\\u006[Ee])(?:s|\\u0073)(?:t|\\u0074)(?:r|\\u0072)(?:u|\\u0075)(?:c|\\u0063)(?:t|\\u0074)(?:o|\\u006[Ff])(?:r|\\u0072)"\s*:/;
|
|
@@ -272,7 +272,7 @@ Provide keywords in the following comma-separated format: 'KEYWORDS: <keywords>'
|
|
|
272
272
|
var defaultQuestionExtractPrompt = new PromptTemplate({
|
|
273
273
|
templateVars: ["numQuestions", "context"],
|
|
274
274
|
template: `(
|
|
275
|
-
"Given the contextual
|
|
275
|
+
"Given the contextual information below, generate {numQuestions} questions this context can provide specific answers to which are unlikely to be found elsewhere. Higher-level summaries of surrounding context may be provided as well. "
|
|
276
276
|
"Try using these summaries to generate better questions that this context can answer."
|
|
277
277
|
"---------------------"
|
|
278
278
|
"{context}"
|
|
@@ -902,7 +902,7 @@ function loadApiKey({
|
|
|
902
902
|
}
|
|
903
903
|
return apiKey;
|
|
904
904
|
}
|
|
905
|
-
var validatorSymbol = Symbol.for("vercel.ai.validator");
|
|
905
|
+
var validatorSymbol = /* @__PURE__ */ Symbol.for("vercel.ai.validator");
|
|
906
906
|
function validator(validate) {
|
|
907
907
|
return { [validatorSymbol]: true, validate };
|
|
908
908
|
}
|
|
@@ -4141,19 +4141,20 @@ var TitleExtractor = class extends BaseExtractor {
|
|
|
4141
4141
|
const titleCandidates = await this.getTitlesCandidates(nodes);
|
|
4142
4142
|
const combinedTitles = titleCandidates.join(", ");
|
|
4143
4143
|
let title = "";
|
|
4144
|
-
if (this.llm
|
|
4144
|
+
if (agent.isSupportedLanguageModel(this.llm)) {
|
|
4145
4145
|
const miniAgent = new agent.Agent({
|
|
4146
|
+
id: "title-extractor",
|
|
4146
4147
|
model: this.llm,
|
|
4147
4148
|
name: "title-extractor",
|
|
4148
4149
|
instructions: "You are a title extractor. You are given a list of nodes and you need to extract the title from the nodes."
|
|
4149
4150
|
});
|
|
4150
|
-
const result = await miniAgent.generate(
|
|
4151
|
-
|
|
4152
|
-
|
|
4153
|
-
);
|
|
4151
|
+
const result = await miniAgent.generate([
|
|
4152
|
+
{ role: "user", content: this.combineTemplate.format({ context: combinedTitles }) }
|
|
4153
|
+
]);
|
|
4154
4154
|
title = result.text;
|
|
4155
4155
|
} else {
|
|
4156
4156
|
const miniAgent = new agent.Agent({
|
|
4157
|
+
id: "title-extractor-v1",
|
|
4157
4158
|
model: this.llm,
|
|
4158
4159
|
name: "title-extractor",
|
|
4159
4160
|
instructions: "You are a title extractor. You are given a list of nodes and you need to extract the title from the nodes."
|
|
@@ -4172,17 +4173,17 @@ var TitleExtractor = class extends BaseExtractor {
|
|
|
4172
4173
|
}
|
|
4173
4174
|
async getTitlesCandidates(nodes) {
|
|
4174
4175
|
const miniAgent = new agent.Agent({
|
|
4176
|
+
id: "titles-candidates-extractor",
|
|
4175
4177
|
model: this.llm,
|
|
4176
4178
|
name: "titles-candidates-extractor",
|
|
4177
4179
|
instructions: "You are a titles candidates extractor. You are given a list of nodes and you need to extract the titles candidates from the nodes."
|
|
4178
4180
|
});
|
|
4179
4181
|
const titleJobs = nodes.map(async (node) => {
|
|
4180
4182
|
let completion;
|
|
4181
|
-
if (this.llm
|
|
4182
|
-
const result = await miniAgent.generate(
|
|
4183
|
-
|
|
4184
|
-
|
|
4185
|
-
);
|
|
4183
|
+
if (agent.isSupportedLanguageModel(this.llm)) {
|
|
4184
|
+
const result = await miniAgent.generate([
|
|
4185
|
+
{ role: "user", content: this.nodeTemplate.format({ context: node.getContent() }) }
|
|
4186
|
+
]);
|
|
4186
4187
|
completion = result.text;
|
|
4187
4188
|
} else {
|
|
4188
4189
|
const result = await miniAgent.generateLegacy([
|
|
@@ -4239,13 +4240,14 @@ var SummaryExtractor = class extends BaseExtractor {
|
|
|
4239
4240
|
context
|
|
4240
4241
|
});
|
|
4241
4242
|
const miniAgent = new agent.Agent({
|
|
4243
|
+
id: "summary-extractor",
|
|
4242
4244
|
model: this.llm,
|
|
4243
4245
|
name: "summary-extractor",
|
|
4244
4246
|
instructions: "You are a summary extractor. You are given a node and you need to extract the summary from the node."
|
|
4245
4247
|
});
|
|
4246
4248
|
let summary = "";
|
|
4247
|
-
if (this.llm
|
|
4248
|
-
const result = await miniAgent.generate([{ role: "user", content: prompt }]
|
|
4249
|
+
if (agent.isSupportedLanguageModel(this.llm)) {
|
|
4250
|
+
const result = await miniAgent.generate([{ role: "user", content: prompt }]);
|
|
4249
4251
|
summary = result.text;
|
|
4250
4252
|
} else {
|
|
4251
4253
|
const result = await miniAgent.generateLegacy([{ role: "user", content: prompt }]);
|
|
@@ -4324,13 +4326,14 @@ var QuestionsAnsweredExtractor = class extends BaseExtractor {
|
|
|
4324
4326
|
numQuestions: this.questions.toString()
|
|
4325
4327
|
});
|
|
4326
4328
|
const miniAgent = new agent.Agent({
|
|
4329
|
+
id: "question-extractor",
|
|
4327
4330
|
model: this.llm,
|
|
4328
4331
|
name: "question-extractor",
|
|
4329
4332
|
instructions: "You are a question extractor. You are given a node and you need to extract the questions from the node."
|
|
4330
4333
|
});
|
|
4331
4334
|
let questionsText = "";
|
|
4332
|
-
if (this.llm
|
|
4333
|
-
const result2 = await miniAgent.generate([{ role: "user", content: prompt }]
|
|
4335
|
+
if (agent.isSupportedLanguageModel(this.llm)) {
|
|
4336
|
+
const result2 = await miniAgent.generate([{ role: "user", content: prompt }]);
|
|
4334
4337
|
questionsText = result2.text;
|
|
4335
4338
|
} else {
|
|
4336
4339
|
const result2 = await miniAgent.generateLegacy([{ role: "user", content: prompt }]);
|
|
@@ -4396,23 +4399,21 @@ var KeywordExtractor = class extends BaseExtractor {
|
|
|
4396
4399
|
let keywords = "";
|
|
4397
4400
|
try {
|
|
4398
4401
|
const miniAgent = new agent.Agent({
|
|
4402
|
+
id: "keyword-extractor",
|
|
4399
4403
|
model: this.llm,
|
|
4400
4404
|
name: "keyword-extractor",
|
|
4401
4405
|
instructions: "You are a keyword extractor. You are given a node and you need to extract the keywords from the node."
|
|
4402
4406
|
});
|
|
4403
|
-
if (this.llm
|
|
4404
|
-
const result = await miniAgent.generate(
|
|
4405
|
-
|
|
4406
|
-
|
|
4407
|
-
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
],
|
|
4414
|
-
{ format: "mastra" }
|
|
4415
|
-
);
|
|
4407
|
+
if (agent.isSupportedLanguageModel(this.llm)) {
|
|
4408
|
+
const result = await miniAgent.generate([
|
|
4409
|
+
{
|
|
4410
|
+
role: "user",
|
|
4411
|
+
content: this.promptTemplate.format({
|
|
4412
|
+
context: node.getContent(),
|
|
4413
|
+
maxKeywords: this.keywords.toString()
|
|
4414
|
+
})
|
|
4415
|
+
}
|
|
4416
|
+
]);
|
|
4416
4417
|
keywords = result.text;
|
|
4417
4418
|
} else {
|
|
4418
4419
|
const result = await miniAgent.generateLegacy([
|
|
@@ -4449,6 +4450,44 @@ var KeywordExtractor = class extends BaseExtractor {
|
|
|
4449
4450
|
return results;
|
|
4450
4451
|
}
|
|
4451
4452
|
};
|
|
4453
|
+
var SchemaExtractor = class extends BaseExtractor {
|
|
4454
|
+
schema;
|
|
4455
|
+
llm;
|
|
4456
|
+
instructions;
|
|
4457
|
+
metadataKey;
|
|
4458
|
+
constructor(options) {
|
|
4459
|
+
super();
|
|
4460
|
+
this.schema = options.schema;
|
|
4461
|
+
this.llm = options.llm;
|
|
4462
|
+
this.instructions = options.instructions;
|
|
4463
|
+
this.metadataKey = options.metadataKey;
|
|
4464
|
+
}
|
|
4465
|
+
async extract(nodes) {
|
|
4466
|
+
const agent$1 = new agent.Agent({
|
|
4467
|
+
name: "schema-extractor",
|
|
4468
|
+
id: "schema-extractor",
|
|
4469
|
+
instructions: this.instructions ?? "Extract structured data from the provided text.",
|
|
4470
|
+
model: this.llm ?? baseLLM
|
|
4471
|
+
});
|
|
4472
|
+
const results = await Promise.all(
|
|
4473
|
+
nodes.map(async (node) => {
|
|
4474
|
+
try {
|
|
4475
|
+
const result = await agent$1.generate([{ role: "user", content: node.getContent() }], {
|
|
4476
|
+
structuredOutput: { schema: this.schema }
|
|
4477
|
+
});
|
|
4478
|
+
if (this.metadataKey) {
|
|
4479
|
+
return { [this.metadataKey]: result.object };
|
|
4480
|
+
}
|
|
4481
|
+
return result.object;
|
|
4482
|
+
} catch (error) {
|
|
4483
|
+
console.error("Schema extraction failed:", error);
|
|
4484
|
+
return {};
|
|
4485
|
+
}
|
|
4486
|
+
})
|
|
4487
|
+
);
|
|
4488
|
+
return results;
|
|
4489
|
+
}
|
|
4490
|
+
};
|
|
4452
4491
|
|
|
4453
4492
|
// src/document/types.ts
|
|
4454
4493
|
var Language = /* @__PURE__ */ ((Language2) => {
|
|
@@ -4486,14 +4525,14 @@ var TextTransformer = class {
|
|
|
4486
4525
|
maxSize;
|
|
4487
4526
|
overlap;
|
|
4488
4527
|
lengthFunction;
|
|
4489
|
-
|
|
4528
|
+
separatorPosition;
|
|
4490
4529
|
addStartIndex;
|
|
4491
4530
|
stripWhitespace;
|
|
4492
4531
|
constructor({
|
|
4493
4532
|
maxSize = 4e3,
|
|
4494
4533
|
overlap = 200,
|
|
4495
4534
|
lengthFunction = (text) => text.length,
|
|
4496
|
-
|
|
4535
|
+
separatorPosition,
|
|
4497
4536
|
addStartIndex = false,
|
|
4498
4537
|
stripWhitespace = true
|
|
4499
4538
|
}) {
|
|
@@ -4503,7 +4542,7 @@ var TextTransformer = class {
|
|
|
4503
4542
|
this.maxSize = maxSize;
|
|
4504
4543
|
this.overlap = overlap;
|
|
4505
4544
|
this.lengthFunction = lengthFunction;
|
|
4506
|
-
this.
|
|
4545
|
+
this.separatorPosition = separatorPosition;
|
|
4507
4546
|
this.addStartIndex = addStartIndex;
|
|
4508
4547
|
this.stripWhitespace = stripWhitespace;
|
|
4509
4548
|
}
|
|
@@ -4609,11 +4648,11 @@ var TextTransformer = class {
|
|
|
4609
4648
|
};
|
|
4610
4649
|
|
|
4611
4650
|
// src/document/transformers/character.ts
|
|
4612
|
-
function splitTextWithRegex(text, separator,
|
|
4651
|
+
function splitTextWithRegex(text, separator, separatorPosition) {
|
|
4613
4652
|
if (!separator) {
|
|
4614
4653
|
return text.split("");
|
|
4615
4654
|
}
|
|
4616
|
-
if (!
|
|
4655
|
+
if (!separatorPosition) {
|
|
4617
4656
|
return text.split(new RegExp(separator)).filter((s) => s !== "");
|
|
4618
4657
|
}
|
|
4619
4658
|
if (!text) {
|
|
@@ -4621,7 +4660,7 @@ function splitTextWithRegex(text, separator, keepSeparator) {
|
|
|
4621
4660
|
}
|
|
4622
4661
|
const splits = text.split(new RegExp(`(${separator})`));
|
|
4623
4662
|
const result = [];
|
|
4624
|
-
if (
|
|
4663
|
+
if (separatorPosition === "end") {
|
|
4625
4664
|
for (let i = 0; i < splits.length - 1; i += 2) {
|
|
4626
4665
|
if (i + 1 < splits.length) {
|
|
4627
4666
|
const chunk = splits[i] + (splits[i + 1] || "");
|
|
@@ -4653,7 +4692,7 @@ var CharacterTransformer = class extends TextTransformer {
|
|
|
4653
4692
|
}
|
|
4654
4693
|
splitText({ text }) {
|
|
4655
4694
|
const separator = this.isSeparatorRegex ? this.separator : this.separator.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
4656
|
-
const initialSplits = splitTextWithRegex(text, separator, this.
|
|
4695
|
+
const initialSplits = splitTextWithRegex(text, separator, this.separatorPosition);
|
|
4657
4696
|
const chunks = [];
|
|
4658
4697
|
for (const split of initialSplits) {
|
|
4659
4698
|
if (this.lengthFunction(split) <= this.maxSize) {
|
|
@@ -4708,9 +4747,9 @@ var RecursiveCharacterTransformer = class _RecursiveCharacterTransformer extends
|
|
|
4708
4747
|
}
|
|
4709
4748
|
}
|
|
4710
4749
|
const _separator = this.isSeparatorRegex ? separator : separator?.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
4711
|
-
const splits = splitTextWithRegex(text, _separator, this.
|
|
4750
|
+
const splits = splitTextWithRegex(text, _separator, this.separatorPosition);
|
|
4712
4751
|
const goodSplits = [];
|
|
4713
|
-
const mergeSeparator = this.
|
|
4752
|
+
const mergeSeparator = this.separatorPosition ? "" : separator;
|
|
4714
4753
|
for (const s of splits) {
|
|
4715
4754
|
if (this.lengthFunction(s) < this.maxSize) {
|
|
4716
4755
|
goodSplits.push(s);
|
|
@@ -4820,7 +4859,352 @@ var RecursiveCharacterTransformer = class _RecursiveCharacterTransformer extends
|
|
|
4820
4859
|
" ",
|
|
4821
4860
|
""
|
|
4822
4861
|
];
|
|
4823
|
-
|
|
4862
|
+
case "php" /* PHP */:
|
|
4863
|
+
return [
|
|
4864
|
+
"\\nnamespace ",
|
|
4865
|
+
"\\nuse ",
|
|
4866
|
+
"\\nclass ",
|
|
4867
|
+
"\\ninterface ",
|
|
4868
|
+
"\\ntrait ",
|
|
4869
|
+
"\\nfunction ",
|
|
4870
|
+
"\\npublic ",
|
|
4871
|
+
"\\nprivate ",
|
|
4872
|
+
"\\nprotected ",
|
|
4873
|
+
"\\nif ",
|
|
4874
|
+
"\\nforeach ",
|
|
4875
|
+
"\\nfor ",
|
|
4876
|
+
"\\nwhile ",
|
|
4877
|
+
"\\nswitch ",
|
|
4878
|
+
"\\ncase ",
|
|
4879
|
+
"\\n\\n",
|
|
4880
|
+
"\\n",
|
|
4881
|
+
" ",
|
|
4882
|
+
""
|
|
4883
|
+
];
|
|
4884
|
+
case "go" /* GO */:
|
|
4885
|
+
return [
|
|
4886
|
+
"\npackage ",
|
|
4887
|
+
"\nimport ",
|
|
4888
|
+
"\nfunc ",
|
|
4889
|
+
"\ntype ",
|
|
4890
|
+
"\nstruct ",
|
|
4891
|
+
"\ninterface ",
|
|
4892
|
+
"\nconst ",
|
|
4893
|
+
"\nvar ",
|
|
4894
|
+
"\nif ",
|
|
4895
|
+
"\nfor ",
|
|
4896
|
+
"\nswitch ",
|
|
4897
|
+
"\ncase ",
|
|
4898
|
+
"\n\n",
|
|
4899
|
+
"\n",
|
|
4900
|
+
" ",
|
|
4901
|
+
""
|
|
4902
|
+
];
|
|
4903
|
+
case "java" /* JAVA */:
|
|
4904
|
+
case "scala" /* SCALA */:
|
|
4905
|
+
return [
|
|
4906
|
+
"\npackage ",
|
|
4907
|
+
"\nimport ",
|
|
4908
|
+
"\nclass ",
|
|
4909
|
+
"\ninterface ",
|
|
4910
|
+
"\nenum ",
|
|
4911
|
+
"\npublic ",
|
|
4912
|
+
"\nprotected ",
|
|
4913
|
+
"\nprivate ",
|
|
4914
|
+
"\nstatic ",
|
|
4915
|
+
"\nvoid ",
|
|
4916
|
+
"\nif ",
|
|
4917
|
+
"\nfor ",
|
|
4918
|
+
"\nwhile ",
|
|
4919
|
+
"\nswitch ",
|
|
4920
|
+
"\ncase ",
|
|
4921
|
+
"\n\n",
|
|
4922
|
+
"\n",
|
|
4923
|
+
" ",
|
|
4924
|
+
""
|
|
4925
|
+
];
|
|
4926
|
+
case "kotlin" /* KOTLIN */:
|
|
4927
|
+
return [
|
|
4928
|
+
"\npackage ",
|
|
4929
|
+
"\nimport ",
|
|
4930
|
+
"\nclass ",
|
|
4931
|
+
"\ninterface ",
|
|
4932
|
+
"\nfun ",
|
|
4933
|
+
"\nval ",
|
|
4934
|
+
"\nvar ",
|
|
4935
|
+
"\nobject ",
|
|
4936
|
+
"\ncompanion ",
|
|
4937
|
+
"\nif ",
|
|
4938
|
+
"\nfor ",
|
|
4939
|
+
"\nwhile ",
|
|
4940
|
+
"\nwhen ",
|
|
4941
|
+
"\n\n",
|
|
4942
|
+
"\n",
|
|
4943
|
+
" ",
|
|
4944
|
+
""
|
|
4945
|
+
];
|
|
4946
|
+
case "js" /* JS */:
|
|
4947
|
+
return [
|
|
4948
|
+
"\nclass ",
|
|
4949
|
+
"\nfunction ",
|
|
4950
|
+
"\nconst ",
|
|
4951
|
+
"\nlet ",
|
|
4952
|
+
"\nvar ",
|
|
4953
|
+
"\nif ",
|
|
4954
|
+
"\nfor ",
|
|
4955
|
+
"\nwhile ",
|
|
4956
|
+
"\nswitch ",
|
|
4957
|
+
"\ncase ",
|
|
4958
|
+
"\ndefault ",
|
|
4959
|
+
"\n\n",
|
|
4960
|
+
"\n",
|
|
4961
|
+
" ",
|
|
4962
|
+
""
|
|
4963
|
+
];
|
|
4964
|
+
case "python" /* PYTHON */:
|
|
4965
|
+
return [
|
|
4966
|
+
"\nclass ",
|
|
4967
|
+
"\ndef ",
|
|
4968
|
+
"\n def ",
|
|
4969
|
+
"\n def ",
|
|
4970
|
+
"\nasync def ",
|
|
4971
|
+
"\nif ",
|
|
4972
|
+
"\nfor ",
|
|
4973
|
+
"\nwhile ",
|
|
4974
|
+
"\ntry ",
|
|
4975
|
+
"\nexcept ",
|
|
4976
|
+
"\n\n",
|
|
4977
|
+
"\n",
|
|
4978
|
+
" ",
|
|
4979
|
+
""
|
|
4980
|
+
];
|
|
4981
|
+
case "ruby" /* RUBY */:
|
|
4982
|
+
return [
|
|
4983
|
+
"\nmodule ",
|
|
4984
|
+
"\nclass ",
|
|
4985
|
+
"\ndef ",
|
|
4986
|
+
"\nif ",
|
|
4987
|
+
"\nunless ",
|
|
4988
|
+
"\nwhile ",
|
|
4989
|
+
"\nfor ",
|
|
4990
|
+
"\nbegin ",
|
|
4991
|
+
"\nrescue ",
|
|
4992
|
+
"\n\n",
|
|
4993
|
+
"\n",
|
|
4994
|
+
" ",
|
|
4995
|
+
""
|
|
4996
|
+
];
|
|
4997
|
+
case "rust" /* RUST */:
|
|
4998
|
+
return [
|
|
4999
|
+
"\nmod ",
|
|
5000
|
+
"\nuse ",
|
|
5001
|
+
"\nfn ",
|
|
5002
|
+
"\nstruct ",
|
|
5003
|
+
"\nenum ",
|
|
5004
|
+
"\nimpl ",
|
|
5005
|
+
"\ntrait ",
|
|
5006
|
+
"\nconst ",
|
|
5007
|
+
"\nlet ",
|
|
5008
|
+
"\nif ",
|
|
5009
|
+
"\nfor ",
|
|
5010
|
+
"\nwhile ",
|
|
5011
|
+
"\nmatch ",
|
|
5012
|
+
"\n\n",
|
|
5013
|
+
"\n",
|
|
5014
|
+
" ",
|
|
5015
|
+
""
|
|
5016
|
+
];
|
|
5017
|
+
case "swift" /* SWIFT */:
|
|
5018
|
+
return [
|
|
5019
|
+
"\nimport ",
|
|
5020
|
+
"\nclass ",
|
|
5021
|
+
"\nstruct ",
|
|
5022
|
+
"\nenum ",
|
|
5023
|
+
"\nprotocol ",
|
|
5024
|
+
"\nfunc ",
|
|
5025
|
+
"\nvar ",
|
|
5026
|
+
"\nlet ",
|
|
5027
|
+
"\nif ",
|
|
5028
|
+
"\nfor ",
|
|
5029
|
+
"\nwhile ",
|
|
5030
|
+
"\nswitch ",
|
|
5031
|
+
"\ncase ",
|
|
5032
|
+
"\n\n",
|
|
5033
|
+
"\n",
|
|
5034
|
+
" ",
|
|
5035
|
+
""
|
|
5036
|
+
];
|
|
5037
|
+
case "html" /* HTML */:
|
|
5038
|
+
return [
|
|
5039
|
+
"<body",
|
|
5040
|
+
"<div",
|
|
5041
|
+
"<p",
|
|
5042
|
+
"<br",
|
|
5043
|
+
"<li",
|
|
5044
|
+
"<h1",
|
|
5045
|
+
"<h2",
|
|
5046
|
+
"<h3",
|
|
5047
|
+
"<h4",
|
|
5048
|
+
"<h5",
|
|
5049
|
+
"<h6",
|
|
5050
|
+
"<span",
|
|
5051
|
+
"<table",
|
|
5052
|
+
"<tr",
|
|
5053
|
+
"<td",
|
|
5054
|
+
"<th",
|
|
5055
|
+
"<ul",
|
|
5056
|
+
"<ol",
|
|
5057
|
+
"<header",
|
|
5058
|
+
"<footer",
|
|
5059
|
+
"<nav",
|
|
5060
|
+
"\n\n",
|
|
5061
|
+
"\n",
|
|
5062
|
+
" ",
|
|
5063
|
+
""
|
|
5064
|
+
];
|
|
5065
|
+
case "sol" /* SOL */:
|
|
5066
|
+
return [
|
|
5067
|
+
"\npragma ",
|
|
5068
|
+
"\nimport ",
|
|
5069
|
+
"\ncontract ",
|
|
5070
|
+
"\ninterface ",
|
|
5071
|
+
"\nlibrary ",
|
|
5072
|
+
"\nfunction ",
|
|
5073
|
+
"\nevent ",
|
|
5074
|
+
"\nmodifier ",
|
|
5075
|
+
"\nerror ",
|
|
5076
|
+
"\nstruct ",
|
|
5077
|
+
"\nenum ",
|
|
5078
|
+
"\nif ",
|
|
5079
|
+
"\nfor ",
|
|
5080
|
+
"\nwhile ",
|
|
5081
|
+
"\n\n",
|
|
5082
|
+
"\n",
|
|
5083
|
+
" ",
|
|
5084
|
+
""
|
|
5085
|
+
];
|
|
5086
|
+
case "csharp" /* CSHARP */:
|
|
5087
|
+
return [
|
|
5088
|
+
"\nnamespace ",
|
|
5089
|
+
"\nusing ",
|
|
5090
|
+
"\nclass ",
|
|
5091
|
+
"\ninterface ",
|
|
5092
|
+
"\nstruct ",
|
|
5093
|
+
"\nenum ",
|
|
5094
|
+
"\npublic ",
|
|
5095
|
+
"\nprivate ",
|
|
5096
|
+
"\nprotected ",
|
|
5097
|
+
"\ninternal ",
|
|
5098
|
+
"\nvoid ",
|
|
5099
|
+
"\nif ",
|
|
5100
|
+
"\nfor ",
|
|
5101
|
+
"\nwhile ",
|
|
5102
|
+
"\nswitch ",
|
|
5103
|
+
"\ncase ",
|
|
5104
|
+
"\n\n",
|
|
5105
|
+
"\n",
|
|
5106
|
+
" ",
|
|
5107
|
+
""
|
|
5108
|
+
];
|
|
5109
|
+
case "cobol" /* COBOL */:
|
|
5110
|
+
return [
|
|
5111
|
+
"\nIDENTIFICATION DIVISION",
|
|
5112
|
+
"\nENVIRONMENT DIVISION",
|
|
5113
|
+
"\nDATA DIVISION",
|
|
5114
|
+
"\nPROCEDURE DIVISION",
|
|
5115
|
+
"\nWORKING-STORAGE SECTION",
|
|
5116
|
+
"\nPERFORM ",
|
|
5117
|
+
"\nIF ",
|
|
5118
|
+
"\nEVALUATE ",
|
|
5119
|
+
"\n\n",
|
|
5120
|
+
"\n",
|
|
5121
|
+
" ",
|
|
5122
|
+
""
|
|
5123
|
+
];
|
|
5124
|
+
case "lua" /* LUA */:
|
|
5125
|
+
return ["\nfunction ", "\nlocal function ", "\nif ", "\nfor ", "\nwhile ", "\nrepeat ", "\n\n", "\n", " ", ""];
|
|
5126
|
+
case "perl" /* PERL */:
|
|
5127
|
+
return [
|
|
5128
|
+
"\nsub ",
|
|
5129
|
+
"\npackage ",
|
|
5130
|
+
"\nuse ",
|
|
5131
|
+
"\nmy ",
|
|
5132
|
+
"\nour ",
|
|
5133
|
+
"\nif ",
|
|
5134
|
+
"\nunless ",
|
|
5135
|
+
"\nwhile ",
|
|
5136
|
+
"\nfor ",
|
|
5137
|
+
"\nforeach ",
|
|
5138
|
+
"\n\n",
|
|
5139
|
+
"\n",
|
|
5140
|
+
" ",
|
|
5141
|
+
""
|
|
5142
|
+
];
|
|
5143
|
+
case "haskell" /* HASKELL */:
|
|
5144
|
+
return [
|
|
5145
|
+
"\nmodule ",
|
|
5146
|
+
"\nimport ",
|
|
5147
|
+
"\ndata ",
|
|
5148
|
+
"\ntype ",
|
|
5149
|
+
"\nnewtype ",
|
|
5150
|
+
"\nclass ",
|
|
5151
|
+
"\ninstance ",
|
|
5152
|
+
"\nwhere ",
|
|
5153
|
+
"\nlet ",
|
|
5154
|
+
"\nif ",
|
|
5155
|
+
"\ncase ",
|
|
5156
|
+
"\n\n",
|
|
5157
|
+
"\n",
|
|
5158
|
+
" ",
|
|
5159
|
+
""
|
|
5160
|
+
];
|
|
5161
|
+
case "elixir" /* ELIXIR */:
|
|
5162
|
+
return [
|
|
5163
|
+
"\ndefmodule ",
|
|
5164
|
+
"\ndefprotocol ",
|
|
5165
|
+
"\ndefimpl ",
|
|
5166
|
+
"\ndef ",
|
|
5167
|
+
"\ndefp ",
|
|
5168
|
+
"\ndefmacro ",
|
|
5169
|
+
"\nif ",
|
|
5170
|
+
"\ncase ",
|
|
5171
|
+
"\ncond ",
|
|
5172
|
+
"\n\n",
|
|
5173
|
+
"\n",
|
|
5174
|
+
" ",
|
|
5175
|
+
""
|
|
5176
|
+
];
|
|
5177
|
+
case "powershell" /* POWERSHELL */:
|
|
5178
|
+
return [
|
|
5179
|
+
"\nfunction ",
|
|
5180
|
+
"\nfilter ",
|
|
5181
|
+
"\nparam ",
|
|
5182
|
+
"\nif ",
|
|
5183
|
+
"\nforeach ",
|
|
5184
|
+
"\nfor ",
|
|
5185
|
+
"\nwhile ",
|
|
5186
|
+
"\nswitch ",
|
|
5187
|
+
"\n\n",
|
|
5188
|
+
"\n",
|
|
5189
|
+
" ",
|
|
5190
|
+
""
|
|
5191
|
+
];
|
|
5192
|
+
case "proto" /* PROTO */:
|
|
5193
|
+
return [
|
|
5194
|
+
"\nsyntax ",
|
|
5195
|
+
"\npackage ",
|
|
5196
|
+
"\nimport ",
|
|
5197
|
+
"\nmessage ",
|
|
5198
|
+
"\nservice ",
|
|
5199
|
+
"\nenum ",
|
|
5200
|
+
"\nrpc ",
|
|
5201
|
+
"\n\n",
|
|
5202
|
+
"\n",
|
|
5203
|
+
" ",
|
|
5204
|
+
""
|
|
5205
|
+
];
|
|
5206
|
+
case "rst" /* RST */:
|
|
5207
|
+
return ["\n=+\n", "\n-+\n", "\n\\*+\n", "\n\\.\\. ", "\n\n", "\n", " ", ""];
|
|
4824
5208
|
default:
|
|
4825
5209
|
throw new Error(`Language ${language} is not supported! Please choose from ${Object.values(Language)}`);
|
|
4826
5210
|
}
|
|
@@ -4998,23 +5382,47 @@ var HTMLSectionTransformer = class {
|
|
|
4998
5382
|
}
|
|
4999
5383
|
return "/" + parts.join("/");
|
|
5000
5384
|
}
|
|
5385
|
+
getTextContent(element) {
|
|
5386
|
+
if (!element) return "";
|
|
5387
|
+
if (!element.tagName) {
|
|
5388
|
+
return element.text || "";
|
|
5389
|
+
}
|
|
5390
|
+
let content = element.text || "";
|
|
5391
|
+
if (element.childNodes) {
|
|
5392
|
+
for (const child of element.childNodes) {
|
|
5393
|
+
const childText = this.getTextContent(child);
|
|
5394
|
+
if (childText) {
|
|
5395
|
+
content += " " + childText;
|
|
5396
|
+
}
|
|
5397
|
+
}
|
|
5398
|
+
}
|
|
5399
|
+
return content.trim();
|
|
5400
|
+
}
|
|
5001
5401
|
splitHtmlByHeaders(htmlDoc) {
|
|
5002
5402
|
const sections = [];
|
|
5003
5403
|
const root = nodeHtmlBetterParser.parse(htmlDoc);
|
|
5004
5404
|
const headers = Object.keys(this.headersToSplitOn);
|
|
5005
5405
|
const headerElements = root.querySelectorAll(headers.join(","));
|
|
5006
|
-
headerElements.forEach((headerElement
|
|
5406
|
+
headerElements.forEach((headerElement) => {
|
|
5007
5407
|
const header = headerElement.text?.trim() || "";
|
|
5008
5408
|
const tagName = headerElement.tagName;
|
|
5009
5409
|
const xpath = this.getXPath(headerElement);
|
|
5010
5410
|
let content = "";
|
|
5011
|
-
|
|
5012
|
-
|
|
5013
|
-
|
|
5014
|
-
|
|
5015
|
-
|
|
5411
|
+
const parentNode = headerElement.parentNode;
|
|
5412
|
+
if (parentNode && parentNode.childNodes) {
|
|
5413
|
+
let foundHeader = false;
|
|
5414
|
+
for (const node of parentNode.childNodes) {
|
|
5415
|
+
if (node === headerElement) {
|
|
5416
|
+
foundHeader = true;
|
|
5417
|
+
continue;
|
|
5418
|
+
}
|
|
5419
|
+
if (foundHeader && node.tagName && headers.includes(node.tagName.toLowerCase())) {
|
|
5420
|
+
break;
|
|
5421
|
+
}
|
|
5422
|
+
if (foundHeader) {
|
|
5423
|
+
content += this.getTextContent(node) + " ";
|
|
5424
|
+
}
|
|
5016
5425
|
}
|
|
5017
|
-
currentElement = currentElement.nextElementSibling;
|
|
5018
5426
|
}
|
|
5019
5427
|
content = content.trim();
|
|
5020
5428
|
sections.push({
|
|
@@ -5566,6 +5974,11 @@ var MarkdownHeaderTransformer = class {
|
|
|
5566
5974
|
currentContent.push(line);
|
|
5567
5975
|
continue;
|
|
5568
5976
|
}
|
|
5977
|
+
const isTableLine = strippedLine.includes("|") && strippedLine.length > 0;
|
|
5978
|
+
if (isTableLine) {
|
|
5979
|
+
currentContent.push(line);
|
|
5980
|
+
continue;
|
|
5981
|
+
}
|
|
5569
5982
|
let headerMatched = false;
|
|
5570
5983
|
for (const [sep, name14] of this.headersToSplitOn) {
|
|
5571
5984
|
if (strippedLine.startsWith(sep) && (strippedLine.length === sep.length || strippedLine[sep.length] === " ")) {
|
|
@@ -5659,31 +6072,34 @@ var MarkdownHeaderTransformer = class {
|
|
|
5659
6072
|
var SemanticMarkdownTransformer = class _SemanticMarkdownTransformer extends TextTransformer {
|
|
5660
6073
|
tokenizer;
|
|
5661
6074
|
joinThreshold;
|
|
5662
|
-
|
|
5663
|
-
|
|
6075
|
+
allowedArray;
|
|
6076
|
+
disallowedArray;
|
|
5664
6077
|
constructor({
|
|
5665
6078
|
joinThreshold = 500,
|
|
5666
6079
|
encodingName = "cl100k_base",
|
|
5667
6080
|
modelName,
|
|
6081
|
+
tokenizer: existingTokenizer,
|
|
5668
6082
|
allowedSpecial = /* @__PURE__ */ new Set(),
|
|
5669
6083
|
disallowedSpecial = "all",
|
|
5670
6084
|
...baseOptions
|
|
5671
6085
|
} = {}) {
|
|
5672
6086
|
super(baseOptions);
|
|
5673
6087
|
this.joinThreshold = joinThreshold;
|
|
5674
|
-
this.
|
|
5675
|
-
this.
|
|
5676
|
-
|
|
5677
|
-
this.tokenizer =
|
|
5678
|
-
}
|
|
5679
|
-
|
|
6088
|
+
this.allowedArray = allowedSpecial === "all" ? "all" : Array.from(allowedSpecial);
|
|
6089
|
+
this.disallowedArray = disallowedSpecial === "all" ? "all" : Array.from(disallowedSpecial);
|
|
6090
|
+
if (existingTokenizer) {
|
|
6091
|
+
this.tokenizer = existingTokenizer;
|
|
6092
|
+
} else {
|
|
6093
|
+
try {
|
|
6094
|
+
this.tokenizer = modelName ? jsTiktoken.encodingForModel(modelName) : jsTiktoken.getEncoding(encodingName);
|
|
6095
|
+
} catch {
|
|
6096
|
+
throw new Error("Could not load tiktoken encoding. Please install it with `npm install js-tiktoken`.");
|
|
6097
|
+
}
|
|
5680
6098
|
}
|
|
5681
6099
|
}
|
|
5682
6100
|
countTokens(text) {
|
|
5683
|
-
const allowed = this.allowedSpecial === "all" ? "all" : Array.from(this.allowedSpecial);
|
|
5684
|
-
const disallowed = this.disallowedSpecial === "all" ? "all" : Array.from(this.disallowedSpecial);
|
|
5685
6101
|
const processedText = this.stripWhitespace ? text.trim() : text;
|
|
5686
|
-
return this.tokenizer.encode(processedText,
|
|
6102
|
+
return this.tokenizer.encode(processedText, this.allowedArray, this.disallowedArray).length;
|
|
5687
6103
|
}
|
|
5688
6104
|
splitMarkdownByHeaders(markdown) {
|
|
5689
6105
|
const sections = [];
|
|
@@ -5737,14 +6153,21 @@ var SemanticMarkdownTransformer = class _SemanticMarkdownTransformer extends Tex
|
|
|
5737
6153
|
const current = workingSections[j];
|
|
5738
6154
|
if (current.depth === depth) {
|
|
5739
6155
|
const prev = workingSections[j - 1];
|
|
5740
|
-
|
|
5741
|
-
|
|
5742
|
-
const formattedTitle = `
|
|
6156
|
+
const title = `${"#".repeat(current.depth)} ${current.title}`;
|
|
6157
|
+
const formattedTitle = `
|
|
5743
6158
|
|
|
5744
6159
|
${title}`;
|
|
6160
|
+
const headerLength = this.tokenizer.encode(
|
|
6161
|
+
`${formattedTitle}
|
|
6162
|
+
`,
|
|
6163
|
+
this.allowedArray,
|
|
6164
|
+
this.disallowedArray
|
|
6165
|
+
).length;
|
|
6166
|
+
const mergedLength = prev.length + current.length + headerLength;
|
|
6167
|
+
if (mergedLength < this.joinThreshold && prev.depth <= current.depth) {
|
|
5745
6168
|
prev.content += `${formattedTitle}
|
|
5746
6169
|
${current.content}`;
|
|
5747
|
-
prev.length =
|
|
6170
|
+
prev.length = mergedLength;
|
|
5748
6171
|
workingSections.splice(j, 1);
|
|
5749
6172
|
j--;
|
|
5750
6173
|
}
|
|
@@ -5814,6 +6237,7 @@ ${section.content}`;
|
|
|
5814
6237
|
...options,
|
|
5815
6238
|
encodingName,
|
|
5816
6239
|
modelName,
|
|
6240
|
+
tokenizer,
|
|
5817
6241
|
lengthFunction: tikTokenCounter
|
|
5818
6242
|
});
|
|
5819
6243
|
}
|
|
@@ -5827,7 +6251,6 @@ var SentenceTransformer = class extends TextTransformer {
|
|
|
5827
6251
|
sentenceEnders;
|
|
5828
6252
|
fallbackToWords;
|
|
5829
6253
|
fallbackToCharacters;
|
|
5830
|
-
keepSeparator;
|
|
5831
6254
|
constructor(options) {
|
|
5832
6255
|
const parentOverlap = Math.min(options.overlap ?? 0, options.maxSize - 1);
|
|
5833
6256
|
const baseOptions = {
|
|
@@ -5842,7 +6265,6 @@ var SentenceTransformer = class extends TextTransformer {
|
|
|
5842
6265
|
this.sentenceEnders = options.sentenceEnders ?? [".", "!", "?"];
|
|
5843
6266
|
this.fallbackToWords = options.fallbackToWords ?? true;
|
|
5844
6267
|
this.fallbackToCharacters = options.fallbackToCharacters ?? true;
|
|
5845
|
-
this.keepSeparator = options.keepSeparator ?? false;
|
|
5846
6268
|
this.overlap = options.overlap ?? 0;
|
|
5847
6269
|
}
|
|
5848
6270
|
detectSentenceBoundaries(text) {
|
|
@@ -6070,30 +6492,33 @@ function splitTextOnTokens({ text, tokenizer }) {
|
|
|
6070
6492
|
}
|
|
6071
6493
|
var TokenTransformer = class _TokenTransformer extends TextTransformer {
|
|
6072
6494
|
tokenizer;
|
|
6073
|
-
|
|
6074
|
-
|
|
6495
|
+
allowedArray;
|
|
6496
|
+
disallowedArray;
|
|
6075
6497
|
constructor({
|
|
6076
6498
|
encodingName = "cl100k_base",
|
|
6077
6499
|
modelName,
|
|
6500
|
+
tokenizer: existingTokenizer,
|
|
6078
6501
|
allowedSpecial = /* @__PURE__ */ new Set(),
|
|
6079
6502
|
disallowedSpecial = "all",
|
|
6080
6503
|
options = {}
|
|
6081
6504
|
}) {
|
|
6082
6505
|
super(options);
|
|
6083
|
-
|
|
6084
|
-
this.tokenizer =
|
|
6085
|
-
}
|
|
6086
|
-
|
|
6506
|
+
if (existingTokenizer) {
|
|
6507
|
+
this.tokenizer = existingTokenizer;
|
|
6508
|
+
} else {
|
|
6509
|
+
try {
|
|
6510
|
+
this.tokenizer = modelName ? jsTiktoken.encodingForModel(modelName) : jsTiktoken.getEncoding(encodingName);
|
|
6511
|
+
} catch {
|
|
6512
|
+
throw new Error("Could not load tiktoken encoding. Please install it with `npm install js-tiktoken`.");
|
|
6513
|
+
}
|
|
6087
6514
|
}
|
|
6088
|
-
this.
|
|
6089
|
-
this.
|
|
6515
|
+
this.allowedArray = allowedSpecial === "all" ? "all" : Array.from(allowedSpecial);
|
|
6516
|
+
this.disallowedArray = disallowedSpecial === "all" ? "all" : Array.from(disallowedSpecial);
|
|
6090
6517
|
}
|
|
6091
6518
|
splitText({ text }) {
|
|
6092
6519
|
const encode = (text2) => {
|
|
6093
|
-
const allowed = this.allowedSpecial === "all" ? "all" : Array.from(this.allowedSpecial);
|
|
6094
|
-
const disallowed = this.disallowedSpecial === "all" ? "all" : Array.from(this.disallowedSpecial);
|
|
6095
6520
|
const processedText = this.stripWhitespace ? text2.trim() : text2;
|
|
6096
|
-
return Array.from(this.tokenizer.encode(processedText,
|
|
6521
|
+
return Array.from(this.tokenizer.encode(processedText, this.allowedArray, this.disallowedArray));
|
|
6097
6522
|
};
|
|
6098
6523
|
const decode = (tokens) => {
|
|
6099
6524
|
const text2 = this.tokenizer.decode(tokens);
|
|
@@ -6130,6 +6555,7 @@ var TokenTransformer = class _TokenTransformer extends TextTransformer {
|
|
|
6130
6555
|
return new _TokenTransformer({
|
|
6131
6556
|
encodingName,
|
|
6132
6557
|
modelName,
|
|
6558
|
+
tokenizer,
|
|
6133
6559
|
allowedSpecial: options.allowedSpecial,
|
|
6134
6560
|
disallowedSpecial: options.disallowedSpecial,
|
|
6135
6561
|
options: {
|
|
@@ -6140,24 +6566,11 @@ var TokenTransformer = class _TokenTransformer extends TextTransformer {
|
|
|
6140
6566
|
});
|
|
6141
6567
|
}
|
|
6142
6568
|
};
|
|
6143
|
-
function handleDeprecatedSize(data) {
|
|
6144
|
-
if (data.size !== void 0) {
|
|
6145
|
-
console.warn(
|
|
6146
|
-
"[DEPRECATION] `size` is deprecated. Use `maxSize` instead. This will be removed in the next major version."
|
|
6147
|
-
);
|
|
6148
|
-
if (data.maxSize === void 0) {
|
|
6149
|
-
data.maxSize = data.size;
|
|
6150
|
-
}
|
|
6151
|
-
}
|
|
6152
|
-
const { size, ...rest } = data;
|
|
6153
|
-
return rest;
|
|
6154
|
-
}
|
|
6155
6569
|
var baseChunkOptionsSchema = zod.z.object({
|
|
6156
|
-
size: zod.z.number().positive().optional(),
|
|
6157
6570
|
maxSize: zod.z.number().positive().optional(),
|
|
6158
6571
|
overlap: zod.z.number().min(0).optional(),
|
|
6159
|
-
lengthFunction: zod.z.
|
|
6160
|
-
|
|
6572
|
+
lengthFunction: zod.z.optional(zod.z.function()),
|
|
6573
|
+
separatorPosition: zod.z.enum(["start", "end"]).optional(),
|
|
6161
6574
|
addStartIndex: zod.z.boolean().optional(),
|
|
6162
6575
|
stripWhitespace: zod.z.boolean().optional()
|
|
6163
6576
|
});
|
|
@@ -6214,15 +6627,15 @@ var semanticMarkdownChunkOptionsSchema = baseChunkOptionsSchema.extend({
|
|
|
6214
6627
|
}).strict();
|
|
6215
6628
|
var latexChunkOptionsSchema = baseChunkOptionsSchema.strict();
|
|
6216
6629
|
var validationSchemas = {
|
|
6217
|
-
character: characterChunkOptionsSchema
|
|
6218
|
-
recursive: recursiveChunkOptionsSchema
|
|
6219
|
-
sentence: sentenceChunkOptionsSchema
|
|
6220
|
-
token: tokenChunkOptionsSchema
|
|
6221
|
-
json: jsonChunkOptionsSchema
|
|
6222
|
-
html: htmlChunkOptionsSchema
|
|
6223
|
-
markdown: markdownChunkOptionsSchema
|
|
6224
|
-
"semantic-markdown": semanticMarkdownChunkOptionsSchema
|
|
6225
|
-
latex: latexChunkOptionsSchema
|
|
6630
|
+
character: characterChunkOptionsSchema,
|
|
6631
|
+
recursive: recursiveChunkOptionsSchema,
|
|
6632
|
+
sentence: sentenceChunkOptionsSchema,
|
|
6633
|
+
token: tokenChunkOptionsSchema,
|
|
6634
|
+
json: jsonChunkOptionsSchema,
|
|
6635
|
+
html: htmlChunkOptionsSchema,
|
|
6636
|
+
markdown: markdownChunkOptionsSchema,
|
|
6637
|
+
"semantic-markdown": semanticMarkdownChunkOptionsSchema,
|
|
6638
|
+
latex: latexChunkOptionsSchema
|
|
6226
6639
|
};
|
|
6227
6640
|
function validateChunkParams(strategy, params) {
|
|
6228
6641
|
const schema = validationSchemas[strategy];
|
|
@@ -6252,8 +6665,11 @@ var MDocument = class _MDocument {
|
|
|
6252
6665
|
});
|
|
6253
6666
|
this.type = type;
|
|
6254
6667
|
}
|
|
6255
|
-
async extractMetadata({ title, summary, questions, keywords }) {
|
|
6668
|
+
async extractMetadata({ title, summary, questions, keywords, schema }) {
|
|
6256
6669
|
const transformations = [];
|
|
6670
|
+
if (schema) {
|
|
6671
|
+
transformations.push(new SchemaExtractor(schema));
|
|
6672
|
+
}
|
|
6257
6673
|
if (typeof summary !== "undefined") {
|
|
6258
6674
|
transformations.push(new SummaryExtractor(typeof summary === "boolean" ? {} : summary));
|
|
6259
6675
|
}
|
|
@@ -6399,13 +6815,33 @@ var MDocument = class _MDocument {
|
|
|
6399
6815
|
async chunkHTML(options) {
|
|
6400
6816
|
if (options?.headers?.length) {
|
|
6401
6817
|
const rt = new HTMLHeaderTransformer(options);
|
|
6402
|
-
|
|
6818
|
+
let textSplit = rt.transformDocuments(this.chunks);
|
|
6819
|
+
if (options?.maxSize) {
|
|
6820
|
+
const textSplitter = new RecursiveCharacterTransformer({
|
|
6821
|
+
maxSize: options.maxSize,
|
|
6822
|
+
overlap: options.overlap,
|
|
6823
|
+
separatorPosition: options.separatorPosition,
|
|
6824
|
+
addStartIndex: options.addStartIndex,
|
|
6825
|
+
stripWhitespace: options.stripWhitespace
|
|
6826
|
+
});
|
|
6827
|
+
textSplit = textSplitter.splitDocuments(textSplit);
|
|
6828
|
+
}
|
|
6403
6829
|
this.chunks = textSplit;
|
|
6404
6830
|
return;
|
|
6405
6831
|
}
|
|
6406
6832
|
if (options?.sections?.length) {
|
|
6407
6833
|
const rt = new HTMLSectionTransformer(options);
|
|
6408
|
-
|
|
6834
|
+
let textSplit = rt.transformDocuments(this.chunks);
|
|
6835
|
+
if (options?.maxSize) {
|
|
6836
|
+
const textSplitter = new RecursiveCharacterTransformer({
|
|
6837
|
+
maxSize: options.maxSize,
|
|
6838
|
+
overlap: options.overlap,
|
|
6839
|
+
separatorPosition: options.separatorPosition,
|
|
6840
|
+
addStartIndex: options.addStartIndex,
|
|
6841
|
+
stripWhitespace: options.stripWhitespace
|
|
6842
|
+
});
|
|
6843
|
+
textSplit = textSplitter.splitDocuments(textSplit);
|
|
6844
|
+
}
|
|
6409
6845
|
this.chunks = textSplit;
|
|
6410
6846
|
return;
|
|
6411
6847
|
}
|
|
@@ -6463,7 +6899,7 @@ var MDocument = class _MDocument {
|
|
|
6463
6899
|
sentenceEnders: options?.sentenceEnders,
|
|
6464
6900
|
fallbackToWords: options?.fallbackToWords,
|
|
6465
6901
|
fallbackToCharacters: options?.fallbackToCharacters,
|
|
6466
|
-
|
|
6902
|
+
separatorPosition: options?.separatorPosition,
|
|
6467
6903
|
lengthFunction: options?.lengthFunction,
|
|
6468
6904
|
addStartIndex: options?.addStartIndex,
|
|
6469
6905
|
stripWhitespace: options?.stripWhitespace
|
|
@@ -6538,6 +6974,7 @@ var MastraAgentRelevanceScorer = class {
|
|
|
6538
6974
|
agent;
|
|
6539
6975
|
constructor(name14, model) {
|
|
6540
6976
|
this.agent = new agent.Agent({
|
|
6977
|
+
id: `relevance-scorer-${name14}`,
|
|
6541
6978
|
name: `Relevance Scorer ${name14}`,
|
|
6542
6979
|
instructions: `You are a specialized agent for evaluating the relevance of text to queries.
|
|
6543
6980
|
Your task is to rate how well a text passage answers a given query.
|
|
@@ -6556,7 +6993,7 @@ Always return just the number, no explanation.`,
|
|
|
6556
6993
|
const prompt = relevance.createSimilarityPrompt(query, text);
|
|
6557
6994
|
const model = await this.agent.getModel();
|
|
6558
6995
|
let response;
|
|
6559
|
-
if (model
|
|
6996
|
+
if (agent.isSupportedLanguageModel(model)) {
|
|
6560
6997
|
response = await this.agent.generate(prompt);
|
|
6561
6998
|
} else {
|
|
6562
6999
|
response = await this.agent.generateLegacy(prompt);
|
|
@@ -6814,7 +7251,7 @@ var GraphRAG = class {
|
|
|
6814
7251
|
return neighbors[neighbors.length - 1]?.id;
|
|
6815
7252
|
}
|
|
6816
7253
|
// Perform random walk with restart
|
|
6817
|
-
randomWalkWithRestart(startNodeId, steps, restartProb) {
|
|
7254
|
+
randomWalkWithRestart(startNodeId, steps, restartProb, allowedNodeIds) {
|
|
6818
7255
|
const visits = /* @__PURE__ */ new Map();
|
|
6819
7256
|
let currentNodeId = startNodeId;
|
|
6820
7257
|
for (let step = 0; step < steps; step++) {
|
|
@@ -6823,7 +7260,10 @@ var GraphRAG = class {
|
|
|
6823
7260
|
currentNodeId = startNodeId;
|
|
6824
7261
|
continue;
|
|
6825
7262
|
}
|
|
6826
|
-
|
|
7263
|
+
let neighbors = this.getNeighbors(currentNodeId);
|
|
7264
|
+
if (allowedNodeIds) {
|
|
7265
|
+
neighbors = neighbors.filter((n) => allowedNodeIds.has(n.id));
|
|
7266
|
+
}
|
|
6827
7267
|
if (neighbors.length === 0) {
|
|
6828
7268
|
currentNodeId = startNodeId;
|
|
6829
7269
|
continue;
|
|
@@ -6837,12 +7277,22 @@ var GraphRAG = class {
|
|
|
6837
7277
|
}
|
|
6838
7278
|
return normalizedVisits;
|
|
6839
7279
|
}
|
|
7280
|
+
/**
|
|
7281
|
+
* Query the graph with a dense embedding and optional metadata filter.
|
|
7282
|
+
*
|
|
7283
|
+
* @param query - The embedding vector to query.
|
|
7284
|
+
* @param topK - Number of top results to return.
|
|
7285
|
+
* @param randomWalkSteps - Steps for random walk reranking.
|
|
7286
|
+
* @param restartProb - Restart probability for random walk.
|
|
7287
|
+
* @param filter - Optional strict metadata filter. All key-value pairs must match exactly.
|
|
7288
|
+
*/
|
|
6840
7289
|
// Retrieve relevant nodes using hybrid approach
|
|
6841
7290
|
query({
|
|
6842
7291
|
query,
|
|
6843
7292
|
topK = 10,
|
|
6844
7293
|
randomWalkSteps = 100,
|
|
6845
|
-
restartProb = 0.15
|
|
7294
|
+
restartProb = 0.15,
|
|
7295
|
+
filter
|
|
6846
7296
|
}) {
|
|
6847
7297
|
if (!query || query.length !== this.dimension) {
|
|
6848
7298
|
throw new Error(`Query embedding must have dimension ${this.dimension}`);
|
|
@@ -6856,15 +7306,20 @@ var GraphRAG = class {
|
|
|
6856
7306
|
if (restartProb <= 0 || restartProb >= 1) {
|
|
6857
7307
|
throw new Error("Restart probability must be between 0 and 1");
|
|
6858
7308
|
}
|
|
6859
|
-
const
|
|
7309
|
+
const filterEntries = Object.entries(filter ?? {});
|
|
7310
|
+
const matchesFilter = (node) => filterEntries.length === 0 ? true : filterEntries.every(([key, value]) => node.metadata?.[key] === value);
|
|
7311
|
+
const nodesToSearch = Array.from(this.nodes.values()).filter(matchesFilter);
|
|
7312
|
+
const similarities = nodesToSearch.map((node) => ({
|
|
6860
7313
|
node,
|
|
6861
7314
|
similarity: this.cosineSimilarity(query, node.embedding)
|
|
6862
7315
|
}));
|
|
6863
7316
|
similarities.sort((a, b) => b.similarity - a.similarity);
|
|
6864
7317
|
const topNodes = similarities.slice(0, topK);
|
|
7318
|
+
const useFilter = filterEntries.length > 0;
|
|
7319
|
+
const allowedNodeIds = useFilter ? new Set(nodesToSearch.map((n) => n.id)) : void 0;
|
|
6865
7320
|
const rerankedNodes = /* @__PURE__ */ new Map();
|
|
6866
7321
|
for (const { node, similarity } of topNodes) {
|
|
6867
|
-
const walkScores = this.randomWalkWithRestart(node.id, randomWalkSteps, restartProb);
|
|
7322
|
+
const walkScores = this.randomWalkWithRestart(node.id, randomWalkSteps, restartProb, allowedNodeIds);
|
|
6868
7323
|
for (const [nodeId, walkScore] of walkScores) {
|
|
6869
7324
|
const node2 = this.nodes.get(nodeId);
|
|
6870
7325
|
const existingScore = rerankedNodes.get(nodeId)?.score || 0;
|
|
@@ -6924,11 +7379,20 @@ var vectorQuerySearch = async ({
|
|
|
6924
7379
|
providerOptions
|
|
6925
7380
|
}) => {
|
|
6926
7381
|
let embeddingResult;
|
|
6927
|
-
if (model.specificationVersion === "
|
|
7382
|
+
if (model.specificationVersion === "v3") {
|
|
7383
|
+
embeddingResult = await vector.embedV3({
|
|
7384
|
+
model,
|
|
7385
|
+
value: queryText,
|
|
7386
|
+
maxRetries,
|
|
7387
|
+
// Type assertion needed: providerOptions type is a union, but embedV3 expects specific version
|
|
7388
|
+
...providerOptions && { providerOptions }
|
|
7389
|
+
});
|
|
7390
|
+
} else if (model.specificationVersion === "v2") {
|
|
6928
7391
|
embeddingResult = await vector.embedV2({
|
|
6929
7392
|
model,
|
|
6930
7393
|
value: queryText,
|
|
6931
7394
|
maxRetries,
|
|
7395
|
+
// Type assertion needed: providerOptions type is a union, but embedV2 expects specific version
|
|
6932
7396
|
...providerOptions && { providerOptions }
|
|
6933
7397
|
});
|
|
6934
7398
|
} else {
|
|
@@ -7052,31 +7516,131 @@ var filterSchema = zod.z.object({
|
|
|
7052
7516
|
filter: zod.z.coerce.string().describe(filterDescription)
|
|
7053
7517
|
});
|
|
7054
7518
|
|
|
7519
|
+
// src/utils/tool-helpers.ts
|
|
7520
|
+
function isValidMastraVector(value) {
|
|
7521
|
+
return value !== null && value !== void 0 && typeof value === "object";
|
|
7522
|
+
}
|
|
7523
|
+
function buildContextString(context) {
|
|
7524
|
+
const parts = [];
|
|
7525
|
+
if (context.vectorStoreName) {
|
|
7526
|
+
parts.push(`vectorStoreName="${context.vectorStoreName}"`);
|
|
7527
|
+
}
|
|
7528
|
+
if (context.requestContext) {
|
|
7529
|
+
const schemaId = context.requestContext.get?.("schemaId");
|
|
7530
|
+
const tenantId = context.requestContext.get?.("tenantId");
|
|
7531
|
+
if (schemaId) parts.push(`schemaId="${schemaId}"`);
|
|
7532
|
+
if (tenantId) parts.push(`tenantId="${tenantId}"`);
|
|
7533
|
+
}
|
|
7534
|
+
if (context.mastra) {
|
|
7535
|
+
parts.push("mastra=provided");
|
|
7536
|
+
}
|
|
7537
|
+
return parts.length > 0 ? ` (context: ${parts.join(", ")})` : "";
|
|
7538
|
+
}
|
|
7539
|
+
async function resolveVectorStore(options, context, logger) {
|
|
7540
|
+
const { requestContext, mastra, vectorStoreName, fallbackOnInvalid = false } = context;
|
|
7541
|
+
if ("vectorStore" in options && options.vectorStore !== void 0) {
|
|
7542
|
+
const vectorStoreOption = options.vectorStore;
|
|
7543
|
+
if (typeof vectorStoreOption === "function") {
|
|
7544
|
+
const resolved = await vectorStoreOption({ requestContext, mastra });
|
|
7545
|
+
if (!isValidMastraVector(resolved)) {
|
|
7546
|
+
const contextStr = buildContextString(context);
|
|
7547
|
+
const receivedType = resolved === null ? "null" : resolved === void 0 ? "undefined" : typeof resolved;
|
|
7548
|
+
if (fallbackOnInvalid) {
|
|
7549
|
+
if (mastra && vectorStoreName) {
|
|
7550
|
+
return mastra.getVector(vectorStoreName);
|
|
7551
|
+
}
|
|
7552
|
+
return void 0;
|
|
7553
|
+
}
|
|
7554
|
+
throw new Error(
|
|
7555
|
+
`VectorStoreResolver returned invalid value: expected MastraVector instance, got ${receivedType}${contextStr}`
|
|
7556
|
+
);
|
|
7557
|
+
}
|
|
7558
|
+
return resolved;
|
|
7559
|
+
}
|
|
7560
|
+
if (!isValidMastraVector(vectorStoreOption)) {
|
|
7561
|
+
const contextStr = buildContextString(context);
|
|
7562
|
+
const receivedType = vectorStoreOption === null ? "null" : vectorStoreOption === void 0 ? "undefined" : typeof vectorStoreOption;
|
|
7563
|
+
if (fallbackOnInvalid) {
|
|
7564
|
+
if (mastra && vectorStoreName) {
|
|
7565
|
+
return mastra.getVector(vectorStoreName);
|
|
7566
|
+
}
|
|
7567
|
+
return void 0;
|
|
7568
|
+
}
|
|
7569
|
+
throw new Error(`vectorStore option is not a valid MastraVector instance: got ${receivedType}${contextStr}`);
|
|
7570
|
+
}
|
|
7571
|
+
return vectorStoreOption;
|
|
7572
|
+
}
|
|
7573
|
+
if (mastra) {
|
|
7574
|
+
return mastra.getVector(vectorStoreName);
|
|
7575
|
+
}
|
|
7576
|
+
return void 0;
|
|
7577
|
+
}
|
|
7578
|
+
function coerceTopK(topK, defaultValue = 10) {
|
|
7579
|
+
if (typeof topK === "number") {
|
|
7580
|
+
if (Number.isFinite(topK) && topK > 0) {
|
|
7581
|
+
return topK;
|
|
7582
|
+
}
|
|
7583
|
+
return defaultValue;
|
|
7584
|
+
}
|
|
7585
|
+
if (typeof topK === "string") {
|
|
7586
|
+
const parsed = Number(topK);
|
|
7587
|
+
if (Number.isFinite(parsed) && parsed > 0) {
|
|
7588
|
+
return parsed;
|
|
7589
|
+
}
|
|
7590
|
+
return defaultValue;
|
|
7591
|
+
}
|
|
7592
|
+
return defaultValue;
|
|
7593
|
+
}
|
|
7594
|
+
function parseFilterValue(filter, logger) {
|
|
7595
|
+
if (!filter) {
|
|
7596
|
+
return {};
|
|
7597
|
+
}
|
|
7598
|
+
if (typeof filter === "string") {
|
|
7599
|
+
try {
|
|
7600
|
+
return JSON.parse(filter);
|
|
7601
|
+
} catch (error) {
|
|
7602
|
+
if (logger) {
|
|
7603
|
+
logger.error("Invalid filter", { filter, error });
|
|
7604
|
+
}
|
|
7605
|
+
throw new Error(`Invalid filter format: ${error instanceof Error ? error.message : String(error)}`);
|
|
7606
|
+
}
|
|
7607
|
+
}
|
|
7608
|
+
if (typeof filter !== "object" || filter === null || Array.isArray(filter)) {
|
|
7609
|
+
if (logger) {
|
|
7610
|
+
logger.error("Invalid filter", { filter, error: "Filter must be a plain object" });
|
|
7611
|
+
}
|
|
7612
|
+
throw new Error(
|
|
7613
|
+
`Invalid filter format: expected a plain object, got ${Array.isArray(filter) ? "array" : typeof filter}`
|
|
7614
|
+
);
|
|
7615
|
+
}
|
|
7616
|
+
return filter;
|
|
7617
|
+
}
|
|
7618
|
+
|
|
7055
7619
|
// src/utils/convert-sources.ts
|
|
7056
7620
|
var convertToSources = (results) => {
|
|
7057
7621
|
return results.map((result) => {
|
|
7058
7622
|
if ("content" in result) {
|
|
7059
7623
|
return {
|
|
7060
|
-
id: result.id,
|
|
7624
|
+
id: result.id || "",
|
|
7061
7625
|
vector: result.embedding || [],
|
|
7062
|
-
score: result.score,
|
|
7626
|
+
score: result.score || 0,
|
|
7063
7627
|
metadata: result.metadata,
|
|
7064
7628
|
document: result.content || ""
|
|
7065
7629
|
};
|
|
7066
7630
|
}
|
|
7067
7631
|
if ("result" in result) {
|
|
7068
7632
|
return {
|
|
7069
|
-
id: result.result.id,
|
|
7633
|
+
id: result.result.id || "",
|
|
7070
7634
|
vector: result.result.vector || [],
|
|
7071
|
-
score: result.score,
|
|
7635
|
+
score: result.score || 0,
|
|
7072
7636
|
metadata: result.result.metadata,
|
|
7073
7637
|
document: result.result.document || ""
|
|
7074
7638
|
};
|
|
7075
7639
|
}
|
|
7076
7640
|
return {
|
|
7077
|
-
id: result.id,
|
|
7641
|
+
id: result.id || "",
|
|
7078
7642
|
vector: result.vector || [],
|
|
7079
|
-
score: result.score,
|
|
7643
|
+
score: result.score || 0,
|
|
7080
7644
|
metadata: result.metadata,
|
|
7081
7645
|
document: result.document || ""
|
|
7082
7646
|
};
|
|
@@ -7094,7 +7658,8 @@ var defaultGraphOptions = {
|
|
|
7094
7658
|
// src/tools/graph-rag.ts
|
|
7095
7659
|
var createGraphRAGTool = (options) => {
|
|
7096
7660
|
const { model, id, description } = options;
|
|
7097
|
-
const
|
|
7661
|
+
const storeName = options["vectorStoreName"] ? options.vectorStoreName : "DirectVectorStore";
|
|
7662
|
+
const toolId = id || `GraphRAG ${storeName} ${options.indexName} Tool`;
|
|
7098
7663
|
const toolDescription = description || defaultGraphRagDescription();
|
|
7099
7664
|
const graphOptions = {
|
|
7100
7665
|
...defaultGraphOptions,
|
|
@@ -7108,50 +7673,34 @@ var createGraphRAGTool = (options) => {
|
|
|
7108
7673
|
inputSchema,
|
|
7109
7674
|
outputSchema,
|
|
7110
7675
|
description: toolDescription,
|
|
7111
|
-
execute: async (
|
|
7112
|
-
const
|
|
7113
|
-
const
|
|
7676
|
+
execute: async (inputData, context) => {
|
|
7677
|
+
const { requestContext, mastra } = context || {};
|
|
7678
|
+
const indexName = requestContext?.get("indexName") ?? options.indexName;
|
|
7679
|
+
const vectorStoreName = "vectorStore" in options ? storeName : requestContext?.get("vectorStoreName") ?? storeName;
|
|
7114
7680
|
if (!indexName) throw new Error(`indexName is required, got: ${indexName}`);
|
|
7115
7681
|
if (!vectorStoreName) throw new Error(`vectorStoreName is required, got: ${vectorStoreName}`);
|
|
7116
|
-
const includeSources =
|
|
7117
|
-
const randomWalkSteps =
|
|
7118
|
-
const restartProb =
|
|
7119
|
-
const topK =
|
|
7120
|
-
const filter =
|
|
7121
|
-
const queryText =
|
|
7122
|
-
const providerOptions =
|
|
7123
|
-
const enableFilter = !!
|
|
7682
|
+
const includeSources = requestContext?.get("includeSources") ?? options.includeSources ?? true;
|
|
7683
|
+
const randomWalkSteps = requestContext?.get("randomWalkSteps") ?? graphOptions.randomWalkSteps;
|
|
7684
|
+
const restartProb = requestContext?.get("restartProb") ?? graphOptions.restartProb;
|
|
7685
|
+
const topK = requestContext?.get("topK") ?? inputData.topK ?? 10;
|
|
7686
|
+
const filter = requestContext?.get("filter") ?? inputData.filter;
|
|
7687
|
+
const queryText = inputData.queryText;
|
|
7688
|
+
const providerOptions = requestContext?.get("providerOptions") ?? options.providerOptions;
|
|
7689
|
+
const enableFilter = !!requestContext?.get("filter") || (options.enableFilter ?? false);
|
|
7124
7690
|
const logger = mastra?.getLogger();
|
|
7125
|
-
if (!logger) {
|
|
7126
|
-
console.warn(
|
|
7127
|
-
"[GraphRAGTool] Logger not initialized: no debug or error logs will be recorded for this tool execution."
|
|
7128
|
-
);
|
|
7129
|
-
}
|
|
7130
7691
|
if (logger) {
|
|
7131
7692
|
logger.debug("[GraphRAGTool] execute called with:", { queryText, topK, filter });
|
|
7132
7693
|
}
|
|
7133
7694
|
try {
|
|
7134
|
-
const topKValue =
|
|
7135
|
-
const vectorStore = mastra
|
|
7695
|
+
const topKValue = coerceTopK(topK);
|
|
7696
|
+
const vectorStore = await resolveVectorStore(options, { requestContext, mastra, vectorStoreName });
|
|
7136
7697
|
if (!vectorStore) {
|
|
7137
7698
|
if (logger) {
|
|
7138
|
-
logger.error(
|
|
7699
|
+
logger.error(`Vector store '${vectorStoreName}' not found`);
|
|
7139
7700
|
}
|
|
7140
7701
|
return { relevantContext: [], sources: [] };
|
|
7141
7702
|
}
|
|
7142
|
-
|
|
7143
|
-
if (enableFilter) {
|
|
7144
|
-
queryFilter = (() => {
|
|
7145
|
-
try {
|
|
7146
|
-
return typeof filter === "string" ? JSON.parse(filter) : filter;
|
|
7147
|
-
} catch (error) {
|
|
7148
|
-
if (logger) {
|
|
7149
|
-
logger.warn("Failed to parse filter as JSON, using empty filter", { filter, error });
|
|
7150
|
-
}
|
|
7151
|
-
return {};
|
|
7152
|
-
}
|
|
7153
|
-
})();
|
|
7154
|
-
}
|
|
7703
|
+
const queryFilter = enableFilter && filter ? parseFilterValue(filter, logger) : {};
|
|
7155
7704
|
if (logger) {
|
|
7156
7705
|
logger.debug("Prepared vector query parameters:", { queryFilter, topK: topKValue });
|
|
7157
7706
|
}
|
|
@@ -7204,7 +7753,7 @@ var createGraphRAGTool = (options) => {
|
|
|
7204
7753
|
};
|
|
7205
7754
|
} catch (err) {
|
|
7206
7755
|
if (logger) {
|
|
7207
|
-
logger.error("Unexpected error in
|
|
7756
|
+
logger.error("Unexpected error in GraphRAGTool execute", {
|
|
7208
7757
|
error: err,
|
|
7209
7758
|
errorMessage: err instanceof Error ? err.message : String(err),
|
|
7210
7759
|
errorStack: err instanceof Error ? err.stack : void 0
|
|
@@ -7227,57 +7776,36 @@ var createVectorQueryTool = (options) => {
|
|
|
7227
7776
|
description: toolDescription,
|
|
7228
7777
|
inputSchema,
|
|
7229
7778
|
outputSchema,
|
|
7230
|
-
execute: async (
|
|
7231
|
-
const
|
|
7232
|
-
const
|
|
7233
|
-
const
|
|
7234
|
-
const
|
|
7235
|
-
const
|
|
7236
|
-
const
|
|
7237
|
-
const
|
|
7238
|
-
const
|
|
7779
|
+
execute: async (inputData, context) => {
|
|
7780
|
+
const { requestContext, mastra } = context || {};
|
|
7781
|
+
const indexName = requestContext?.get("indexName") ?? options.indexName;
|
|
7782
|
+
const vectorStoreName = "vectorStore" in options ? storeName : requestContext?.get("vectorStoreName") ?? storeName;
|
|
7783
|
+
const includeVectors = requestContext?.get("includeVectors") ?? options.includeVectors ?? false;
|
|
7784
|
+
const includeSources = requestContext?.get("includeSources") ?? options.includeSources ?? true;
|
|
7785
|
+
const reranker = requestContext?.get("reranker") ?? options.reranker;
|
|
7786
|
+
const databaseConfig = requestContext?.get("databaseConfig") ?? options.databaseConfig;
|
|
7787
|
+
const model = requestContext?.get("model") ?? options.model;
|
|
7788
|
+
const providerOptions = requestContext?.get("providerOptions") ?? options.providerOptions;
|
|
7239
7789
|
if (!indexName) throw new Error(`indexName is required, got: ${indexName}`);
|
|
7240
7790
|
if (!vectorStoreName) throw new Error(`vectorStoreName is required, got: ${vectorStoreName}`);
|
|
7241
|
-
const topK =
|
|
7242
|
-
const filter =
|
|
7243
|
-
const queryText =
|
|
7244
|
-
const enableFilter = !!
|
|
7791
|
+
const topK = requestContext?.get("topK") ?? inputData.topK ?? 10;
|
|
7792
|
+
const filter = requestContext?.get("filter") ?? inputData.filter;
|
|
7793
|
+
const queryText = inputData.queryText;
|
|
7794
|
+
const enableFilter = !!requestContext?.get("filter") || (options.enableFilter ?? false);
|
|
7245
7795
|
const logger = mastra?.getLogger();
|
|
7246
|
-
if (!logger) {
|
|
7247
|
-
console.warn(
|
|
7248
|
-
"[VectorQueryTool] Logger not initialized: no debug or error logs will be recorded for this tool execution."
|
|
7249
|
-
);
|
|
7250
|
-
}
|
|
7251
7796
|
if (logger) {
|
|
7252
7797
|
logger.debug("[VectorQueryTool] execute called with:", { queryText, topK, filter, databaseConfig });
|
|
7253
7798
|
}
|
|
7254
7799
|
try {
|
|
7255
|
-
const topKValue =
|
|
7256
|
-
|
|
7257
|
-
if ("vectorStore" in options) {
|
|
7258
|
-
vectorStore = options.vectorStore;
|
|
7259
|
-
} else if (mastra) {
|
|
7260
|
-
vectorStore = mastra.getVector(vectorStoreName);
|
|
7261
|
-
}
|
|
7800
|
+
const topKValue = coerceTopK(topK);
|
|
7801
|
+
const vectorStore = await resolveVectorStore(options, { requestContext, mastra, vectorStoreName });
|
|
7262
7802
|
if (!vectorStore) {
|
|
7263
7803
|
if (logger) {
|
|
7264
|
-
logger.error(
|
|
7804
|
+
logger.error(`Vector store '${vectorStoreName}' not found`);
|
|
7265
7805
|
}
|
|
7266
7806
|
return { relevantContext: [], sources: [] };
|
|
7267
7807
|
}
|
|
7268
|
-
|
|
7269
|
-
if (enableFilter && filter) {
|
|
7270
|
-
queryFilter = (() => {
|
|
7271
|
-
try {
|
|
7272
|
-
return typeof filter === "string" ? JSON.parse(filter) : filter;
|
|
7273
|
-
} catch (error) {
|
|
7274
|
-
if (logger) {
|
|
7275
|
-
logger.warn("Failed to parse filter as JSON, using empty filter", { filter, error });
|
|
7276
|
-
}
|
|
7277
|
-
return {};
|
|
7278
|
-
}
|
|
7279
|
-
})();
|
|
7280
|
-
}
|
|
7808
|
+
const queryFilter = enableFilter && filter ? parseFilterValue(filter, logger) : {};
|
|
7281
7809
|
if (logger) {
|
|
7282
7810
|
logger.debug("Prepared vector query parameters", { queryText, topK: topKValue, queryFilter, databaseConfig });
|
|
7283
7811
|
}
|
|
@@ -7350,823 +7878,11 @@ var createVectorQueryTool = (options) => {
|
|
|
7350
7878
|
});
|
|
7351
7879
|
};
|
|
7352
7880
|
|
|
7353
|
-
// src/utils/vector-prompts.ts
|
|
7354
|
-
var ASTRA_PROMPT = `When querying Astra, you can ONLY use the operators listed below. Any other operators will be rejected.
|
|
7355
|
-
Important: Don't explain how to construct the filter - use the specified operators and fields to search the content and return relevant results.
|
|
7356
|
-
If a user tries to give an explicit operator that is not supported, reject the filter entirely and let them know that the operator is not supported.
|
|
7357
|
-
|
|
7358
|
-
Basic Comparison Operators:
|
|
7359
|
-
- $eq: Exact match (default when using field: value)
|
|
7360
|
-
Example: { "category": "electronics" }
|
|
7361
|
-
- $ne: Not equal
|
|
7362
|
-
Example: { "category": { "$ne": "electronics" } }
|
|
7363
|
-
- $gt: Greater than
|
|
7364
|
-
Example: { "price": { "$gt": 100 } }
|
|
7365
|
-
- $gte: Greater than or equal
|
|
7366
|
-
Example: { "price": { "$gte": 100 } }
|
|
7367
|
-
- $lt: Less than
|
|
7368
|
-
Example: { "price": { "$lt": 100 } }
|
|
7369
|
-
- $lte: Less than or equal
|
|
7370
|
-
Example: { "price": { "$lte": 100 } }
|
|
7371
|
-
|
|
7372
|
-
Array Operators:
|
|
7373
|
-
- $in: Match any value in array
|
|
7374
|
-
Example: { "category": { "$in": ["electronics", "books"] } }
|
|
7375
|
-
- $nin: Does not match any value in array
|
|
7376
|
-
Example: { "category": { "$nin": ["electronics", "books"] } }
|
|
7377
|
-
- $all: Match all values in array
|
|
7378
|
-
Example: { "tags": { "$all": ["premium", "sale"] } }
|
|
7379
|
-
|
|
7380
|
-
Logical Operators:
|
|
7381
|
-
- $and: Logical AND (can be implicit or explicit)
|
|
7382
|
-
Implicit Example: { "price": { "$gt": 100 }, "category": "electronics" }
|
|
7383
|
-
Explicit Example: { "$and": [{ "price": { "$gt": 100 } }, { "category": "electronics" }] }
|
|
7384
|
-
- $or: Logical OR
|
|
7385
|
-
Example: { "$or": [{ "price": { "$lt": 50 } }, { "category": "books" }] }
|
|
7386
|
-
- $not: Logical NOT
|
|
7387
|
-
Example: { "$not": { "category": "electronics" } }
|
|
7388
|
-
|
|
7389
|
-
Element Operators:
|
|
7390
|
-
- $exists: Check if field exists
|
|
7391
|
-
Example: { "rating": { "$exists": true } }
|
|
7392
|
-
|
|
7393
|
-
Special Operators:
|
|
7394
|
-
- $size: Array length check
|
|
7395
|
-
Example: { "tags": { "$size": 2 } }
|
|
7396
|
-
|
|
7397
|
-
Restrictions:
|
|
7398
|
-
- Regex patterns are not supported
|
|
7399
|
-
- Only $and, $or, and $not logical operators are supported
|
|
7400
|
-
- Nested fields are supported using dot notation
|
|
7401
|
-
- Multiple conditions on the same field are supported with both implicit and explicit $and
|
|
7402
|
-
- Empty arrays in $in/$nin will return no results
|
|
7403
|
-
- A non-empty array is required for $all operator
|
|
7404
|
-
- Only logical operators ($and, $or, $not) can be used at the top level
|
|
7405
|
-
- All other operators must be used within a field condition
|
|
7406
|
-
Valid: { "field": { "$gt": 100 } }
|
|
7407
|
-
Valid: { "$and": [...] }
|
|
7408
|
-
Invalid: { "$gt": 100 }
|
|
7409
|
-
- Logical operators must contain field conditions, not direct operators
|
|
7410
|
-
Valid: { "$and": [{ "field": { "$gt": 100 } }] }
|
|
7411
|
-
Invalid: { "$and": [{ "$gt": 100 }] }
|
|
7412
|
-
- $not operator:
|
|
7413
|
-
- Must be an object
|
|
7414
|
-
- Cannot be empty
|
|
7415
|
-
- Can be used at field level or top level
|
|
7416
|
-
- Valid: { "$not": { "field": "value" } }
|
|
7417
|
-
- Valid: { "field": { "$not": { "$eq": "value" } } }
|
|
7418
|
-
- Other logical operators ($and, $or):
|
|
7419
|
-
- Can only be used at top level or nested within other logical operators
|
|
7420
|
-
- Can not be used on a field level, or be nested inside a field
|
|
7421
|
-
- Can not be used inside an operator
|
|
7422
|
-
- Valid: { "$and": [{ "field": { "$gt": 100 } }] }
|
|
7423
|
-
- Valid: { "$or": [{ "$and": [{ "field": { "$gt": 100 } }] }] }
|
|
7424
|
-
- Invalid: { "field": { "$and": [{ "$gt": 100 }] } }
|
|
7425
|
-
- Invalid: { "field": { "$or": [{ "$gt": 100 }] } }
|
|
7426
|
-
- Invalid: { "field": { "$gt": { "$and": [{...}] } } }
|
|
7427
|
-
|
|
7428
|
-
Example Complex Query:
|
|
7429
|
-
{
|
|
7430
|
-
"$and": [
|
|
7431
|
-
{ "category": { "$in": ["electronics", "computers"] } },
|
|
7432
|
-
{ "price": { "$gte": 100, "$lte": 1000 } },
|
|
7433
|
-
{ "tags": { "$all": ["premium"] } },
|
|
7434
|
-
{ "rating": { "$exists": true, "$gt": 4 } },
|
|
7435
|
-
{ "$or": [
|
|
7436
|
-
{ "stock": { "$gt": 0 } },
|
|
7437
|
-
{ "preorder": true }
|
|
7438
|
-
]}
|
|
7439
|
-
]
|
|
7440
|
-
}`;
|
|
7441
|
-
var CHROMA_PROMPT = `When querying Chroma, you can ONLY use the operators listed below. Any other operators will be rejected.
|
|
7442
|
-
Important: Don't explain how to construct the filter - use the specified operators and fields to search the content and return relevant results.
|
|
7443
|
-
If a user tries to give an explicit operator that is not supported, reject the filter entirely and let them know that the operator is not supported.
|
|
7444
|
-
|
|
7445
|
-
Basic Comparison Operators:
|
|
7446
|
-
- $eq: Exact match (default when using field: value)
|
|
7447
|
-
Example: { "category": "electronics" }
|
|
7448
|
-
- $ne: Not equal
|
|
7449
|
-
Example: { "category": { "$ne": "electronics" } }
|
|
7450
|
-
- $gt: Greater than
|
|
7451
|
-
Example: { "price": { "$gt": 100 } }
|
|
7452
|
-
- $gte: Greater than or equal
|
|
7453
|
-
Example: { "price": { "$gte": 100 } }
|
|
7454
|
-
- $lt: Less than
|
|
7455
|
-
Example: { "price": { "$lt": 100 } }
|
|
7456
|
-
- $lte: Less than or equal
|
|
7457
|
-
Example: { "price": { "$lte": 100 } }
|
|
7458
|
-
|
|
7459
|
-
Array Operators:
|
|
7460
|
-
- $in: Match any value in array
|
|
7461
|
-
Example: { "category": { "$in": ["electronics", "books"] } }
|
|
7462
|
-
- $nin: Does not match any value in array
|
|
7463
|
-
Example: { "category": { "$nin": ["electronics", "books"] } }
|
|
7464
|
-
|
|
7465
|
-
Logical Operators:
|
|
7466
|
-
- $and: Logical AND
|
|
7467
|
-
Example: { "$and": [{ "price": { "$gt": 100 } }, { "category": "electronics" }] }
|
|
7468
|
-
- $or: Logical OR
|
|
7469
|
-
Example: { "$or": [{ "price": { "$lt": 50 } }, { "category": "books" }] }
|
|
7470
|
-
|
|
7471
|
-
Restrictions:
|
|
7472
|
-
- Regex patterns are not supported
|
|
7473
|
-
- Element operators are not supported
|
|
7474
|
-
- Only $and and $or logical operators are supported
|
|
7475
|
-
- Nested fields are supported using dot notation
|
|
7476
|
-
- Multiple conditions on the same field are supported with both implicit and explicit $and
|
|
7477
|
-
- Empty arrays in $in/$nin will return no results
|
|
7478
|
-
- If multiple top-level fields exist, they're wrapped in $and
|
|
7479
|
-
- Only logical operators ($and, $or) can be used at the top level
|
|
7480
|
-
- All other operators must be used within a field condition
|
|
7481
|
-
Valid: { "field": { "$gt": 100 } }
|
|
7482
|
-
Valid: { "$and": [...] }
|
|
7483
|
-
Invalid: { "$gt": 100 }
|
|
7484
|
-
Invalid: { "$in": [...] }
|
|
7485
|
-
- Logical operators must contain field conditions, not direct operators
|
|
7486
|
-
Valid: { "$and": [{ "field": { "$gt": 100 } }] }
|
|
7487
|
-
Invalid: { "$and": [{ "$gt": 100 }] }
|
|
7488
|
-
- Logical operators ($and, $or):
|
|
7489
|
-
- Can only be used at top level or nested within other logical operators
|
|
7490
|
-
- Can not be used on a field level, or be nested inside a field
|
|
7491
|
-
- Can not be used inside an operator
|
|
7492
|
-
- Valid: { "$and": [{ "field": { "$gt": 100 } }] }
|
|
7493
|
-
- Valid: { "$or": [{ "$and": [{ "field": { "$gt": 100 } }] }] }
|
|
7494
|
-
- Invalid: { "field": { "$and": [{ "$gt": 100 }] } }
|
|
7495
|
-
- Invalid: { "field": { "$or": [{ "$gt": 100 }] } }
|
|
7496
|
-
- Invalid: { "field": { "$gt": { "$and": [{...}] } } }
|
|
7497
|
-
Example Complex Query:
|
|
7498
|
-
{
|
|
7499
|
-
"$and": [
|
|
7500
|
-
{ "category": { "$in": ["electronics", "computers"] } },
|
|
7501
|
-
{ "price": { "$gte": 100, "$lte": 1000 } },
|
|
7502
|
-
{ "$or": [
|
|
7503
|
-
{ "inStock": true },
|
|
7504
|
-
{ "preorder": true }
|
|
7505
|
-
]}
|
|
7506
|
-
]
|
|
7507
|
-
}`;
|
|
7508
|
-
var LIBSQL_PROMPT = `When querying LibSQL Vector, you can ONLY use the operators listed below. Any other operators will be rejected.
|
|
7509
|
-
Important: Don't explain how to construct the filter - use the specified operators and fields to search the content and return relevant results.
|
|
7510
|
-
If a user tries to give an explicit operator that is not supported, reject the filter entirely and let them know that the operator is not supported.
|
|
7511
|
-
|
|
7512
|
-
Basic Comparison Operators:
|
|
7513
|
-
- $eq: Exact match (default when using field: value)
|
|
7514
|
-
Example: { "category": "electronics" }
|
|
7515
|
-
- $ne: Not equal
|
|
7516
|
-
Example: { "category": { "$ne": "electronics" } }
|
|
7517
|
-
- $gt: Greater than
|
|
7518
|
-
Example: { "price": { "$gt": 100 } }
|
|
7519
|
-
- $gte: Greater than or equal
|
|
7520
|
-
Example: { "price": { "$gte": 100 } }
|
|
7521
|
-
- $lt: Less than
|
|
7522
|
-
Example: { "price": { "$lt": 100 } }
|
|
7523
|
-
- $lte: Less than or equal
|
|
7524
|
-
Example: { "price": { "$lte": 100 } }
|
|
7525
|
-
|
|
7526
|
-
Array Operators:
|
|
7527
|
-
- $in: Match any value in array
|
|
7528
|
-
Example: { "category": { "$in": ["electronics", "books"] } }
|
|
7529
|
-
- $nin: Does not match any value in array
|
|
7530
|
-
Example: { "category": { "$nin": ["electronics", "books"] } }
|
|
7531
|
-
- $all: Match all values in array
|
|
7532
|
-
Example: { "tags": { "$all": ["premium", "sale"] } }
|
|
7533
|
-
- $elemMatch: Match array elements that meet all specified conditions
|
|
7534
|
-
Example: { "items": { "$elemMatch": { "price": { "$gt": 100 } } } }
|
|
7535
|
-
- $contains: Check if array contains value
|
|
7536
|
-
Example: { "tags": { "$contains": "premium" } }
|
|
7537
|
-
|
|
7538
|
-
Logical Operators:
|
|
7539
|
-
- $and: Logical AND (implicit when using multiple conditions)
|
|
7540
|
-
Example: { "$and": [{ "price": { "$gt": 100 } }, { "category": "electronics" }] }
|
|
7541
|
-
- $or: Logical OR
|
|
7542
|
-
Example: { "$or": [{ "price": { "$lt": 50 } }, { "category": "books" }] }
|
|
7543
|
-
- $not: Logical NOT
|
|
7544
|
-
Example: { "$not": { "category": "electronics" } }
|
|
7545
|
-
- $nor: Logical NOR
|
|
7546
|
-
Example: { "$nor": [{ "price": { "$lt": 50 } }, { "category": "books" }] }
|
|
7547
|
-
|
|
7548
|
-
Element Operators:
|
|
7549
|
-
- $exists: Check if field exists
|
|
7550
|
-
Example: { "rating": { "$exists": true } }
|
|
7551
|
-
|
|
7552
|
-
Special Operators:
|
|
7553
|
-
- $size: Array length check
|
|
7554
|
-
Example: { "tags": { "$size": 2 } }
|
|
7555
|
-
|
|
7556
|
-
Restrictions:
|
|
7557
|
-
- Regex patterns are not supported
|
|
7558
|
-
- Direct RegExp patterns will throw an error
|
|
7559
|
-
- Nested fields are supported using dot notation
|
|
7560
|
-
- Multiple conditions on the same field are supported with both implicit and explicit $and
|
|
7561
|
-
- Array operations work on array fields only
|
|
7562
|
-
- Basic operators handle array values as JSON strings
|
|
7563
|
-
- Empty arrays in conditions are handled gracefully
|
|
7564
|
-
- Only logical operators ($and, $or, $not, $nor) can be used at the top level
|
|
7565
|
-
- All other operators must be used within a field condition
|
|
7566
|
-
Valid: { "field": { "$gt": 100 } }
|
|
7567
|
-
Valid: { "$and": [...] }
|
|
7568
|
-
Invalid: { "$gt": 100 }
|
|
7569
|
-
Invalid: { "$contains": "value" }
|
|
7570
|
-
- Logical operators must contain field conditions, not direct operators
|
|
7571
|
-
Valid: { "$and": [{ "field": { "$gt": 100 } }] }
|
|
7572
|
-
Invalid: { "$and": [{ "$gt": 100 }] }
|
|
7573
|
-
- $not operator:
|
|
7574
|
-
- Must be an object
|
|
7575
|
-
- Cannot be empty
|
|
7576
|
-
- Can be used at field level or top level
|
|
7577
|
-
- Valid: { "$not": { "field": "value" } }
|
|
7578
|
-
- Valid: { "field": { "$not": { "$eq": "value" } } }
|
|
7579
|
-
- Other logical operators ($and, $or, $nor):
|
|
7580
|
-
- Can only be used at top level or nested within other logical operators
|
|
7581
|
-
- Can not be used on a field level, or be nested inside a field
|
|
7582
|
-
- Can not be used inside an operator
|
|
7583
|
-
- Valid: { "$and": [{ "field": { "$gt": 100 } }] }
|
|
7584
|
-
- Valid: { "$or": [{ "$and": [{ "field": { "$gt": 100 } }] }] }
|
|
7585
|
-
- Invalid: { "field": { "$and": [{ "$gt": 100 }] } }
|
|
7586
|
-
- Invalid: { "field": { "$or": [{ "$gt": 100 }] } }
|
|
7587
|
-
- Invalid: { "field": { "$gt": { "$and": [{...}] } } }
|
|
7588
|
-
- $elemMatch requires an object with conditions
|
|
7589
|
-
Valid: { "array": { "$elemMatch": { "field": "value" } } }
|
|
7590
|
-
Invalid: { "array": { "$elemMatch": "value" } }
|
|
7591
|
-
|
|
7592
|
-
Example Complex Query:
|
|
7593
|
-
{
|
|
7594
|
-
"$and": [
|
|
7595
|
-
{ "category": { "$in": ["electronics", "computers"] } },
|
|
7596
|
-
{ "price": { "$gte": 100, "$lte": 1000 } },
|
|
7597
|
-
{ "tags": { "$all": ["premium", "sale"] } },
|
|
7598
|
-
{ "items": { "$elemMatch": { "price": { "$gt": 50 }, "inStock": true } } },
|
|
7599
|
-
{ "$or": [
|
|
7600
|
-
{ "stock": { "$gt": 0 } },
|
|
7601
|
-
{ "preorder": true }
|
|
7602
|
-
]}
|
|
7603
|
-
]
|
|
7604
|
-
}`;
|
|
7605
|
-
var PGVECTOR_PROMPT = `When querying PG Vector, you can ONLY use the operators listed below. Any other operators will be rejected.
|
|
7606
|
-
Important: Don't explain how to construct the filter - use the specified operators and fields to search the content and return relevant results.
|
|
7607
|
-
If a user tries to give an explicit operator that is not supported, reject the filter entirely and let them know that the operator is not supported.
|
|
7608
|
-
|
|
7609
|
-
Basic Comparison Operators:
|
|
7610
|
-
- $eq: Exact match (default when using field: value)
|
|
7611
|
-
Example: { "category": "electronics" }
|
|
7612
|
-
- $ne: Not equal
|
|
7613
|
-
Example: { "category": { "$ne": "electronics" } }
|
|
7614
|
-
- $gt: Greater than
|
|
7615
|
-
Example: { "price": { "$gt": 100 } }
|
|
7616
|
-
- $gte: Greater than or equal
|
|
7617
|
-
Example: { "price": { "$gte": 100 } }
|
|
7618
|
-
- $lt: Less than
|
|
7619
|
-
Example: { "price": { "$lt": 100 } }
|
|
7620
|
-
- $lte: Less than or equal
|
|
7621
|
-
Example: { "price": { "$lte": 100 } }
|
|
7622
|
-
|
|
7623
|
-
Array Operators:
|
|
7624
|
-
- $in: Match any value in array
|
|
7625
|
-
Example: { "category": { "$in": ["electronics", "books"] } }
|
|
7626
|
-
- $nin: Does not match any value in array
|
|
7627
|
-
Example: { "category": { "$nin": ["electronics", "books"] } }
|
|
7628
|
-
- $all: Match all values in array
|
|
7629
|
-
Example: { "tags": { "$all": ["premium", "sale"] } }
|
|
7630
|
-
- $elemMatch: Match array elements that meet all specified conditions
|
|
7631
|
-
Example: { "items": { "$elemMatch": { "price": { "$gt": 100 } } } }
|
|
7632
|
-
- $contains: Check if array contains value
|
|
7633
|
-
Example: { "tags": { "$contains": "premium" } }
|
|
7634
|
-
|
|
7635
|
-
Logical Operators:
|
|
7636
|
-
- $and: Logical AND
|
|
7637
|
-
Example: { "$and": [{ "price": { "$gt": 100 } }, { "category": "electronics" }] }
|
|
7638
|
-
- $or: Logical OR
|
|
7639
|
-
Example: { "$or": [{ "price": { "$lt": 50 } }, { "category": "books" }] }
|
|
7640
|
-
- $not: Logical NOT
|
|
7641
|
-
Example: { "$not": { "category": "electronics" } }
|
|
7642
|
-
- $nor: Logical NOR
|
|
7643
|
-
Example: { "$nor": [{ "price": { "$lt": 50 } }, { "category": "books" }] }
|
|
7644
|
-
|
|
7645
|
-
Element Operators:
|
|
7646
|
-
- $exists: Check if field exists
|
|
7647
|
-
Example: { "rating": { "$exists": true } }
|
|
7648
|
-
|
|
7649
|
-
Special Operators:
|
|
7650
|
-
- $size: Array length check
|
|
7651
|
-
Example: { "tags": { "$size": 2 } }
|
|
7652
|
-
- $regex: Pattern matching (PostgreSQL regex syntax)
|
|
7653
|
-
Example: { "name": { "$regex": "^iphone" } }
|
|
7654
|
-
- $options: Regex options (used with $regex)
|
|
7655
|
-
Example: { "name": { "$regex": "iphone", "$options": "i" } }
|
|
7656
|
-
|
|
7657
|
-
Restrictions:
|
|
7658
|
-
- Direct RegExp patterns are supported
|
|
7659
|
-
- Nested fields are supported using dot notation
|
|
7660
|
-
- Multiple conditions on the same field are supported with both implicit and explicit $and
|
|
7661
|
-
- Array operations work on array fields only
|
|
7662
|
-
- Regex patterns must follow PostgreSQL syntax
|
|
7663
|
-
- Empty arrays in conditions are handled gracefully
|
|
7664
|
-
- Only logical operators ($and, $or, $not, $nor) can be used at the top level
|
|
7665
|
-
- All other operators must be used within a field condition
|
|
7666
|
-
Valid: { "field": { "$gt": 100 } }
|
|
7667
|
-
Valid: { "$and": [...] }
|
|
7668
|
-
Invalid: { "$gt": 100 }
|
|
7669
|
-
Invalid: { "$regex": "pattern" }
|
|
7670
|
-
- Logical operators must contain field conditions, not direct operators
|
|
7671
|
-
Valid: { "$and": [{ "field": { "$gt": 100 } }] }
|
|
7672
|
-
Invalid: { "$and": [{ "$gt": 100 }] }
|
|
7673
|
-
- $not operator:
|
|
7674
|
-
- Must be an object
|
|
7675
|
-
- Cannot be empty
|
|
7676
|
-
- Can be used at field level or top level
|
|
7677
|
-
- Valid: { "$not": { "field": "value" } }
|
|
7678
|
-
- Valid: { "field": { "$not": { "$eq": "value" } } }
|
|
7679
|
-
- Other logical operators ($and, $or, $nor):
|
|
7680
|
-
- Can only be used at top level or nested within other logical operators
|
|
7681
|
-
- Can not be used on a field level, or be nested inside a field
|
|
7682
|
-
- Can not be used inside an operator
|
|
7683
|
-
- Valid: { "$and": [{ "field": { "$gt": 100 } }] }
|
|
7684
|
-
- Valid: { "$or": [{ "$and": [{ "field": { "$gt": 100 } }] }] }
|
|
7685
|
-
- Invalid: { "field": { "$and": [{ "$gt": 100 }] } }
|
|
7686
|
-
- Invalid: { "field": { "$or": [{ "$gt": 100 }] } }
|
|
7687
|
-
- Invalid: { "field": { "$gt": { "$and": [{...}] } } }
|
|
7688
|
-
- $elemMatch requires an object with conditions
|
|
7689
|
-
Valid: { "array": { "$elemMatch": { "field": "value" } } }
|
|
7690
|
-
Invalid: { "array": { "$elemMatch": "value" } }
|
|
7691
|
-
|
|
7692
|
-
Example Complex Query:
|
|
7693
|
-
{
|
|
7694
|
-
"$and": [
|
|
7695
|
-
{ "category": { "$in": ["electronics", "computers"] } },
|
|
7696
|
-
{ "price": { "$gte": 100, "$lte": 1000 } },
|
|
7697
|
-
{ "tags": { "$all": ["premium", "sale"] } },
|
|
7698
|
-
{ "items": { "$elemMatch": { "price": { "$gt": 50 }, "inStock": true } } },
|
|
7699
|
-
{ "$or": [
|
|
7700
|
-
{ "name": { "$regex": "^iphone", "$options": "i" } },
|
|
7701
|
-
{ "description": { "$regex": ".*apple.*" } }
|
|
7702
|
-
]}
|
|
7703
|
-
]
|
|
7704
|
-
}`;
|
|
7705
|
-
var PINECONE_PROMPT = `When querying Pinecone, you can ONLY use the operators listed below. Any other operators will be rejected.
|
|
7706
|
-
Important: Don't explain how to construct the filter - use the specified operators and fields to search the content and return relevant results.
|
|
7707
|
-
If a user tries to give an explicit operator that is not supported, reject the filter entirely and let them know that the operator is not supported.
|
|
7708
|
-
|
|
7709
|
-
Basic Comparison Operators:
|
|
7710
|
-
- $eq: Exact match (default when using field: value)
|
|
7711
|
-
Example: { "category": "electronics" }
|
|
7712
|
-
- $ne: Not equal
|
|
7713
|
-
Example: { "category": { "$ne": "electronics" } }
|
|
7714
|
-
- $gt: Greater than
|
|
7715
|
-
Example: { "price": { "$gt": 100 } }
|
|
7716
|
-
- $gte: Greater than or equal
|
|
7717
|
-
Example: { "price": { "$gte": 100 } }
|
|
7718
|
-
- $lt: Less than
|
|
7719
|
-
Example: { "price": { "$lt": 100 } }
|
|
7720
|
-
- $lte: Less than or equal
|
|
7721
|
-
Example: { "price": { "$lte": 100 } }
|
|
7722
|
-
|
|
7723
|
-
Array Operators:
|
|
7724
|
-
- $in: Match any value in array
|
|
7725
|
-
Example: { "category": { "$in": ["electronics", "books"] } }
|
|
7726
|
-
- $nin: Does not match any value in array
|
|
7727
|
-
Example: { "category": { "$nin": ["electronics", "books"] } }
|
|
7728
|
-
- $all: Match all values in array
|
|
7729
|
-
Example: { "tags": { "$all": ["premium", "sale"] } }
|
|
7730
|
-
|
|
7731
|
-
Logical Operators:
|
|
7732
|
-
- $and: Logical AND (can be implicit or explicit)
|
|
7733
|
-
Implicit Example: { "price": { "$gt": 100 }, "category": "electronics" }
|
|
7734
|
-
Explicit Example: { "$and": [{ "price": { "$gt": 100 } }, { "category": "electronics" }] }
|
|
7735
|
-
- $or: Logical OR
|
|
7736
|
-
Example: { "$or": [{ "price": { "$lt": 50 } }, { "category": "books" }] }
|
|
7737
|
-
|
|
7738
|
-
Element Operators:
|
|
7739
|
-
- $exists: Check if field exists
|
|
7740
|
-
Example: { "rating": { "$exists": true } }
|
|
7741
|
-
|
|
7742
|
-
Restrictions:
|
|
7743
|
-
- Regex patterns are not supported
|
|
7744
|
-
- Only $and and $or logical operators are supported at the top level
|
|
7745
|
-
- Empty arrays in $in/$nin will return no results
|
|
7746
|
-
- A non-empty array is required for $all operator
|
|
7747
|
-
- Nested fields are supported using dot notation
|
|
7748
|
-
- Multiple conditions on the same field are supported with both implicit and explicit $and
|
|
7749
|
-
- At least one key-value pair is required in filter object
|
|
7750
|
-
- Empty objects and undefined values are treated as no filter
|
|
7751
|
-
- Invalid types in comparison operators will throw errors
|
|
7752
|
-
- All non-logical operators must be used within a field condition
|
|
7753
|
-
Valid: { "field": { "$gt": 100 } }
|
|
7754
|
-
Valid: { "$and": [...] }
|
|
7755
|
-
Invalid: { "$gt": 100 }
|
|
7756
|
-
- Logical operators must contain field conditions, not direct operators
|
|
7757
|
-
Valid: { "$and": [{ "field": { "$gt": 100 } }] }
|
|
7758
|
-
Invalid: { "$and": [{ "$gt": 100 }] }
|
|
7759
|
-
- Logical operators ($and, $or):
|
|
7760
|
-
- Can only be used at top level or nested within other logical operators
|
|
7761
|
-
- Can not be used on a field level, or be nested inside a field
|
|
7762
|
-
- Can not be used inside an operator
|
|
7763
|
-
- Valid: { "$and": [{ "field": { "$gt": 100 } }] }
|
|
7764
|
-
- Valid: { "$or": [{ "$and": [{ "field": { "$gt": 100 } }] }] }
|
|
7765
|
-
- Invalid: { "field": { "$and": [{ "$gt": 100 }] } }
|
|
7766
|
-
- Invalid: { "field": { "$or": [{ "$gt": 100 }] } }
|
|
7767
|
-
- Invalid: { "field": { "$gt": { "$and": [{...}] } } }
|
|
7768
|
-
Example Complex Query:
|
|
7769
|
-
{
|
|
7770
|
-
"$and": [
|
|
7771
|
-
{ "category": { "$in": ["electronics", "computers"] } },
|
|
7772
|
-
{ "price": { "$gte": 100, "$lte": 1000 } },
|
|
7773
|
-
{ "tags": { "$all": ["premium", "sale"] } },
|
|
7774
|
-
{ "rating": { "$exists": true, "$gt": 4 } },
|
|
7775
|
-
{ "$or": [
|
|
7776
|
-
{ "stock": { "$gt": 0 } },
|
|
7777
|
-
{ "preorder": true }
|
|
7778
|
-
]}
|
|
7779
|
-
]
|
|
7780
|
-
}`;
|
|
7781
|
-
var QDRANT_PROMPT = `When querying Qdrant, you can ONLY use the operators listed below. Any other operators will be rejected.
|
|
7782
|
-
Important: Don't explain how to construct the filter - use the specified operators and fields to search the content and return relevant results.
|
|
7783
|
-
If a user tries to give an explicit operator that is not supported, reject the filter entirely and let them know that the operator is not supported.
|
|
7784
|
-
|
|
7785
|
-
Basic Comparison Operators:
|
|
7786
|
-
- $eq: Exact match (default when using field: value)
|
|
7787
|
-
Example: { "category": "electronics" }
|
|
7788
|
-
- $ne: Not equal
|
|
7789
|
-
Example: { "category": { "$ne": "electronics" } }
|
|
7790
|
-
- $gt: Greater than
|
|
7791
|
-
Example: { "price": { "$gt": 100 } }
|
|
7792
|
-
- $gte: Greater than or equal
|
|
7793
|
-
Example: { "price": { "$gte": 100 } }
|
|
7794
|
-
- $lt: Less than
|
|
7795
|
-
Example: { "price": { "$lt": 100 } }
|
|
7796
|
-
- $lte: Less than or equal
|
|
7797
|
-
Example: { "price": { "$lte": 100 } }
|
|
7798
|
-
|
|
7799
|
-
Array Operators:
|
|
7800
|
-
- $in: Match any value in array
|
|
7801
|
-
Example: { "category": { "$in": ["electronics", "books"] } }
|
|
7802
|
-
- $nin: Does not match any value in array
|
|
7803
|
-
Example: { "category": { "$nin": ["electronics", "books"] } }
|
|
7804
|
-
|
|
7805
|
-
Logical Operators:
|
|
7806
|
-
- $and: Logical AND (implicit when using multiple conditions)
|
|
7807
|
-
Example: { "$and": [{ "price": { "$gt": 100 } }, { "category": "electronics" }] }
|
|
7808
|
-
- $or: Logical OR
|
|
7809
|
-
Example: { "$or": [{ "price": { "$lt": 50 } }, { "category": "books" }] }
|
|
7810
|
-
- $not: Logical NOT
|
|
7811
|
-
Example: { "$not": { "category": "electronics" } }
|
|
7812
|
-
|
|
7813
|
-
Element Operators:
|
|
7814
|
-
- $exists: Check if field exists
|
|
7815
|
-
Example: { "rating": { "$exists": true } }
|
|
7816
|
-
|
|
7817
|
-
Special Operators:
|
|
7818
|
-
- $regex: Pattern matching
|
|
7819
|
-
Example: { "name": { "$regex": "iphone.*" } }
|
|
7820
|
-
- $count: Array length/value count
|
|
7821
|
-
Example: { "tags": { "$count": { "$gt": 2 } } }
|
|
7822
|
-
- $geo: Geographical filters (supports radius, box, polygon)
|
|
7823
|
-
Example: {
|
|
7824
|
-
"location": {
|
|
7825
|
-
"$geo": {
|
|
7826
|
-
"type": "radius",
|
|
7827
|
-
"center": { "lat": 52.5, "lon": 13.4 },
|
|
7828
|
-
"radius": 10000
|
|
7829
|
-
}
|
|
7830
|
-
}
|
|
7831
|
-
}
|
|
7832
|
-
- $hasId: Match specific document IDs
|
|
7833
|
-
Example: { "$hasId": ["doc1", "doc2"] }
|
|
7834
|
-
- $hasVector: Check vector existence
|
|
7835
|
-
Example: { "$hasVector": "" }
|
|
7836
|
-
- $datetime: RFC 3339 datetime range
|
|
7837
|
-
Example: {
|
|
7838
|
-
"created_at": {
|
|
7839
|
-
"$datetime": {
|
|
7840
|
-
"range": {
|
|
7841
|
-
"gt": "2024-01-01T00:00:00Z",
|
|
7842
|
-
"lt": "2024-12-31T23:59:59Z"
|
|
7843
|
-
}
|
|
7844
|
-
}
|
|
7845
|
-
}
|
|
7846
|
-
}
|
|
7847
|
-
- $null: Check for null values
|
|
7848
|
-
Example: { "field": { "$null": true } }
|
|
7849
|
-
- $empty: Check for empty values
|
|
7850
|
-
Example: { "array": { "$empty": true } }
|
|
7851
|
-
- $nested: Nested object filters
|
|
7852
|
-
Example: {
|
|
7853
|
-
"items[]": {
|
|
7854
|
-
"$nested": {
|
|
7855
|
-
"price": { "$gt": 100 },
|
|
7856
|
-
"stock": { "$gt": 0 }
|
|
7857
|
-
}
|
|
7858
|
-
}
|
|
7859
|
-
}
|
|
7860
|
-
|
|
7861
|
-
Restrictions:
|
|
7862
|
-
- Only logical operators ($and, $or, $not) and collection operators ($hasId, $hasVector) can be used at the top level
|
|
7863
|
-
- All other operators must be used within a field condition
|
|
7864
|
-
Valid: { "field": { "$gt": 100 } }
|
|
7865
|
-
Valid: { "$and": [...] }
|
|
7866
|
-
Valid: { "$hasId": [...] }
|
|
7867
|
-
Invalid: { "$gt": 100 }
|
|
7868
|
-
- Nested fields are supported using dot notation
|
|
7869
|
-
- Array fields with nested objects use [] suffix: "items[]"
|
|
7870
|
-
- Geo filtering requires specific format for radius, box, or polygon
|
|
7871
|
-
- Datetime values must be in RFC 3339 format
|
|
7872
|
-
- Empty arrays in conditions are handled as empty values
|
|
7873
|
-
- Null values are handled with $null operator
|
|
7874
|
-
- Empty values are handled with $empty operator
|
|
7875
|
-
- $regex uses standard regex syntax
|
|
7876
|
-
- $count can only be used with numeric comparison operators
|
|
7877
|
-
- $nested requires an object with conditions
|
|
7878
|
-
- Logical operators must contain field conditions, not direct operators
|
|
7879
|
-
Valid: { "$and": [{ "field": { "$gt": 100 } }] }
|
|
7880
|
-
Invalid: { "$and": [{ "$gt": 100 }] }
|
|
7881
|
-
- $not operator:
|
|
7882
|
-
- Must be an object
|
|
7883
|
-
- Cannot be empty
|
|
7884
|
-
- Can be used at field level or top level
|
|
7885
|
-
- Valid: { "$not": { "field": "value" } }
|
|
7886
|
-
- Valid: { "field": { "$not": { "$eq": "value" } } }
|
|
7887
|
-
- Other logical operators ($and, $or):
|
|
7888
|
-
- Can only be used at top level or nested within other logical operators
|
|
7889
|
-
- Can not be used on a field level, or be nested inside a field
|
|
7890
|
-
- Can not be used inside an operator
|
|
7891
|
-
- Valid: { "$and": [{ "field": { "$gt": 100 } }] }
|
|
7892
|
-
- Valid: { "$or": [{ "$and": [{ "field": { "$gt": 100 } }] }] }
|
|
7893
|
-
- Invalid: { "field": { "$and": [{ "$gt": 100 }] } }
|
|
7894
|
-
- Invalid: { "field": { "$or": [{ "$gt": 100 }] } }
|
|
7895
|
-
- Invalid: { "field": { "$gt": { "$and": [{...}] } } }
|
|
7896
|
-
Example Complex Query:
|
|
7897
|
-
{
|
|
7898
|
-
"$and": [
|
|
7899
|
-
{ "category": { "$in": ["electronics"] } },
|
|
7900
|
-
{ "price": { "$gt": 100 } },
|
|
7901
|
-
{ "location": {
|
|
7902
|
-
"$geo": {
|
|
7903
|
-
"type": "radius",
|
|
7904
|
-
"center": { "lat": 52.5, "lon": 13.4 },
|
|
7905
|
-
"radius": 5000
|
|
7906
|
-
}
|
|
7907
|
-
}},
|
|
7908
|
-
{ "items[]": {
|
|
7909
|
-
"$nested": {
|
|
7910
|
-
"price": { "$gt": 50 },
|
|
7911
|
-
"stock": { "$gt": 0 }
|
|
7912
|
-
}
|
|
7913
|
-
}},
|
|
7914
|
-
{ "created_at": {
|
|
7915
|
-
"$datetime": {
|
|
7916
|
-
"range": {
|
|
7917
|
-
"gt": "2024-01-01T00:00:00Z"
|
|
7918
|
-
}
|
|
7919
|
-
}
|
|
7920
|
-
}},
|
|
7921
|
-
{ "$or": [
|
|
7922
|
-
{ "status": { "$ne": "discontinued" } },
|
|
7923
|
-
{ "clearance": true }
|
|
7924
|
-
]}
|
|
7925
|
-
]
|
|
7926
|
-
}`;
|
|
7927
|
-
var UPSTASH_PROMPT = `When querying Upstash Vector, you can ONLY use the operators listed below. Any other operators will be rejected.
|
|
7928
|
-
Important: Don't explain how to construct the filter - use the specified operators and fields to search the content and return relevant results.
|
|
7929
|
-
If a user tries to give an explicit operator that is not supported, reject the filter entirely and let them know that the operator is not supported.
|
|
7930
|
-
|
|
7931
|
-
Basic Comparison Operators:
|
|
7932
|
-
- $eq: Exact match (default when using field: value)
|
|
7933
|
-
Example: { "category": "electronics" } or { "category": { "$eq": "electronics" } }
|
|
7934
|
-
- $ne: Not equal
|
|
7935
|
-
Example: { "category": { "$ne": "electronics" } }
|
|
7936
|
-
- $gt: Greater than
|
|
7937
|
-
Example: { "price": { "$gt": 100 } }
|
|
7938
|
-
- $gte: Greater than or equal
|
|
7939
|
-
Example: { "price": { "$gte": 100 } }
|
|
7940
|
-
- $lt: Less than
|
|
7941
|
-
Example: { "price": { "$lt": 100 } }
|
|
7942
|
-
- $lte: Less than or equal
|
|
7943
|
-
Example: { "price": { "$lte": 100 } }
|
|
7944
|
-
|
|
7945
|
-
Array Operators:
|
|
7946
|
-
- $in: Match any value in array
|
|
7947
|
-
Example: { "category": { "$in": ["electronics", "books"] } }
|
|
7948
|
-
- $nin: Does not match any value in array
|
|
7949
|
-
Example: { "category": { "$nin": ["electronics", "books"] } }
|
|
7950
|
-
- $all: Matches all values in array
|
|
7951
|
-
Example: { "tags": { "$all": ["premium", "new"] } }
|
|
7952
|
-
|
|
7953
|
-
Logical Operators:
|
|
7954
|
-
- $and: Logical AND (implicit when using multiple conditions)
|
|
7955
|
-
Example: { "$and": [{ "price": { "$gt": 100 } }, { "category": "electronics" }] }
|
|
7956
|
-
- $or: Logical OR
|
|
7957
|
-
Example: { "$or": [{ "price": { "$lt": 50 } }, { "category": "books" }] }
|
|
7958
|
-
- $not: Logical NOT
|
|
7959
|
-
Example: { "$not": { "category": "electronics" } }
|
|
7960
|
-
- $nor: Logical NOR
|
|
7961
|
-
Example: { "$nor": [{ "price": { "$lt": 50 } }, { "category": "books" }] }
|
|
7962
|
-
|
|
7963
|
-
Element Operators:
|
|
7964
|
-
- $exists: Check if field exists
|
|
7965
|
-
Example: { "rating": { "$exists": true } }
|
|
7966
|
-
|
|
7967
|
-
Special Operators:
|
|
7968
|
-
- $regex: Pattern matching using glob syntax (only as operator, not direct RegExp)
|
|
7969
|
-
Example: { "name": { "$regex": "iphone*" } }
|
|
7970
|
-
- $contains: Check if array/string contains value
|
|
7971
|
-
Example: { "tags": { "$contains": "premium" } }
|
|
7972
|
-
|
|
7973
|
-
Restrictions:
|
|
7974
|
-
- Null/undefined values are not supported in any operator
|
|
7975
|
-
- Empty arrays are only supported in $in/$nin operators
|
|
7976
|
-
- Direct RegExp patterns are not supported, use $regex with glob syntax
|
|
7977
|
-
- Nested fields are supported using dot notation
|
|
7978
|
-
- Multiple conditions on same field are combined with AND
|
|
7979
|
-
- String values with quotes are automatically escaped
|
|
7980
|
-
- Only logical operators ($and, $or, $not, $nor) can be used at the top level
|
|
7981
|
-
- All other operators must be used within a field condition
|
|
7982
|
-
Valid: { "field": { "$gt": 100 } }
|
|
7983
|
-
Valid: { "$and": [...] }
|
|
7984
|
-
Invalid: { "$gt": 100 }
|
|
7985
|
-
- $regex uses glob syntax (*, ?) not standard regex patterns
|
|
7986
|
-
- $contains works on both arrays and string fields
|
|
7987
|
-
- Logical operators must contain field conditions, not direct operators
|
|
7988
|
-
Valid: { "$and": [{ "field": { "$gt": 100 } }] }
|
|
7989
|
-
Invalid: { "$and": [{ "$gt": 100 }] }
|
|
7990
|
-
- $not operator:
|
|
7991
|
-
- Must be an object
|
|
7992
|
-
- Cannot be empty
|
|
7993
|
-
- Can be used at field level or top level
|
|
7994
|
-
- Valid: { "$not": { "field": "value" } }
|
|
7995
|
-
- Valid: { "field": { "$not": { "$eq": "value" } } }
|
|
7996
|
-
- Other logical operators ($and, $or, $nor):
|
|
7997
|
-
- Can only be used at top level or nested within other logical operators
|
|
7998
|
-
- Can not be used on a field level, or be nested inside a field
|
|
7999
|
-
- Can not be used inside an operator
|
|
8000
|
-
- Valid: { "$and": [{ "field": { "$gt": 100 } }] }
|
|
8001
|
-
- Valid: { "$or": [{ "$and": [{ "field": { "$gt": 100 } }] }] }
|
|
8002
|
-
- Invalid: { "field": { "$and": [{ "$gt": 100 }] } }
|
|
8003
|
-
- Invalid: { "field": { "$or": [{ "$gt": 100 }] } }
|
|
8004
|
-
- Invalid: { "field": { "$gt": { "$and": [{...}] } } }
|
|
8005
|
-
Example Complex Query:
|
|
8006
|
-
{
|
|
8007
|
-
"$and": [
|
|
8008
|
-
{ "category": { "$in": ["electronics", "computers"] } },
|
|
8009
|
-
{ "price": { "$gt": 100, "$lt": 1000 } },
|
|
8010
|
-
{ "tags": { "$all": ["premium", "new"] } },
|
|
8011
|
-
{ "name": { "$regex": "iphone*" } },
|
|
8012
|
-
{ "description": { "$contains": "latest" } },
|
|
8013
|
-
{ "$or": [
|
|
8014
|
-
{ "brand": "Apple" },
|
|
8015
|
-
{ "rating": { "$gte": 4.5 } }
|
|
8016
|
-
]}
|
|
8017
|
-
]
|
|
8018
|
-
}`;
|
|
8019
|
-
var VECTORIZE_PROMPT = `When querying Vectorize, you can ONLY use the operators listed below. Any other operators will be rejected.
|
|
8020
|
-
Important: Don't explain how to construct the filter - use the specified operators and fields to search the content and return relevant results.
|
|
8021
|
-
If a user tries to give an explicit operator that is not supported, reject the filter entirely and let them know that the operator is not supported.
|
|
8022
|
-
|
|
8023
|
-
Basic Comparison Operators:
|
|
8024
|
-
- $eq: Exact match (default when using field: value)
|
|
8025
|
-
Example: { "category": "electronics" }
|
|
8026
|
-
- $ne: Not equal
|
|
8027
|
-
Example: { "category": { "$ne": "electronics" } }
|
|
8028
|
-
- $gt: Greater than
|
|
8029
|
-
Example: { "price": { "$gt": 100 } }
|
|
8030
|
-
- $gte: Greater than or equal
|
|
8031
|
-
Example: { "price": { "$gte": 100 } }
|
|
8032
|
-
- $lt: Less than
|
|
8033
|
-
Example: { "price": { "$lt": 100 } }
|
|
8034
|
-
- $lte: Less than or equal
|
|
8035
|
-
Example: { "price": { "$lte": 100 } }
|
|
8036
|
-
|
|
8037
|
-
Array Operators:
|
|
8038
|
-
- $in: Match any value in array
|
|
8039
|
-
Example: { "category": { "$in": ["electronics", "books"] } }
|
|
8040
|
-
- $nin: Does not match any value in array
|
|
8041
|
-
Example: { "category": { "$nin": ["electronics", "books"] } }
|
|
8042
|
-
|
|
8043
|
-
Restrictions:
|
|
8044
|
-
- Regex patterns are not supported
|
|
8045
|
-
- Logical operators are not supported
|
|
8046
|
-
- Element operators are not supported
|
|
8047
|
-
- Fields must have a flat structure, as nested fields are not supported
|
|
8048
|
-
- Multiple conditions on the same field are supported
|
|
8049
|
-
- Empty arrays in $in/$nin will return no results
|
|
8050
|
-
- Filter keys cannot be longer than 512 characters
|
|
8051
|
-
- Filter keys cannot contain invalid characters ($, ", empty)
|
|
8052
|
-
- Filter size is limited to prevent oversized queries
|
|
8053
|
-
- Invalid types in operators return no results instead of throwing errors
|
|
8054
|
-
- Empty objects are accepted in filters
|
|
8055
|
-
- Metadata must use flat structure with dot notation (no nested objects)
|
|
8056
|
-
- Must explicitly create metadata indexes for filterable fields (limit 10 per index)
|
|
8057
|
-
- Can only effectively filter on indexed metadata fields
|
|
8058
|
-
- Metadata values can be strings, numbers, booleans, or homogeneous arrays
|
|
8059
|
-
- No operators can be used at the top level (no logical operators supported)
|
|
8060
|
-
- All operators must be used within a field condition
|
|
8061
|
-
Valid: { "field": { "$gt": 100 } }
|
|
8062
|
-
Invalid: { "$gt": 100 }
|
|
8063
|
-
Invalid: { "$in": [...] }
|
|
8064
|
-
|
|
8065
|
-
Example Complex Query:
|
|
8066
|
-
{
|
|
8067
|
-
"category": { "$in": ["electronics", "computers"] },
|
|
8068
|
-
"price": { "$gte": 100, "$lte": 1000 },
|
|
8069
|
-
"inStock": true
|
|
8070
|
-
}`;
|
|
8071
|
-
var MONGODB_PROMPT = `When querying MongoDB, you can ONLY use the operators listed below. Any other operators will be rejected.
|
|
8072
|
-
Important: Don't explain how to construct the filter - use the specified operators and fields to search the content and return relevant results.
|
|
8073
|
-
If a user tries to give an explicit operator that is not supported, reject the filter entirely and let them know that the operator is not supported.
|
|
8074
|
-
|
|
8075
|
-
Basic Comparison Operators:
|
|
8076
|
-
- $eq: Exact match (default when using field: value)
|
|
8077
|
-
Example: { "category": "electronics" }
|
|
8078
|
-
- $ne: Not equal
|
|
8079
|
-
Example: { "category": { "$ne": "electronics" } }
|
|
8080
|
-
- $gt: Greater than
|
|
8081
|
-
Example: { "price": { "$gt": 100 } }
|
|
8082
|
-
- $gte: Greater than or equal
|
|
8083
|
-
Example: { "price": { "$gte": 100 } }
|
|
8084
|
-
- $lt: Less than
|
|
8085
|
-
Example: { "price": { "$lt": 100 } }
|
|
8086
|
-
- $lte: Less than or equal
|
|
8087
|
-
Example: { "price": { "$lte": 100 } }
|
|
8088
|
-
|
|
8089
|
-
Array Operators:
|
|
8090
|
-
- $in: Match any value in array
|
|
8091
|
-
Example: { "category": { "$in": ["electronics", "books"] } }
|
|
8092
|
-
- $nin: Does not match any value in array
|
|
8093
|
-
Example: { "category": { "$nin": ["electronics", "books"] } }
|
|
8094
|
-
- $all: Match all values in array
|
|
8095
|
-
Example: { "tags": { "$all": ["premium", "sale"] } }
|
|
8096
|
-
- $elemMatch: Match array elements by criteria
|
|
8097
|
-
Example: { "scores": { "$elemMatch": { "$gt": 80 } } }
|
|
8098
|
-
|
|
8099
|
-
Logical Operators:
|
|
8100
|
-
- $and: Logical AND (can be implicit or explicit)
|
|
8101
|
-
Implicit Example: { "price": { "$gt": 100 }, "category": "electronics" }
|
|
8102
|
-
Explicit Example: { "$and": [{ "price": { "$gt": 100 } }, { "category": "electronics" }] }
|
|
8103
|
-
- $or: Logical OR
|
|
8104
|
-
Example: { "$or": [{ "price": { "$lt": 50 } }, { "category": "books" }] }
|
|
8105
|
-
- $not: Logical NOT
|
|
8106
|
-
Example: { "field": { "$not": { "$eq": "value" } } }
|
|
8107
|
-
- $nor: Logical NOR
|
|
8108
|
-
Example: { "$nor": [{ "price": { "$lt": 50 } }, { "category": "books" }] }
|
|
8109
|
-
|
|
8110
|
-
Element Operators:
|
|
8111
|
-
- $exists: Check if field exists
|
|
8112
|
-
Example: { "rating": { "$exists": true } }
|
|
8113
|
-
|
|
8114
|
-
Special Operators:
|
|
8115
|
-
- $regex: Regular expression match
|
|
8116
|
-
Example: { "title": { "$regex": "^laptop", "$options": "i" } }
|
|
8117
|
-
- $size: Array length check
|
|
8118
|
-
Example: { "tags": { "$size": 2 } }
|
|
8119
|
-
|
|
8120
|
-
Usage Notes:
|
|
8121
|
-
- You can use both 'filter' (for metadata fields) and 'documentFilter' (for document content fields).
|
|
8122
|
-
- Nested fields are supported using dot notation (e.g., "metadata.author.name").
|
|
8123
|
-
- Multiple conditions on the same field are supported with both implicit and explicit $and.
|
|
8124
|
-
- Empty arrays in $in/$nin will return no results.
|
|
8125
|
-
- All logical operators ($and, $or, $not, $nor) can be used at the top level or nested.
|
|
8126
|
-
- All other operators must be used within a field condition.
|
|
8127
|
-
Valid: { "field": { "$gt": 100 } }
|
|
8128
|
-
Valid: { "$and": [...] }
|
|
8129
|
-
Invalid: { "$gt": 100 }
|
|
8130
|
-
- $not operator:
|
|
8131
|
-
- Must be an object
|
|
8132
|
-
- Cannot be empty
|
|
8133
|
-
- Can be used at field level or top level
|
|
8134
|
-
- Valid: { "$not": { "field": "value" } }
|
|
8135
|
-
- Valid: { "field": { "$not": { "$eq": "value" } } }
|
|
8136
|
-
- Logical operators must contain field conditions, not direct operators
|
|
8137
|
-
Valid: { "$and": [{ "field": { "$gt": 100 } }] }
|
|
8138
|
-
Invalid: { "$and": [{ "$gt": 100 }] }
|
|
8139
|
-
- Regex uses standard MongoDB regex syntax (with optional $options).
|
|
8140
|
-
- Metadata values can be strings, numbers, booleans, or arrays.
|
|
8141
|
-
- Metadata and document fields can be filtered in the same query.
|
|
8142
|
-
|
|
8143
|
-
Example Complex Query:
|
|
8144
|
-
{
|
|
8145
|
-
"category": { "$in": ["electronics", "computers"] },
|
|
8146
|
-
"price": { "$gte": 100, "$lte": 1000 },
|
|
8147
|
-
"inStock": true,
|
|
8148
|
-
"title": { "$regex": "laptop", "$options": "i" },
|
|
8149
|
-
"$or": [
|
|
8150
|
-
{ "brand": "Apple" },
|
|
8151
|
-
{ "rating": { "$gte": 4.5 } }
|
|
8152
|
-
]
|
|
8153
|
-
}
|
|
8154
|
-
`;
|
|
8155
|
-
|
|
8156
|
-
exports.ASTRA_PROMPT = ASTRA_PROMPT;
|
|
8157
|
-
exports.CHROMA_PROMPT = CHROMA_PROMPT;
|
|
8158
7881
|
exports.CohereRelevanceScorer = CohereRelevanceScorer;
|
|
8159
7882
|
exports.GraphRAG = GraphRAG;
|
|
8160
|
-
exports.LIBSQL_PROMPT = LIBSQL_PROMPT;
|
|
8161
7883
|
exports.Language = Language;
|
|
8162
7884
|
exports.MDocument = MDocument;
|
|
8163
|
-
exports.MONGODB_PROMPT = MONGODB_PROMPT;
|
|
8164
7885
|
exports.MastraAgentRelevanceScorer = MastraAgentRelevanceScorer;
|
|
8165
|
-
exports.PGVECTOR_PROMPT = PGVECTOR_PROMPT;
|
|
8166
|
-
exports.PINECONE_PROMPT = PINECONE_PROMPT;
|
|
8167
|
-
exports.QDRANT_PROMPT = QDRANT_PROMPT;
|
|
8168
|
-
exports.UPSTASH_PROMPT = UPSTASH_PROMPT;
|
|
8169
|
-
exports.VECTORIZE_PROMPT = VECTORIZE_PROMPT;
|
|
8170
7886
|
exports.ZeroEntropyRelevanceScorer = ZeroEntropyRelevanceScorer;
|
|
8171
7887
|
exports.createDocumentChunkerTool = createDocumentChunkerTool;
|
|
8172
7888
|
exports.createGraphRAGTool = createGraphRAGTool;
|