@robthepcguy/rag-vault 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +0 -0
- package/README.md +0 -0
- package/dist/bin/install-skills.d.ts +0 -0
- package/dist/bin/install-skills.d.ts.map +0 -0
- package/dist/bin/install-skills.js +0 -0
- package/dist/bin/install-skills.js.map +0 -0
- package/dist/chunker/index.d.ts +0 -0
- package/dist/chunker/index.d.ts.map +0 -0
- package/dist/chunker/index.js +0 -0
- package/dist/chunker/index.js.map +0 -0
- package/dist/chunker/semantic-chunker.d.ts +0 -0
- package/dist/chunker/semantic-chunker.d.ts.map +0 -0
- package/dist/chunker/semantic-chunker.js +0 -0
- package/dist/chunker/semantic-chunker.js.map +0 -0
- package/dist/chunker/sentence-splitter.d.ts +0 -0
- package/dist/chunker/sentence-splitter.d.ts.map +0 -0
- package/dist/chunker/sentence-splitter.js +0 -0
- package/dist/chunker/sentence-splitter.js.map +0 -0
- package/dist/embedder/index.d.ts +0 -0
- package/dist/embedder/index.d.ts.map +0 -0
- package/dist/embedder/index.js +0 -0
- package/dist/embedder/index.js.map +0 -0
- package/dist/errors/index.d.ts +0 -0
- package/dist/errors/index.d.ts.map +0 -0
- package/dist/errors/index.js +0 -0
- package/dist/errors/index.js.map +0 -0
- package/dist/explainability/index.d.ts +0 -0
- package/dist/explainability/index.d.ts.map +0 -0
- package/dist/explainability/index.js +0 -0
- package/dist/explainability/index.js.map +0 -0
- package/dist/explainability/keywords.d.ts +0 -0
- package/dist/explainability/keywords.d.ts.map +0 -0
- package/dist/explainability/keywords.js +0 -0
- package/dist/explainability/keywords.js.map +0 -0
- package/dist/flywheel/feedback.d.ts +12 -1
- package/dist/flywheel/feedback.d.ts.map +1 -1
- package/dist/flywheel/feedback.js +63 -8
- package/dist/flywheel/feedback.js.map +1 -1
- package/dist/flywheel/index.d.ts +1 -1
- package/dist/flywheel/index.d.ts.map +1 -1
- package/dist/flywheel/index.js +1 -2
- package/dist/flywheel/index.js.map +1 -1
- package/dist/index.d.ts +0 -0
- package/dist/index.d.ts.map +0 -0
- package/dist/index.js +11 -1
- package/dist/index.js.map +1 -1
- package/dist/parser/html-parser.d.ts +0 -0
- package/dist/parser/html-parser.d.ts.map +0 -0
- package/dist/parser/html-parser.js +0 -0
- package/dist/parser/html-parser.js.map +0 -0
- package/dist/parser/index.d.ts +0 -0
- package/dist/parser/index.d.ts.map +0 -0
- package/dist/parser/index.js +0 -0
- package/dist/parser/index.js.map +0 -0
- package/dist/parser/pdf-filter.d.ts +0 -0
- package/dist/parser/pdf-filter.d.ts.map +0 -0
- package/dist/parser/pdf-filter.js +0 -0
- package/dist/parser/pdf-filter.js.map +0 -0
- package/dist/query/index.d.ts +2 -0
- package/dist/query/index.d.ts.map +1 -0
- package/dist/query/index.js +10 -0
- package/dist/query/index.js.map +1 -0
- package/dist/query/parser.d.ts +57 -0
- package/dist/query/parser.d.ts.map +1 -0
- package/dist/query/parser.js +213 -0
- package/dist/query/parser.js.map +1 -0
- package/dist/server/index.d.ts +19 -0
- package/dist/server/index.d.ts.map +1 -1
- package/dist/server/index.js +209 -43
- package/dist/server/index.js.map +1 -1
- package/dist/server/raw-data-utils.d.ts +0 -0
- package/dist/server/raw-data-utils.d.ts.map +0 -0
- package/dist/server/raw-data-utils.js +0 -0
- package/dist/server/raw-data-utils.js.map +0 -0
- package/dist/server/schemas.d.ts +34 -0
- package/dist/server/schemas.d.ts.map +1 -1
- package/dist/server/schemas.js +15 -1
- package/dist/server/schemas.js.map +1 -1
- package/dist/utils/config-parsers.d.ts +0 -0
- package/dist/utils/config-parsers.d.ts.map +0 -0
- package/dist/utils/config-parsers.js +0 -0
- package/dist/utils/config-parsers.js.map +0 -0
- package/dist/utils/config.d.ts +0 -0
- package/dist/utils/config.d.ts.map +0 -0
- package/dist/utils/config.js +0 -0
- package/dist/utils/config.js.map +0 -0
- package/dist/utils/file-utils.d.ts +9 -0
- package/dist/utils/file-utils.d.ts.map +1 -0
- package/dist/utils/file-utils.js +41 -0
- package/dist/utils/file-utils.js.map +1 -0
- package/dist/utils/math.d.ts +0 -0
- package/dist/utils/math.d.ts.map +0 -0
- package/dist/utils/math.js +0 -0
- package/dist/utils/math.js.map +0 -0
- package/dist/utils/process-handlers.d.ts +6 -2
- package/dist/utils/process-handlers.d.ts.map +1 -1
- package/dist/utils/process-handlers.js +84 -17
- package/dist/utils/process-handlers.js.map +1 -1
- package/dist/vectordb/index.d.ts +16 -3
- package/dist/vectordb/index.d.ts.map +1 -1
- package/dist/vectordb/index.js +140 -33
- package/dist/vectordb/index.js.map +1 -1
- package/dist/web/api-routes.d.ts +0 -0
- package/dist/web/api-routes.d.ts.map +0 -0
- package/dist/web/api-routes.js +0 -0
- package/dist/web/api-routes.js.map +0 -0
- package/dist/web/config-routes.d.ts +0 -0
- package/dist/web/config-routes.d.ts.map +0 -0
- package/dist/web/config-routes.js +0 -0
- package/dist/web/config-routes.js.map +0 -0
- package/dist/web/database-manager.d.ts +2 -0
- package/dist/web/database-manager.d.ts.map +1 -1
- package/dist/web/database-manager.js +53 -19
- package/dist/web/database-manager.js.map +1 -1
- package/dist/web/http-server.d.ts +0 -0
- package/dist/web/http-server.d.ts.map +1 -1
- package/dist/web/http-server.js +6 -2
- package/dist/web/http-server.js.map +1 -1
- package/dist/web/index.d.ts +0 -0
- package/dist/web/index.d.ts.map +0 -0
- package/dist/web/index.js +5 -2
- package/dist/web/index.js.map +1 -1
- package/dist/web/middleware/async-handler.d.ts +0 -0
- package/dist/web/middleware/async-handler.d.ts.map +0 -0
- package/dist/web/middleware/async-handler.js +0 -0
- package/dist/web/middleware/async-handler.js.map +0 -0
- package/dist/web/middleware/auth.d.ts +0 -0
- package/dist/web/middleware/auth.d.ts.map +0 -0
- package/dist/web/middleware/auth.js +0 -0
- package/dist/web/middleware/auth.js.map +0 -0
- package/dist/web/middleware/error-handler.d.ts +0 -0
- package/dist/web/middleware/error-handler.d.ts.map +0 -0
- package/dist/web/middleware/error-handler.js +0 -0
- package/dist/web/middleware/error-handler.js.map +0 -0
- package/dist/web/middleware/index.d.ts +0 -0
- package/dist/web/middleware/index.d.ts.map +0 -0
- package/dist/web/middleware/index.js +0 -0
- package/dist/web/middleware/index.js.map +0 -0
- package/dist/web/middleware/rate-limit.d.ts +0 -0
- package/dist/web/middleware/rate-limit.d.ts.map +1 -1
- package/dist/web/middleware/rate-limit.js +16 -9
- package/dist/web/middleware/rate-limit.js.map +1 -1
- package/dist/web/middleware/request-logger.d.ts +0 -0
- package/dist/web/middleware/request-logger.d.ts.map +0 -0
- package/dist/web/middleware/request-logger.js +0 -0
- package/dist/web/middleware/request-logger.js.map +0 -0
- package/dist/web/types.d.ts +0 -0
- package/dist/web/types.d.ts.map +0 -0
- package/dist/web/types.js +0 -0
- package/dist/web/types.js.map +0 -0
- package/package.json +116 -129
- package/skills/rag-vault/SKILL.md +0 -0
- package/skills/rag-vault/references/html-ingestion.md +0 -0
- package/skills/rag-vault/references/query-optimization.md +0 -0
- package/skills/rag-vault/references/result-refinement.md +0 -0
- package/web-ui/dist/assets/index-BcRp9-z9.js +0 -0
- package/web-ui/dist/assets/index-ej8i4PGl.css +0 -0
- package/web-ui/dist/index.html +0 -0
- package/web-ui/dist/vite.svg +0 -0
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Advanced Query Syntax Parser
|
|
3
|
+
// Supports: phrases, boolean operators, field filters, exclusions
|
|
4
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
+
exports.parseQuery = parseQuery;
|
|
6
|
+
exports.toSemanticQuery = toSemanticQuery;
|
|
7
|
+
exports.toFtsQuery = toFtsQuery;
|
|
8
|
+
exports.shouldExclude = shouldExclude;
|
|
9
|
+
exports.matchesFilters = matchesFilters;
|
|
10
|
+
/**
|
|
11
|
+
* Tokenize a query string into tokens
|
|
12
|
+
*/
|
|
13
|
+
function tokenize(query) {
|
|
14
|
+
const tokens = [];
|
|
15
|
+
let i = 0;
|
|
16
|
+
while (i < query.length) {
|
|
17
|
+
// Skip whitespace
|
|
18
|
+
if (/\s/.test(query[i] ?? '')) {
|
|
19
|
+
i++;
|
|
20
|
+
continue;
|
|
21
|
+
}
|
|
22
|
+
// Handle quoted phrases
|
|
23
|
+
if (query[i] === '"') {
|
|
24
|
+
const start = i + 1;
|
|
25
|
+
i++;
|
|
26
|
+
while (i < query.length && query[i] !== '"') {
|
|
27
|
+
i++;
|
|
28
|
+
}
|
|
29
|
+
const phrase = query.slice(start, i);
|
|
30
|
+
if (phrase.length > 0) {
|
|
31
|
+
tokens.push({ type: 'PHRASE', value: phrase });
|
|
32
|
+
}
|
|
33
|
+
i++; // Skip closing quote
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
// Handle parentheses
|
|
37
|
+
if (query[i] === '(') {
|
|
38
|
+
tokens.push({ type: 'LPAREN', value: '(' });
|
|
39
|
+
i++;
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
if (query[i] === ')') {
|
|
43
|
+
tokens.push({ type: 'RPAREN', value: ')' });
|
|
44
|
+
i++;
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
// Handle exclusion prefix
|
|
48
|
+
if (query[i] === '-') {
|
|
49
|
+
const start = i + 1;
|
|
50
|
+
i++;
|
|
51
|
+
// Read the term after -
|
|
52
|
+
while (i < query.length && !/[\s()"]/.test(query[i] ?? '')) {
|
|
53
|
+
i++;
|
|
54
|
+
}
|
|
55
|
+
const term = query.slice(start, i);
|
|
56
|
+
if (term.length > 0) {
|
|
57
|
+
tokens.push({ type: 'EXCLUDE', value: term });
|
|
58
|
+
}
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
// Read a word (including potential field:value)
|
|
62
|
+
const start = i;
|
|
63
|
+
while (i < query.length && !/[\s()"]/.test(query[i] ?? '')) {
|
|
64
|
+
i++;
|
|
65
|
+
}
|
|
66
|
+
const word = query.slice(start, i);
|
|
67
|
+
if (word.length === 0)
|
|
68
|
+
continue;
|
|
69
|
+
// Check for boolean operators (case-insensitive)
|
|
70
|
+
const upperWord = word.toUpperCase();
|
|
71
|
+
if (upperWord === 'AND') {
|
|
72
|
+
tokens.push({ type: 'AND', value: 'AND' });
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
if (upperWord === 'OR') {
|
|
76
|
+
tokens.push({ type: 'OR', value: 'OR' });
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
// Check for field:value filter
|
|
80
|
+
const colonIdx = word.indexOf(':');
|
|
81
|
+
if (colonIdx > 0 && colonIdx < word.length - 1) {
|
|
82
|
+
const field = word.slice(0, colonIdx);
|
|
83
|
+
const value = word.slice(colonIdx + 1);
|
|
84
|
+
tokens.push({ type: 'FILTER', value, field });
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
// Regular term
|
|
88
|
+
tokens.push({ type: 'TERM', value: word });
|
|
89
|
+
}
|
|
90
|
+
return tokens;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Parse a query string into structured query parts
|
|
94
|
+
*
|
|
95
|
+
* Syntax:
|
|
96
|
+
* - "exact phrase" → Match phrase exactly
|
|
97
|
+
* - field:value → Filter by metadata field
|
|
98
|
+
* - -term → Exclude term from results
|
|
99
|
+
* - term1 AND term2 → Both terms required (default)
|
|
100
|
+
* - term1 OR term2 → Either term matches
|
|
101
|
+
* - (group) → Group expressions (future: for complex boolean)
|
|
102
|
+
*
|
|
103
|
+
* @param query - Raw query string
|
|
104
|
+
* @returns Parsed query structure
|
|
105
|
+
*/
|
|
106
|
+
function parseQuery(query) {
|
|
107
|
+
const result = {
|
|
108
|
+
semanticTerms: [],
|
|
109
|
+
phrases: [],
|
|
110
|
+
filters: [],
|
|
111
|
+
excludeTerms: [],
|
|
112
|
+
booleanOp: 'AND',
|
|
113
|
+
originalQuery: query,
|
|
114
|
+
};
|
|
115
|
+
if (!query || query.trim().length === 0) {
|
|
116
|
+
return result;
|
|
117
|
+
}
|
|
118
|
+
const tokens = tokenize(query);
|
|
119
|
+
// Track if we've seen OR (switches to OR mode)
|
|
120
|
+
let hasOr = false;
|
|
121
|
+
for (const token of tokens) {
|
|
122
|
+
switch (token.type) {
|
|
123
|
+
case 'PHRASE':
|
|
124
|
+
result.phrases.push(token.value);
|
|
125
|
+
// Also add phrase words to semantic terms for vector search
|
|
126
|
+
result.semanticTerms.push(token.value);
|
|
127
|
+
break;
|
|
128
|
+
case 'FILTER':
|
|
129
|
+
if (token.field) {
|
|
130
|
+
result.filters.push({ field: token.field, value: token.value });
|
|
131
|
+
}
|
|
132
|
+
break;
|
|
133
|
+
case 'EXCLUDE':
|
|
134
|
+
result.excludeTerms.push(token.value);
|
|
135
|
+
break;
|
|
136
|
+
case 'OR':
|
|
137
|
+
hasOr = true;
|
|
138
|
+
break;
|
|
139
|
+
case 'AND':
|
|
140
|
+
// AND is default, no action needed
|
|
141
|
+
break;
|
|
142
|
+
case 'TERM':
|
|
143
|
+
result.semanticTerms.push(token.value);
|
|
144
|
+
break;
|
|
145
|
+
case 'LPAREN':
|
|
146
|
+
case 'RPAREN':
|
|
147
|
+
// Parentheses are captured for future complex boolean support
|
|
148
|
+
// Currently ignored in favor of simple AND/OR detection
|
|
149
|
+
break;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
// Set boolean mode based on presence of OR
|
|
153
|
+
if (hasOr) {
|
|
154
|
+
result.booleanOp = 'OR';
|
|
155
|
+
}
|
|
156
|
+
return result;
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Convert parsed query to a semantic search query string
|
|
160
|
+
* Joins semantic terms and phrases for embedding
|
|
161
|
+
*/
|
|
162
|
+
function toSemanticQuery(parsed) {
|
|
163
|
+
// Combine semantic terms for vector search
|
|
164
|
+
const terms = [...parsed.semanticTerms];
|
|
165
|
+
// Filter out excluded terms from semantic query
|
|
166
|
+
const filtered = terms.filter((term) => {
|
|
167
|
+
const lowerTerm = term.toLowerCase();
|
|
168
|
+
return !parsed.excludeTerms.some((ex) => lowerTerm.includes(ex.toLowerCase()));
|
|
169
|
+
});
|
|
170
|
+
return filtered.join(' ');
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Convert parsed query to a full-text search query string
|
|
174
|
+
* Uses FTS-compatible syntax for phrase matching
|
|
175
|
+
*/
|
|
176
|
+
function toFtsQuery(parsed) {
|
|
177
|
+
const parts = [];
|
|
178
|
+
// Add phrases with quotes for exact match
|
|
179
|
+
for (const phrase of parsed.phrases) {
|
|
180
|
+
parts.push(`"${phrase}"`);
|
|
181
|
+
}
|
|
182
|
+
// Add regular terms
|
|
183
|
+
for (const term of parsed.semanticTerms) {
|
|
184
|
+
// Skip if term is already in a phrase
|
|
185
|
+
if (!parsed.phrases.some((p) => p.includes(term))) {
|
|
186
|
+
parts.push(term);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
return parts.join(' ');
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Check if a result should be excluded based on exclude terms
|
|
193
|
+
*/
|
|
194
|
+
function shouldExclude(text, excludeTerms) {
|
|
195
|
+
const lowerText = text.toLowerCase();
|
|
196
|
+
return excludeTerms.some((term) => lowerText.includes(term.toLowerCase()));
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Check if a result matches the metadata filters
|
|
200
|
+
*/
|
|
201
|
+
function matchesFilters(metadata, filters) {
|
|
202
|
+
if (filters.length === 0)
|
|
203
|
+
return true;
|
|
204
|
+
if (!metadata)
|
|
205
|
+
return false;
|
|
206
|
+
return filters.every((filter) => {
|
|
207
|
+
const value = metadata[filter.field];
|
|
208
|
+
if (value === undefined)
|
|
209
|
+
return false;
|
|
210
|
+
return value.toLowerCase().includes(filter.value.toLowerCase());
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
//# sourceMappingURL=parser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/query/parser.ts"],"names":[],"mappings":";AAAA,+BAA+B;AAC/B,kEAAkE;;AAyIlE,gCA+DC;AAMD,0CAWC;AAMD,gCAiBC;AAKD,sCAGC;AAKD,wCAYC;AA1OD;;GAEG;AACH,SAAS,QAAQ,CAAC,KAAa;IAC7B,MAAM,MAAM,GAAY,EAAE,CAAA;IAC1B,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,kBAAkB;QAClB,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YAC9B,CAAC,EAAE,CAAA;YACH,SAAQ;QACV,CAAC;QAED,wBAAwB;QACxB,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YACrB,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,CAAA;YACnB,CAAC,EAAE,CAAA;YACH,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;gBAC5C,CAAC,EAAE,CAAA;YACL,CAAC;YACD,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;YACpC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAA;YAChD,CAAC;YACD,CAAC,EAAE,CAAA,CAAC,qBAAqB;YACzB,SAAQ;QACV,CAAC;QAED,qBAAqB;QACrB,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;YAC3C,CAAC,EAAE,CAAA;YACH,SAAQ;QACV,CAAC;QACD,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAA;YAC3C,CAAC,EAAE,CAAA;YACH,SAAQ;QACV,CAAC;QAED,0BAA0B;QAC1B,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YACrB,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,CAAA;YACnB,CAAC,EAAE,CAAA;YACH,wBAAwB;YACxB,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;gBAC3D,CAAC,EAAE,CAAA;YACL,CAAC;YACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;YAClC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;YAC/C,CAAC;YACD,SAAQ;QACV,CAAC;QAED,gDAAgD;QAChD,MAAM,KAAK,GAAG,CAAC,CAAA;QACf,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YAC3D,CAAC,EAAE,CAAA;QACL,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAElC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,SAAQ;QAE/B,iDAAiD;QACjD,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAA;QACpC,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAA;YAC1C,SAAQ;QACV,CAAC;QACD,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;YACxC,SAAQ;QACV,CAAC;QAED,+BAA+B;QAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAClC,IAAI,QAAQ,GAAG,CAAC,IAAI,QAAQ,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAA;YACrC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAA;YACtC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAA;YAC7C,SAAQ;QACV,CAAC;QAED,eAAe;QACf,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAgB,UAAU,CAAC,KAAa;IACtC,MAAM,MAAM,GAAgB;QAC1B,aAAa,EAAE,EAAE;QACjB,OAAO,EAAE,EAAE;QACX,OAAO,EAAE,EAAE;QACX,YAAY,EAAE,EAAE;QAChB,SAAS,EAAE,KAAK;QAChB,aAAa,EAAE,KAAK;KACrB,CAAA;IAED,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxC,OAAO,MAAM,CAAA;IACf,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;IAE9B,+CAA+C;IAC/C,IAAI,KAAK,GAAG,KAAK,CAAA;IAEjB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,QAAQ;gBACX,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBAChC,4DAA4D;gBAC5D,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBACtC,MAAK;YAEP,KAAK,QAAQ;gBACX,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;oBAChB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAA;gBACjE,CAAC;gBACD,MAAK;YAEP,KAAK,SAAS;gBACZ,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBACrC,MAAK;YAEP,KAAK,IAAI;gBACP,KAAK,GAAG,IAAI,CAAA;gBACZ,MAAK;YAEP,KAAK,KAAK;gBACR,mCAAmC;gBACnC,MAAK;YAEP,KAAK,MAAM;gBACT,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;gBACtC,MAAK;YAEP,KAAK,QAAQ,CAAC;YACd,KAAK,QAAQ;gBACX,8DAA8D;gBAC9D,wDAAwD;gBACxD,MAAK;QACT,CAAC;IACH,CAAC;IAED,2CAA2C;IAC3C,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,CAAC,SAAS,GAAG,IAAI,CAAA;IACzB,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;;GAGG;AACH,SAAgB,eAAe,CAAC,MAAmB;IACjD,2CAA2C;IAC3C,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC,CAAA;IAEvC,gDAAgD;IAChD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAA;QACpC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;IAChF,CAAC,CAAC,CAAA;IAEF,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AAC3B,CAAC;AAED;;;GAGG;AACH,SAAgB,UAAU,CAAC,MAAmB;IAC5C,MAAM,KAAK,GAAa,EAAE,CAAA;IAE1B,0CAA0C;IAC1C,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,IAAI,MAAM,GAAG,CAAC,CAAA;IAC3B,CAAC;IAED,oBAAoB;IACpB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QACxC,sCAAsC;QACtC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YAClD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACxB,CAAC;AAED;;GAEG;AACH,SAAgB,aAAa,CAAC,IAAY,EAAE,YAAsB;IAChE,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAA;IACpC,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;AAC5E,CAAC;AAED;;GAEG;AACH,SAAgB,cAAc,CAC5B,QAA4C,EAC5C,OAA2C;IAE3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IACrC,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAA;IAE3B,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACpC,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAA;QACrC,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;IACjE,CAAC,CAAC,CAAA;AACJ,CAAC"}
|
package/dist/server/index.d.ts
CHANGED
|
@@ -70,6 +70,13 @@ export declare class RAGServer {
|
|
|
70
70
|
setHybridWeight(weight: number): void;
|
|
71
71
|
/**
|
|
72
72
|
* Execute query_documents logic (returns plain data)
|
|
73
|
+
*
|
|
74
|
+
* Supports advanced query syntax:
|
|
75
|
+
* - "exact phrase" → Phrase matching (FTS)
|
|
76
|
+
* - field:value → Metadata filter
|
|
77
|
+
* - term1 AND term2 → Both required (default)
|
|
78
|
+
* - term1 OR term2 → Either matches
|
|
79
|
+
* - -term → Exclude term
|
|
73
80
|
*/
|
|
74
81
|
private executeQueryDocuments;
|
|
75
82
|
/**
|
|
@@ -133,6 +140,18 @@ export declare class RAGServer {
|
|
|
133
140
|
text: string;
|
|
134
141
|
}];
|
|
135
142
|
}>;
|
|
143
|
+
/**
|
|
144
|
+
* Execute feedback_pin logic
|
|
145
|
+
*/
|
|
146
|
+
private executeFeedbackPin;
|
|
147
|
+
/**
|
|
148
|
+
* Execute feedback_dismiss logic
|
|
149
|
+
*/
|
|
150
|
+
private executeFeedbackDismiss;
|
|
151
|
+
/**
|
|
152
|
+
* Execute feedback_stats logic
|
|
153
|
+
*/
|
|
154
|
+
private executeFeedbackStats;
|
|
136
155
|
/**
|
|
137
156
|
* Execute delete_file logic (returns plain data)
|
|
138
157
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":"AAeA,OAAO,EAAE,KAAK,YAAY,EAAiC,MAAM,sBAAsB,CAAA;AAQvF,OAAO,EACL,KAAK,eAAe,EAIpB,KAAK,eAAe,EACpB,KAAK,eAAe,EAEpB,KAAK,mBAAmB,EAGzB,MAAM,cAAc,CAAA;AAMrB;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAA;IACd,iCAAiC;IACjC,SAAS,EAAE,MAAM,CAAA;IACjB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,8BAA8B;IAC9B,OAAO,EAAE,MAAM,CAAA;IACf,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAA;IACnB,kEAAkE;IAClE,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,qDAAqD;IACrD,QAAQ,CAAC,EAAE,YAAY,CAAA;IACvB,sFAAsF;IACtF,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAMD;;;;;;;;GAQG;AACH,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAa;IACzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAU;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiB;IACzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;gBAEnB,MAAM,EAAE,eAAe;IAqCnC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAsLrB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAKjC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B;;OAEG;IACH,SAAS,IAAI;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE;IAOlD;;;OAGG;IACH,eAAe,IAAI,MAAM;IAIzB;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAIrC;;;;;;;;;OASG;YACW,qBAAqB;IAgFnC;;OAEG;IACG,oBAAoB,CACxB,IAAI,EAAE,mBAAmB,GACxB,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAiBzD;;OAEG;YACW,iBAAiB;IAiG/B;;OAEG;IACG,gBAAgB,CACpB,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAkBzD;;OAEG;YACW,iBAAiB;IAgD/B;;OAEG;IACG,gBAAgB,CACpB,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAkBzD;;OAEG;YACW,gBAAgB;IAe9B;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAiB/E;;OAEG;YACW,aAAa;IAI3B;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAiB5E;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAmC1B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAmC9B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAK5B;;OAEG;YACW,iBAAiB;IAyC/B;;OAEG;IACG,gBAAgB,CACpB,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAkBzD;;OAEG;IACG,uBAAuB,CAC3B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IA8BzD;;OAEG;IACG,uBAAuB,CAC3B,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,KAAK,CAAC,EAAE,MAAM,EACd,mBAAmB,CAAC,EAAE,OAAO,GAC5B,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAmCzD;;OAEG;IACG,4BAA4B,CAChC,MAAM,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,EACvD,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IA2CzD;;OAEG;IACG,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAK3B"}
|
package/dist/server/index.js
CHANGED
|
@@ -10,9 +10,12 @@ const zod_1 = require("zod");
|
|
|
10
10
|
const index_js_1 = require("../chunker/index.js");
|
|
11
11
|
const index_js_2 = require("../embedder/index.js");
|
|
12
12
|
const index_js_3 = require("../errors/index.js");
|
|
13
|
+
const keywords_js_1 = require("../explainability/keywords.js");
|
|
14
|
+
const feedback_js_1 = require("../flywheel/feedback.js");
|
|
13
15
|
const html_parser_js_1 = require("../parser/html-parser.js");
|
|
14
16
|
const index_js_4 = require("../parser/index.js");
|
|
15
|
-
const index_js_5 = require("../
|
|
17
|
+
const index_js_5 = require("../query/index.js");
|
|
18
|
+
const index_js_6 = require("../vectordb/index.js");
|
|
16
19
|
const raw_data_utils_js_1 = require("./raw-data-utils.js");
|
|
17
20
|
const schemas_js_1 = require("./schemas.js");
|
|
18
21
|
// ============================================
|
|
@@ -49,7 +52,7 @@ class RAGServer {
|
|
|
49
52
|
if (config.hybridWeight !== undefined) {
|
|
50
53
|
vectorStoreConfig.hybridWeight = config.hybridWeight;
|
|
51
54
|
}
|
|
52
|
-
this.vectorStore = new
|
|
55
|
+
this.vectorStore = new index_js_6.VectorStore(vectorStoreConfig);
|
|
53
56
|
this.embedder = new index_js_2.Embedder({
|
|
54
57
|
modelPath: config.modelName,
|
|
55
58
|
batchSize: 16,
|
|
@@ -68,9 +71,16 @@ class RAGServer {
|
|
|
68
71
|
*/
|
|
69
72
|
setupHandlers() {
|
|
70
73
|
// query_documents tool
|
|
71
|
-
this.server.tool('query_documents',
|
|
74
|
+
this.server.tool('query_documents', `Search ingested documents using hybrid semantic + keyword search. Advanced syntax supported:
|
|
75
|
+
- "exact phrase" → Match phrase exactly
|
|
76
|
+
- field:value → Filter by custom metadata (e.g., domain:legal, author:john)
|
|
77
|
+
- term1 AND term2 → Both terms required (default)
|
|
78
|
+
- term1 OR term2 → Either term matches
|
|
79
|
+
- -term → Exclude results containing term
|
|
80
|
+
Results include score (0 = most relevant, higher = less relevant). Set explain=true to see why each result matched.`, {
|
|
72
81
|
query: zod_1.z.string(),
|
|
73
82
|
limit: zod_1.z.number().optional(),
|
|
83
|
+
explain: zod_1.z.boolean().optional(),
|
|
74
84
|
}, async (args) => {
|
|
75
85
|
const results = await this.executeQueryDocuments(args);
|
|
76
86
|
return {
|
|
@@ -78,8 +88,9 @@ class RAGServer {
|
|
|
78
88
|
};
|
|
79
89
|
});
|
|
80
90
|
// ingest_file tool
|
|
81
|
-
this.server.tool('ingest_file', 'Ingest a document file (PDF, DOCX, TXT, MD, JSON, JSONL) into the vector database for semantic search. File path must be an absolute path. Supports re-ingestion to update existing documents.', {
|
|
91
|
+
this.server.tool('ingest_file', 'Ingest a document file (PDF, DOCX, TXT, MD, JSON, JSONL) into the vector database for semantic search. File path must be an absolute path. Supports re-ingestion to update existing documents. Optional metadata can include author, domain, tags, etc.', {
|
|
82
92
|
filePath: zod_1.z.string(),
|
|
93
|
+
metadata: zod_1.z.record(zod_1.z.string()).optional(),
|
|
83
94
|
}, async (args) => {
|
|
84
95
|
const result = await this.executeIngestFile(args);
|
|
85
96
|
return {
|
|
@@ -87,11 +98,12 @@ class RAGServer {
|
|
|
87
98
|
};
|
|
88
99
|
});
|
|
89
100
|
// ingest_data tool
|
|
90
|
-
this.server.tool('ingest_data', 'Ingest content as a string, not from a file. Use for: fetched web pages (format: html), copied text (format: text), or markdown strings (format: markdown). The source identifier enables re-ingestion to update existing content. For files on disk, use ingest_file instead.', {
|
|
101
|
+
this.server.tool('ingest_data', 'Ingest content as a string, not from a file. Use for: fetched web pages (format: html), copied text (format: text), or markdown strings (format: markdown). The source identifier enables re-ingestion to update existing content. Optional custom metadata can include author, domain, tags, etc. For files on disk, use ingest_file instead.', {
|
|
91
102
|
content: zod_1.z.string(),
|
|
92
103
|
metadata: zod_1.z.object({
|
|
93
104
|
source: zod_1.z.string(),
|
|
94
105
|
format: zod_1.z.enum(['text', 'html', 'markdown']),
|
|
106
|
+
custom: zod_1.z.record(zod_1.z.string()).optional(),
|
|
95
107
|
}),
|
|
96
108
|
}, async (args) => {
|
|
97
109
|
const result = await this.executeIngestData(args);
|
|
@@ -128,6 +140,61 @@ class RAGServer {
|
|
|
128
140
|
content: [{ type: 'text', text: JSON.stringify(status, null, 2) }],
|
|
129
141
|
};
|
|
130
142
|
});
|
|
143
|
+
// feedback_pin tool
|
|
144
|
+
this.server.tool('feedback_pin', 'Pin a search result as relevant for a query. Pinned results will be boosted in future searches. Use when a result was particularly helpful.', {
|
|
145
|
+
sourceQuery: zod_1.z.string().describe('The query that returned this result'),
|
|
146
|
+
targetFilePath: zod_1.z.string().describe('File path of the result to pin'),
|
|
147
|
+
targetChunkIndex: zod_1.z.number().describe('Chunk index of the result to pin'),
|
|
148
|
+
targetFingerprint: zod_1.z.string().optional().describe('Optional fingerprint for resilient matching'),
|
|
149
|
+
}, async (args) => {
|
|
150
|
+
try {
|
|
151
|
+
const result = this.executeFeedbackPin(args);
|
|
152
|
+
return {
|
|
153
|
+
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
return {
|
|
158
|
+
content: [{ type: 'text', text: JSON.stringify({ error: (0, index_js_3.getErrorMessage)(error) }) }],
|
|
159
|
+
isError: true,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
});
|
|
163
|
+
// feedback_dismiss tool
|
|
164
|
+
this.server.tool('feedback_dismiss', 'Dismiss a search result as irrelevant for a query. Dismissed results will be penalized in future searches. Use when a result was unhelpful.', {
|
|
165
|
+
sourceQuery: zod_1.z.string().describe('The query that returned this result'),
|
|
166
|
+
targetFilePath: zod_1.z.string().describe('File path of the result to dismiss'),
|
|
167
|
+
targetChunkIndex: zod_1.z.number().describe('Chunk index of the result to dismiss'),
|
|
168
|
+
targetFingerprint: zod_1.z.string().optional().describe('Optional fingerprint for resilient matching'),
|
|
169
|
+
}, async (args) => {
|
|
170
|
+
try {
|
|
171
|
+
const result = this.executeFeedbackDismiss(args);
|
|
172
|
+
return {
|
|
173
|
+
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
catch (error) {
|
|
177
|
+
return {
|
|
178
|
+
content: [{ type: 'text', text: JSON.stringify({ error: (0, index_js_3.getErrorMessage)(error) }) }],
|
|
179
|
+
isError: true,
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
});
|
|
183
|
+
// feedback_stats tool
|
|
184
|
+
this.server.tool('feedback_stats', 'Get feedback statistics including total events, pinned pairs, and dismissed pairs.', {}, async () => {
|
|
185
|
+
try {
|
|
186
|
+
const stats = this.executeFeedbackStats();
|
|
187
|
+
return {
|
|
188
|
+
content: [{ type: 'text', text: JSON.stringify(stats, null, 2) }],
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
catch (error) {
|
|
192
|
+
return {
|
|
193
|
+
content: [{ type: 'text', text: JSON.stringify({ error: (0, index_js_3.getErrorMessage)(error) }) }],
|
|
194
|
+
isError: true,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
});
|
|
131
198
|
}
|
|
132
199
|
/**
|
|
133
200
|
* Initialization
|
|
@@ -168,12 +235,50 @@ class RAGServer {
|
|
|
168
235
|
}
|
|
169
236
|
/**
|
|
170
237
|
* Execute query_documents logic (returns plain data)
|
|
238
|
+
*
|
|
239
|
+
* Supports advanced query syntax:
|
|
240
|
+
* - "exact phrase" → Phrase matching (FTS)
|
|
241
|
+
* - field:value → Metadata filter
|
|
242
|
+
* - term1 AND term2 → Both required (default)
|
|
243
|
+
* - term1 OR term2 → Either matches
|
|
244
|
+
* - -term → Exclude term
|
|
171
245
|
*/
|
|
172
246
|
async executeQueryDocuments(args) {
|
|
173
|
-
//
|
|
174
|
-
const
|
|
247
|
+
// Parse query for advanced syntax
|
|
248
|
+
const parsed = (0, index_js_5.parseQuery)(args.query);
|
|
249
|
+
const semanticQuery = (0, index_js_5.toSemanticQuery)(parsed);
|
|
250
|
+
// Generate query embedding from semantic terms
|
|
251
|
+
const queryVector = await this.embedder.embed(semanticQuery || args.query);
|
|
252
|
+
// Request extra results to account for post-filtering (capped at 20)
|
|
253
|
+
const userLimit = args.limit || 10;
|
|
254
|
+
const hasFilters = parsed.excludeTerms.length > 0 || parsed.filters.length > 0;
|
|
255
|
+
const requestLimit = hasFilters ? Math.min(userLimit * 2, 20) : userLimit;
|
|
175
256
|
// Hybrid search (vector + BM25 keyword matching)
|
|
176
|
-
|
|
257
|
+
let searchResults = await this.vectorStore.search(queryVector, args.query, requestLimit);
|
|
258
|
+
// Apply flywheel reranking based on user feedback
|
|
259
|
+
const feedbackStore = (0, feedback_js_1.getFeedbackStore)();
|
|
260
|
+
const sourceRef = {
|
|
261
|
+
filePath: '__query__',
|
|
262
|
+
chunkIndex: 0,
|
|
263
|
+
fingerprint: args.query,
|
|
264
|
+
};
|
|
265
|
+
searchResults = feedbackStore.rerankResults(searchResults, sourceRef);
|
|
266
|
+
// Apply post-search filters from parsed query
|
|
267
|
+
if (parsed.excludeTerms.length > 0 || parsed.filters.length > 0) {
|
|
268
|
+
searchResults = searchResults.filter((result) => {
|
|
269
|
+
// Exclude results containing excluded terms
|
|
270
|
+
if ((0, index_js_5.shouldExclude)(result.text, parsed.excludeTerms)) {
|
|
271
|
+
return false;
|
|
272
|
+
}
|
|
273
|
+
// Filter by metadata if filters specified (only if custom metadata exists)
|
|
274
|
+
if (parsed.filters.length > 0 && !(0, index_js_5.matchesFilters)(result.metadata?.custom, parsed.filters)) {
|
|
275
|
+
return false;
|
|
276
|
+
}
|
|
277
|
+
return true;
|
|
278
|
+
});
|
|
279
|
+
}
|
|
280
|
+
// Trim to requested limit after filtering
|
|
281
|
+
searchResults = searchResults.slice(0, args.limit || 10);
|
|
177
282
|
// Format results with source restoration for raw-data files
|
|
178
283
|
return searchResults.map((result) => {
|
|
179
284
|
const queryResult = {
|
|
@@ -189,6 +294,16 @@ class RAGServer {
|
|
|
189
294
|
queryResult.source = source;
|
|
190
295
|
}
|
|
191
296
|
}
|
|
297
|
+
// Include custom metadata if present
|
|
298
|
+
if (result.metadata?.custom) {
|
|
299
|
+
queryResult.metadata = result.metadata.custom;
|
|
300
|
+
}
|
|
301
|
+
// Add explanation if requested
|
|
302
|
+
if (args.explain) {
|
|
303
|
+
const explanation = (0, keywords_js_1.explainChunkSimilarity)(args.query, result.text, false, // Not same document (query vs result)
|
|
304
|
+
result.score);
|
|
305
|
+
queryResult.explanation = explanation;
|
|
306
|
+
}
|
|
192
307
|
return queryResult;
|
|
193
308
|
});
|
|
194
309
|
}
|
|
@@ -216,7 +331,6 @@ class RAGServer {
|
|
|
216
331
|
* Execute ingest_file logic (returns plain data)
|
|
217
332
|
*/
|
|
218
333
|
async executeIngestFile(args) {
|
|
219
|
-
let backup = null;
|
|
220
334
|
// Parse file (with header/footer filtering for PDFs)
|
|
221
335
|
// For raw-data files (from ingest_data), read directly without validation
|
|
222
336
|
// since the path is internally generated and content is already processed
|
|
@@ -237,37 +351,29 @@ class RAGServer {
|
|
|
237
351
|
const chunks = await this.chunker.chunkText(text, this.embedder);
|
|
238
352
|
// Generate embeddings for final chunks
|
|
239
353
|
const embeddings = await this.embedder.embedBatch(chunks.map((chunk) => chunk.text));
|
|
240
|
-
//
|
|
354
|
+
// Note: Full backup with vectors is not implemented because search results don't include vectors.
|
|
355
|
+
// If rollback is needed, re-ingestion from the original file is required.
|
|
356
|
+
// Track if this is a re-ingestion for logging purposes.
|
|
357
|
+
let isReingestion = false;
|
|
241
358
|
try {
|
|
242
359
|
const existingFiles = await this.vectorStore.listFiles();
|
|
243
360
|
const existingFile = existingFiles.find((file) => file.filePath === args.filePath);
|
|
244
361
|
if (existingFile && existingFile.chunkCount > 0) {
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
if (queryVector.length > 0) {
|
|
248
|
-
const allChunks = await this.vectorStore.search(queryVector, undefined, 20); // Retrieve max 20 items
|
|
249
|
-
backup = allChunks
|
|
250
|
-
.filter((chunk) => chunk.filePath === args.filePath)
|
|
251
|
-
.map((chunk) => ({
|
|
252
|
-
id: (0, node_crypto_1.randomUUID)(),
|
|
253
|
-
filePath: chunk.filePath,
|
|
254
|
-
chunkIndex: chunk.chunkIndex,
|
|
255
|
-
text: chunk.text,
|
|
256
|
-
vector: queryVector, // Use dummy vector since actual vector cannot be retrieved
|
|
257
|
-
metadata: chunk.metadata,
|
|
258
|
-
timestamp: new Date().toISOString(),
|
|
259
|
-
}));
|
|
260
|
-
}
|
|
261
|
-
console.error(`Backup created: ${backup?.length || 0} chunks for ${args.filePath}`);
|
|
362
|
+
isReingestion = true;
|
|
363
|
+
console.error(`Re-ingesting existing file: ${args.filePath} (${existingFile.chunkCount} existing chunks)`);
|
|
262
364
|
}
|
|
263
365
|
}
|
|
264
366
|
catch (error) {
|
|
265
|
-
//
|
|
266
|
-
console.warn('Failed to
|
|
367
|
+
// File check failure is warning only (for new files)
|
|
368
|
+
console.warn('Failed to check existing file (new file?):', error);
|
|
267
369
|
}
|
|
268
370
|
// Delete existing data
|
|
269
371
|
await this.vectorStore.deleteChunks(args.filePath);
|
|
270
372
|
console.error(`Deleted existing chunks for: ${args.filePath}`);
|
|
373
|
+
// Validate embeddings and chunks match
|
|
374
|
+
if (embeddings.length !== chunks.length) {
|
|
375
|
+
throw new Error(`Embedding count (${embeddings.length}) doesn't match chunk count (${chunks.length})`);
|
|
376
|
+
}
|
|
271
377
|
// Create vector chunks
|
|
272
378
|
const timestamp = new Date().toISOString();
|
|
273
379
|
const vectorChunks = chunks.map((chunk, index) => {
|
|
@@ -285,6 +391,7 @@ class RAGServer {
|
|
|
285
391
|
fileName: args.filePath.split('/').pop() || args.filePath,
|
|
286
392
|
fileSize: text.length,
|
|
287
393
|
fileType: args.filePath.split('.').pop() || '',
|
|
394
|
+
...(args.metadata && { custom: args.metadata }),
|
|
288
395
|
},
|
|
289
396
|
timestamp,
|
|
290
397
|
};
|
|
@@ -293,21 +400,14 @@ class RAGServer {
|
|
|
293
400
|
try {
|
|
294
401
|
await this.vectorStore.insertChunks(vectorChunks);
|
|
295
402
|
console.error(`Inserted ${vectorChunks.length} chunks for: ${args.filePath}`);
|
|
296
|
-
// Delete backup on success
|
|
297
|
-
backup = null;
|
|
298
403
|
}
|
|
299
404
|
catch (insertError) {
|
|
300
|
-
//
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
}
|
|
307
|
-
catch (rollbackError) {
|
|
308
|
-
console.error('Rollback failed:', rollbackError);
|
|
309
|
-
throw new Error(`Failed to ingest file and rollback failed: ${insertError.message}`);
|
|
310
|
-
}
|
|
405
|
+
// Note: Full rollback is not possible without stored vectors.
|
|
406
|
+
// If this was a re-ingestion and it failed, the old data has been deleted.
|
|
407
|
+
// User should re-ingest from the original file.
|
|
408
|
+
if (isReingestion) {
|
|
409
|
+
console.error('Ingestion failed during re-ingestion. Previous data was deleted. ' +
|
|
410
|
+
'Please re-ingest from the original file to restore.', insertError);
|
|
311
411
|
}
|
|
312
412
|
throw insertError;
|
|
313
413
|
}
|
|
@@ -360,7 +460,10 @@ class RAGServer {
|
|
|
360
460
|
console.error(`Saved raw data: ${args.metadata.source} -> ${rawDataPath}`);
|
|
361
461
|
// Call executeIngestFile internally with rollback on failure
|
|
362
462
|
try {
|
|
363
|
-
return await this.executeIngestFile({
|
|
463
|
+
return await this.executeIngestFile({
|
|
464
|
+
filePath: rawDataPath,
|
|
465
|
+
...(args.metadata.custom && { metadata: args.metadata.custom }),
|
|
466
|
+
});
|
|
364
467
|
}
|
|
365
468
|
catch (ingestError) {
|
|
366
469
|
// Rollback: delete the raw-data file if ingest fails
|
|
@@ -457,6 +560,69 @@ class RAGServer {
|
|
|
457
560
|
throw error;
|
|
458
561
|
}
|
|
459
562
|
}
|
|
563
|
+
/**
|
|
564
|
+
* Execute feedback_pin logic
|
|
565
|
+
*/
|
|
566
|
+
executeFeedbackPin(args) {
|
|
567
|
+
const feedbackStore = (0, feedback_js_1.getFeedbackStore)();
|
|
568
|
+
// Create source reference from query (using query text as fingerprint)
|
|
569
|
+
const sourceRef = {
|
|
570
|
+
filePath: '__query__',
|
|
571
|
+
chunkIndex: 0,
|
|
572
|
+
fingerprint: args.sourceQuery,
|
|
573
|
+
};
|
|
574
|
+
// Create target reference
|
|
575
|
+
const targetRef = {
|
|
576
|
+
filePath: args.targetFilePath,
|
|
577
|
+
chunkIndex: args.targetChunkIndex,
|
|
578
|
+
...(args.targetFingerprint && { fingerprint: args.targetFingerprint }),
|
|
579
|
+
};
|
|
580
|
+
feedbackStore.recordEvent({
|
|
581
|
+
type: 'pin',
|
|
582
|
+
source: sourceRef,
|
|
583
|
+
target: targetRef,
|
|
584
|
+
timestamp: new Date(),
|
|
585
|
+
});
|
|
586
|
+
return {
|
|
587
|
+
success: true,
|
|
588
|
+
message: `Pinned chunk ${args.targetFilePath}:${args.targetChunkIndex} for query "${args.sourceQuery}"`,
|
|
589
|
+
};
|
|
590
|
+
}
|
|
591
|
+
/**
|
|
592
|
+
* Execute feedback_dismiss logic
|
|
593
|
+
*/
|
|
594
|
+
executeFeedbackDismiss(args) {
|
|
595
|
+
const feedbackStore = (0, feedback_js_1.getFeedbackStore)();
|
|
596
|
+
// Create source reference from query (using query text as fingerprint)
|
|
597
|
+
const sourceRef = {
|
|
598
|
+
filePath: '__query__',
|
|
599
|
+
chunkIndex: 0,
|
|
600
|
+
fingerprint: args.sourceQuery,
|
|
601
|
+
};
|
|
602
|
+
// Create target reference
|
|
603
|
+
const targetRef = {
|
|
604
|
+
filePath: args.targetFilePath,
|
|
605
|
+
chunkIndex: args.targetChunkIndex,
|
|
606
|
+
...(args.targetFingerprint && { fingerprint: args.targetFingerprint }),
|
|
607
|
+
};
|
|
608
|
+
feedbackStore.recordEvent({
|
|
609
|
+
type: 'dismiss_inferred',
|
|
610
|
+
source: sourceRef,
|
|
611
|
+
target: targetRef,
|
|
612
|
+
timestamp: new Date(),
|
|
613
|
+
});
|
|
614
|
+
return {
|
|
615
|
+
success: true,
|
|
616
|
+
message: `Dismissed chunk ${args.targetFilePath}:${args.targetChunkIndex} for query "${args.sourceQuery}"`,
|
|
617
|
+
};
|
|
618
|
+
}
|
|
619
|
+
/**
|
|
620
|
+
* Execute feedback_stats logic
|
|
621
|
+
*/
|
|
622
|
+
executeFeedbackStats() {
|
|
623
|
+
const feedbackStore = (0, feedback_js_1.getFeedbackStore)();
|
|
624
|
+
return feedbackStore.getStats();
|
|
625
|
+
}
|
|
460
626
|
/**
|
|
461
627
|
* Execute delete_file logic (returns plain data)
|
|
462
628
|
*/
|