@nexo-labs/payload-typesense 1.4.4 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +1988 -1953
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +485 -965
- package/dist/index.mjs.map +1 -1
- package/package.json +9 -39
package/dist/index.mjs
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { DEFAULT_EMBEDDING_DIMENSIONS, DEFAULT_EMBEDDING_MODEL, DEFAULT_GEMINI_EMBEDDING_MODEL, EmbeddingServiceImpl, GeminiEmbeddingProvider, Logger, MIN_EMBEDDING_TEXT_LENGTH, OpenAIEmbeddingProvider, logger, logger as logger$1 } from "@nexo-labs/payload-indexer";
|
|
2
|
+
import Typesense, { Client } from "typesense";
|
|
2
3
|
import OpenAI from "openai";
|
|
3
4
|
import { GoogleGenerativeAI, TaskType } from "@google/generative-ai";
|
|
4
5
|
import { z } from "zod";
|
|
5
|
-
import { MarkdownTextSplitter, RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
|
|
6
|
-
import { convertLexicalToMarkdown, editorConfigFactory } from "@payloadcms/richtext-lexical";
|
|
7
6
|
|
|
8
7
|
//#region src/core/client/typesense-client.ts
|
|
9
8
|
const createTypesenseClient = (typesenseConfig) => {
|
|
@@ -22,204 +21,6 @@ const testTypesenseConnection = async (client) => {
|
|
|
22
21
|
}
|
|
23
22
|
};
|
|
24
23
|
|
|
25
|
-
//#endregion
|
|
26
|
-
//#region src/core/logging/logger.ts
|
|
27
|
-
const LOG_LEVELS = {
|
|
28
|
-
debug: 0,
|
|
29
|
-
info: 1,
|
|
30
|
-
warn: 2,
|
|
31
|
-
error: 3,
|
|
32
|
-
silent: 4
|
|
33
|
-
};
|
|
34
|
-
var Logger = class {
|
|
35
|
-
level;
|
|
36
|
-
prefix;
|
|
37
|
-
enabled;
|
|
38
|
-
constructor(config = {}) {
|
|
39
|
-
this.level = config.level || "info";
|
|
40
|
-
this.prefix = config.prefix || "[payload-typesense]";
|
|
41
|
-
this.enabled = config.enabled !== false;
|
|
42
|
-
}
|
|
43
|
-
/**
|
|
44
|
-
* Update logger configuration
|
|
45
|
-
*/
|
|
46
|
-
configure(config) {
|
|
47
|
-
if (config.level !== void 0) this.level = config.level;
|
|
48
|
-
if (config.prefix !== void 0) this.prefix = config.prefix;
|
|
49
|
-
if (config.enabled !== void 0) this.enabled = config.enabled;
|
|
50
|
-
}
|
|
51
|
-
/**
|
|
52
|
-
* Check if a log level should be output
|
|
53
|
-
*/
|
|
54
|
-
shouldLog(level) {
|
|
55
|
-
if (!this.enabled) return false;
|
|
56
|
-
return LOG_LEVELS[level] >= LOG_LEVELS[this.level];
|
|
57
|
-
}
|
|
58
|
-
/**
|
|
59
|
-
* Format log message with context
|
|
60
|
-
*/
|
|
61
|
-
formatMessage(message, context) {
|
|
62
|
-
if (!context || Object.keys(context).length === 0) return `${this.prefix} ${message}`;
|
|
63
|
-
return `${this.prefix} ${message} ${JSON.stringify(context)}`;
|
|
64
|
-
}
|
|
65
|
-
/**
|
|
66
|
-
* Debug level logging - detailed information for debugging
|
|
67
|
-
*/
|
|
68
|
-
debug(message, context) {
|
|
69
|
-
if (this.shouldLog("debug")) console.debug(this.formatMessage(message, context));
|
|
70
|
-
}
|
|
71
|
-
/**
|
|
72
|
-
* Info level logging - general informational messages
|
|
73
|
-
*/
|
|
74
|
-
info(message, context) {
|
|
75
|
-
if (this.shouldLog("info")) console.log(this.formatMessage(message, context));
|
|
76
|
-
}
|
|
77
|
-
/**
|
|
78
|
-
* Warning level logging - warning messages
|
|
79
|
-
*/
|
|
80
|
-
warn(message, context) {
|
|
81
|
-
if (this.shouldLog("warn")) console.warn(this.formatMessage(message, context));
|
|
82
|
-
}
|
|
83
|
-
/**
|
|
84
|
-
* Error level logging - error messages
|
|
85
|
-
*/
|
|
86
|
-
error(message, error, context) {
|
|
87
|
-
if (this.shouldLog("error")) {
|
|
88
|
-
const errorContext = {
|
|
89
|
-
...context,
|
|
90
|
-
error: error instanceof Error ? {
|
|
91
|
-
message: error.message,
|
|
92
|
-
stack: error.stack,
|
|
93
|
-
name: error.name
|
|
94
|
-
} : String(error)
|
|
95
|
-
};
|
|
96
|
-
console.error(this.formatMessage(message, errorContext));
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
/**
|
|
100
|
-
* Get current log level
|
|
101
|
-
*/
|
|
102
|
-
getLevel() {
|
|
103
|
-
return this.level;
|
|
104
|
-
}
|
|
105
|
-
/**
|
|
106
|
-
* Check if logger is enabled
|
|
107
|
-
*/
|
|
108
|
-
isEnabled() {
|
|
109
|
-
return this.enabled;
|
|
110
|
-
}
|
|
111
|
-
};
|
|
112
|
-
let defaultLogger = new Logger();
|
|
113
|
-
/**
|
|
114
|
-
* Configure the default logger
|
|
115
|
-
*/
|
|
116
|
-
const configureLogger = (config) => {
|
|
117
|
-
defaultLogger.configure(config);
|
|
118
|
-
};
|
|
119
|
-
/**
|
|
120
|
-
* Create a new logger instance with custom configuration
|
|
121
|
-
*/
|
|
122
|
-
const createLogger = (config) => {
|
|
123
|
-
return new Logger(config);
|
|
124
|
-
};
|
|
125
|
-
const logger = {
|
|
126
|
-
debug: (message, context) => defaultLogger.debug(message, context),
|
|
127
|
-
info: (message, context) => defaultLogger.info(message, context),
|
|
128
|
-
warn: (message, context) => defaultLogger.warn(message, context),
|
|
129
|
-
error: (message, error, context) => defaultLogger.error(message, error, context),
|
|
130
|
-
configure: configureLogger,
|
|
131
|
-
getLevel: () => defaultLogger.getLevel(),
|
|
132
|
-
isEnabled: () => defaultLogger.isEnabled()
|
|
133
|
-
};
|
|
134
|
-
|
|
135
|
-
//#endregion
|
|
136
|
-
//#region src/core/config/constants.ts
|
|
137
|
-
/**
|
|
138
|
-
* Constants for payload-typesense plugin
|
|
139
|
-
* Centralizes all magic numbers and configuration defaults
|
|
140
|
-
*/
|
|
141
|
-
/**
|
|
142
|
-
* Default dimensions for OpenAI text-embedding-3-large model
|
|
143
|
-
*/
|
|
144
|
-
const DEFAULT_EMBEDDING_DIMENSIONS = 3072;
|
|
145
|
-
/**
|
|
146
|
-
* Default OpenAI embedding model
|
|
147
|
-
*/
|
|
148
|
-
const DEFAULT_EMBEDDING_MODEL = "text-embedding-3-large";
|
|
149
|
-
/**
|
|
150
|
-
* Default Gemini embedding model
|
|
151
|
-
*/
|
|
152
|
-
const DEFAULT_GEMINI_EMBEDDING_MODEL = "gemini-embedding-001";
|
|
153
|
-
/**
|
|
154
|
-
* Default chunk size for text splitting (in characters)
|
|
155
|
-
*/
|
|
156
|
-
const DEFAULT_CHUNK_SIZE = 1e3;
|
|
157
|
-
/**
|
|
158
|
-
* Default overlap for text splitting (in characters)
|
|
159
|
-
*/
|
|
160
|
-
const DEFAULT_OVERLAP = 200;
|
|
161
|
-
/**
|
|
162
|
-
* Default overlap between chunks (in characters)
|
|
163
|
-
*/
|
|
164
|
-
const DEFAULT_CHUNK_OVERLAP = 200;
|
|
165
|
-
/**
|
|
166
|
-
* Default alpha value for hybrid search (0 = pure semantic, 1 = pure keyword)
|
|
167
|
-
*/
|
|
168
|
-
const DEFAULT_HYBRID_SEARCH_ALPHA = .5;
|
|
169
|
-
/**
|
|
170
|
-
* Default number of search results to return
|
|
171
|
-
*/
|
|
172
|
-
const DEFAULT_SEARCH_LIMIT = 10;
|
|
173
|
-
/**
|
|
174
|
-
* Default TTL for cache entries (in milliseconds) - 5 minutes
|
|
175
|
-
*/
|
|
176
|
-
const DEFAULT_CACHE_TTL_MS = 300 * 1e3;
|
|
177
|
-
/**
|
|
178
|
-
* Default maximum tokens for RAG responses
|
|
179
|
-
*/
|
|
180
|
-
const DEFAULT_RAG_MAX_TOKENS = 1e3;
|
|
181
|
-
/**
|
|
182
|
-
* Default number of search results to use for RAG context
|
|
183
|
-
*/
|
|
184
|
-
const DEFAULT_RAG_CONTEXT_LIMIT = 5;
|
|
185
|
-
/**
|
|
186
|
-
* Default session TTL (in seconds) - 30 minutes
|
|
187
|
-
*/
|
|
188
|
-
const DEFAULT_SESSION_TTL_SEC = 1800;
|
|
189
|
-
/**
|
|
190
|
-
* Default OpenAI model for RAG chat
|
|
191
|
-
*/
|
|
192
|
-
const DEFAULT_RAG_LLM_MODEL = "gpt-4o-mini";
|
|
193
|
-
/**
|
|
194
|
-
* Minimum required text length for embedding generation
|
|
195
|
-
*/
|
|
196
|
-
const MIN_EMBEDDING_TEXT_LENGTH = 1;
|
|
197
|
-
/**
|
|
198
|
-
* Error codes for structured error handling
|
|
199
|
-
*/
|
|
200
|
-
const ErrorCodes = {
|
|
201
|
-
INVALID_CONFIG: "ERR_1001",
|
|
202
|
-
MISSING_API_KEY: "ERR_1002",
|
|
203
|
-
INVALID_EMBEDDING_CONFIG: "ERR_1003",
|
|
204
|
-
INVALID_RAG_CONFIG: "ERR_1004",
|
|
205
|
-
TYPESENSE_CONNECTION_FAILED: "ERR_2001",
|
|
206
|
-
TYPESENSE_COLLECTION_NOT_FOUND: "ERR_2002",
|
|
207
|
-
TYPESENSE_SEARCH_FAILED: "ERR_2003",
|
|
208
|
-
TYPESENSE_SYNC_FAILED: "ERR_2004",
|
|
209
|
-
TYPESENSE_DELETE_FAILED: "ERR_2005",
|
|
210
|
-
EMBEDDING_GENERATION_FAILED: "ERR_3001",
|
|
211
|
-
INVALID_EMBEDDING_DIMENSIONS: "ERR_3002",
|
|
212
|
-
OPENAI_API_ERROR: "ERR_3003",
|
|
213
|
-
RAG_SEARCH_FAILED: "ERR_4001",
|
|
214
|
-
RAG_SESSION_NOT_FOUND: "ERR_4002",
|
|
215
|
-
RAG_CONVERSATION_FAILED: "ERR_4003",
|
|
216
|
-
RAG_TOKEN_LIMIT_EXCEEDED: "ERR_4004",
|
|
217
|
-
CHUNKING_FAILED: "ERR_5001",
|
|
218
|
-
INVALID_CHUNK_SIZE: "ERR_5002",
|
|
219
|
-
UNKNOWN_ERROR: "ERR_9001",
|
|
220
|
-
VALIDATION_ERROR: "ERR_9002"
|
|
221
|
-
};
|
|
222
|
-
|
|
223
24
|
//#endregion
|
|
224
25
|
//#region src/features/embedding/embeddings.ts
|
|
225
26
|
let openaiClient = null;
|
|
@@ -647,7 +448,7 @@ function parseConversationEvent(line) {
|
|
|
647
448
|
if (parsed.results) event.results = parsed.results;
|
|
648
449
|
return event;
|
|
649
450
|
} catch (e) {
|
|
650
|
-
logger.error("Error parsing SSE data from conversation stream", e);
|
|
451
|
+
logger$1.error("Error parsing SSE data from conversation stream", e);
|
|
651
452
|
return null;
|
|
652
453
|
}
|
|
653
454
|
}
|
|
@@ -798,11 +599,11 @@ function getDefaultDocumentType(collectionName) {
|
|
|
798
599
|
async function ensureConversationCollection(client, collectionName = "conversation_history") {
|
|
799
600
|
try {
|
|
800
601
|
await client.collections(collectionName).retrieve();
|
|
801
|
-
logger.info("Conversation collection already exists", { collection: collectionName });
|
|
602
|
+
logger$1.info("Conversation collection already exists", { collection: collectionName });
|
|
802
603
|
return true;
|
|
803
604
|
} catch (error) {
|
|
804
605
|
if (error?.httpStatus === 404) {
|
|
805
|
-
logger.info("Creating conversation collection", { collection: collectionName });
|
|
606
|
+
logger$1.info("Creating conversation collection", { collection: collectionName });
|
|
806
607
|
try {
|
|
807
608
|
await client.collections().create({
|
|
808
609
|
name: collectionName,
|
|
@@ -829,14 +630,14 @@ async function ensureConversationCollection(client, collectionName = "conversati
|
|
|
829
630
|
}
|
|
830
631
|
]
|
|
831
632
|
});
|
|
832
|
-
logger.info("Conversation collection created successfully", { collection: collectionName });
|
|
633
|
+
logger$1.info("Conversation collection created successfully", { collection: collectionName });
|
|
833
634
|
return true;
|
|
834
635
|
} catch (createError) {
|
|
835
|
-
logger.error("Failed to create conversation collection", createError, { collection: collectionName });
|
|
636
|
+
logger$1.error("Failed to create conversation collection", createError, { collection: collectionName });
|
|
836
637
|
return false;
|
|
837
638
|
}
|
|
838
639
|
}
|
|
839
|
-
logger.error("Error checking conversation collection", error, { collection: collectionName });
|
|
640
|
+
logger$1.error("Error checking conversation collection", error, { collection: collectionName });
|
|
840
641
|
return false;
|
|
841
642
|
}
|
|
842
643
|
}
|
|
@@ -954,11 +755,11 @@ async function fetchChunkById(client, config) {
|
|
|
954
755
|
if (validCollections && !validCollections.includes(collectionName)) throw new Error(`Invalid collection: ${collectionName}. Must be one of: ${validCollections.join(", ")}`);
|
|
955
756
|
try {
|
|
956
757
|
const document = await client.collections(collectionName).documents(chunkId).retrieve();
|
|
957
|
-
const chunkText
|
|
958
|
-
if (!chunkText
|
|
758
|
+
const chunkText = document.chunk_text || "";
|
|
759
|
+
if (!chunkText) throw new Error("Chunk contains no text");
|
|
959
760
|
return {
|
|
960
761
|
id: document.id,
|
|
961
|
-
chunk_text: chunkText
|
|
762
|
+
chunk_text: chunkText,
|
|
962
763
|
title: document.title,
|
|
963
764
|
slug: document.slug,
|
|
964
765
|
chunk_index: document.chunk_index,
|
|
@@ -1117,7 +918,7 @@ async function saveChatSession(payload, userId, conversationId, userMessage, ass
|
|
|
1117
918
|
if (existing.docs.length > 0 && existing.docs[0]) await updateExistingSession(payload, existing.docs[0], newUserMessage, newAssistantMessage, spending, collectionName);
|
|
1118
919
|
else await createNewSession(payload, userId, conversationId, newUserMessage, newAssistantMessage, spending, collectionName);
|
|
1119
920
|
} catch (error) {
|
|
1120
|
-
logger.error("Error saving chat session", error, {
|
|
921
|
+
logger$1.error("Error saving chat session", error, {
|
|
1121
922
|
conversationId,
|
|
1122
923
|
userId
|
|
1123
924
|
});
|
|
@@ -1149,7 +950,7 @@ async function updateExistingSession(payload, session, newUserMessage, newAssist
|
|
|
1149
950
|
status: "active"
|
|
1150
951
|
}
|
|
1151
952
|
});
|
|
1152
|
-
logger.info("Chat session updated successfully", {
|
|
953
|
+
logger$1.info("Chat session updated successfully", {
|
|
1153
954
|
sessionId: session.id,
|
|
1154
955
|
conversationId: session.conversation_id,
|
|
1155
956
|
totalTokens,
|
|
@@ -1175,7 +976,7 @@ async function createNewSession(payload, userId, conversationId, newUserMessage,
|
|
|
1175
976
|
last_activity: (/* @__PURE__ */ new Date()).toISOString()
|
|
1176
977
|
}
|
|
1177
978
|
});
|
|
1178
|
-
logger.info("New chat session created successfully", {
|
|
979
|
+
logger$1.info("New chat session created successfully", {
|
|
1179
980
|
conversationId,
|
|
1180
981
|
userId,
|
|
1181
982
|
totalTokens,
|
|
@@ -1228,159 +1029,6 @@ async function validateChatRequest(request, config) {
|
|
|
1228
1029
|
};
|
|
1229
1030
|
}
|
|
1230
1031
|
|
|
1231
|
-
//#endregion
|
|
1232
|
-
//#region src/features/embedding/services/embedding-service.ts
|
|
1233
|
-
var EmbeddingServiceImpl = class {
|
|
1234
|
-
constructor(provider, logger$1, config) {
|
|
1235
|
-
this.provider = provider;
|
|
1236
|
-
this.logger = logger$1;
|
|
1237
|
-
this.config = config;
|
|
1238
|
-
}
|
|
1239
|
-
async getEmbedding(text) {
|
|
1240
|
-
const result = await this.provider.generateEmbedding(text);
|
|
1241
|
-
if (!result) return null;
|
|
1242
|
-
return result.embedding;
|
|
1243
|
-
}
|
|
1244
|
-
async getEmbeddingsBatch(texts) {
|
|
1245
|
-
const result = await this.provider.generateBatchEmbeddings(texts);
|
|
1246
|
-
if (!result) return null;
|
|
1247
|
-
return result.embeddings;
|
|
1248
|
-
}
|
|
1249
|
-
getDimensions() {
|
|
1250
|
-
return this.config.dimensions || DEFAULT_EMBEDDING_DIMENSIONS;
|
|
1251
|
-
}
|
|
1252
|
-
};
|
|
1253
|
-
|
|
1254
|
-
//#endregion
|
|
1255
|
-
//#region src/features/embedding/providers/openai-provider.ts
|
|
1256
|
-
var OpenAIEmbeddingProvider = class {
|
|
1257
|
-
client;
|
|
1258
|
-
model;
|
|
1259
|
-
dimensions;
|
|
1260
|
-
constructor(config, logger$1) {
|
|
1261
|
-
this.logger = logger$1;
|
|
1262
|
-
if (!config.apiKey) throw new Error("OpenAI API key is required");
|
|
1263
|
-
this.client = new OpenAI({ apiKey: config.apiKey });
|
|
1264
|
-
this.model = config.model || DEFAULT_EMBEDDING_MODEL;
|
|
1265
|
-
this.dimensions = config.dimensions || DEFAULT_EMBEDDING_DIMENSIONS;
|
|
1266
|
-
}
|
|
1267
|
-
async generateEmbedding(text) {
|
|
1268
|
-
if (!text || text.trim().length < MIN_EMBEDDING_TEXT_LENGTH) return null;
|
|
1269
|
-
try {
|
|
1270
|
-
const response = await this.client.embeddings.create({
|
|
1271
|
-
model: this.model,
|
|
1272
|
-
input: text.trim(),
|
|
1273
|
-
dimensions: this.dimensions
|
|
1274
|
-
});
|
|
1275
|
-
const embedding = response.data[0]?.embedding;
|
|
1276
|
-
if (!embedding) return null;
|
|
1277
|
-
return {
|
|
1278
|
-
embedding,
|
|
1279
|
-
usage: {
|
|
1280
|
-
promptTokens: response.usage?.prompt_tokens || 0,
|
|
1281
|
-
totalTokens: response.usage?.total_tokens || 0
|
|
1282
|
-
}
|
|
1283
|
-
};
|
|
1284
|
-
} catch (error) {
|
|
1285
|
-
this.logger.error("OpenAI embedding generation failed", error, { model: this.model });
|
|
1286
|
-
return null;
|
|
1287
|
-
}
|
|
1288
|
-
}
|
|
1289
|
-
async generateBatchEmbeddings(texts) {
|
|
1290
|
-
const validTexts = texts.filter((t) => t && t.trim().length >= MIN_EMBEDDING_TEXT_LENGTH);
|
|
1291
|
-
if (validTexts.length === 0) return null;
|
|
1292
|
-
try {
|
|
1293
|
-
const response = await this.client.embeddings.create({
|
|
1294
|
-
model: this.model,
|
|
1295
|
-
input: validTexts.map((t) => t.trim()),
|
|
1296
|
-
dimensions: this.dimensions
|
|
1297
|
-
});
|
|
1298
|
-
return {
|
|
1299
|
-
embeddings: response.data.map((d) => d.embedding),
|
|
1300
|
-
usage: {
|
|
1301
|
-
promptTokens: response.usage?.prompt_tokens || 0,
|
|
1302
|
-
totalTokens: response.usage?.total_tokens || 0
|
|
1303
|
-
}
|
|
1304
|
-
};
|
|
1305
|
-
} catch (error) {
|
|
1306
|
-
this.logger.error("OpenAI batch embedding generation failed", error, {
|
|
1307
|
-
model: this.model,
|
|
1308
|
-
count: texts.length
|
|
1309
|
-
});
|
|
1310
|
-
return null;
|
|
1311
|
-
}
|
|
1312
|
-
}
|
|
1313
|
-
};
|
|
1314
|
-
|
|
1315
|
-
//#endregion
|
|
1316
|
-
//#region src/features/embedding/providers/gemini-provider.ts
|
|
1317
|
-
var GeminiEmbeddingProvider = class {
|
|
1318
|
-
client;
|
|
1319
|
-
model;
|
|
1320
|
-
constructor(config, logger$1) {
|
|
1321
|
-
this.logger = logger$1;
|
|
1322
|
-
if (!config.apiKey) throw new Error("Gemini API key is required");
|
|
1323
|
-
this.client = new GoogleGenerativeAI(config.apiKey);
|
|
1324
|
-
this.model = config.model || DEFAULT_GEMINI_EMBEDDING_MODEL;
|
|
1325
|
-
}
|
|
1326
|
-
async generateEmbedding(text) {
|
|
1327
|
-
if (!text || text.trim().length < MIN_EMBEDDING_TEXT_LENGTH) return null;
|
|
1328
|
-
try {
|
|
1329
|
-
const embedding = (await this.client.getGenerativeModel({ model: this.model }).embedContent({
|
|
1330
|
-
content: {
|
|
1331
|
-
role: "user",
|
|
1332
|
-
parts: [{ text: text.trim() }]
|
|
1333
|
-
},
|
|
1334
|
-
taskType: TaskType.RETRIEVAL_DOCUMENT
|
|
1335
|
-
})).embedding.values;
|
|
1336
|
-
const estimatedTokens = Math.ceil(text.length / 4);
|
|
1337
|
-
return {
|
|
1338
|
-
embedding,
|
|
1339
|
-
usage: {
|
|
1340
|
-
promptTokens: estimatedTokens,
|
|
1341
|
-
totalTokens: estimatedTokens
|
|
1342
|
-
}
|
|
1343
|
-
};
|
|
1344
|
-
} catch (error) {
|
|
1345
|
-
this.logger.error("Gemini embedding generation failed", error, { model: this.model });
|
|
1346
|
-
return null;
|
|
1347
|
-
}
|
|
1348
|
-
}
|
|
1349
|
-
async generateBatchEmbeddings(texts) {
|
|
1350
|
-
const validTexts = texts.filter((t) => t && t.trim().length >= MIN_EMBEDDING_TEXT_LENGTH);
|
|
1351
|
-
if (validTexts.length === 0) return null;
|
|
1352
|
-
try {
|
|
1353
|
-
const model = this.client.getGenerativeModel({ model: this.model });
|
|
1354
|
-
const embeddings = [];
|
|
1355
|
-
let totalTokens = 0;
|
|
1356
|
-
for (const text of validTexts) {
|
|
1357
|
-
const result = await model.embedContent({
|
|
1358
|
-
content: {
|
|
1359
|
-
role: "user",
|
|
1360
|
-
parts: [{ text: text.trim() }]
|
|
1361
|
-
},
|
|
1362
|
-
taskType: TaskType.RETRIEVAL_DOCUMENT
|
|
1363
|
-
});
|
|
1364
|
-
embeddings.push(result.embedding.values);
|
|
1365
|
-
totalTokens += Math.ceil(text.length / 4);
|
|
1366
|
-
}
|
|
1367
|
-
return {
|
|
1368
|
-
embeddings,
|
|
1369
|
-
usage: {
|
|
1370
|
-
promptTokens: totalTokens,
|
|
1371
|
-
totalTokens
|
|
1372
|
-
}
|
|
1373
|
-
};
|
|
1374
|
-
} catch (error) {
|
|
1375
|
-
this.logger.error("Gemini batch embedding generation failed", error, {
|
|
1376
|
-
model: this.model,
|
|
1377
|
-
count: texts.length
|
|
1378
|
-
});
|
|
1379
|
-
return null;
|
|
1380
|
-
}
|
|
1381
|
-
}
|
|
1382
|
-
};
|
|
1383
|
-
|
|
1384
1032
|
//#endregion
|
|
1385
1033
|
//#region src/features/rag/endpoints/chat/handlers/embedding-handler.ts
|
|
1386
1034
|
/**
|
|
@@ -1435,7 +1083,7 @@ async function generateEmbeddingWithTracking(userMessage, config, spendingEntrie
|
|
|
1435
1083
|
async function saveChatSessionIfNeeded(config, payload, userId, conversationId, userMessage, assistantMessage, sources, spendingEntries) {
|
|
1436
1084
|
if (!conversationId || !config.saveChatSession) return;
|
|
1437
1085
|
await config.saveChatSession(payload, userId, conversationId, userMessage, assistantMessage, sources, spendingEntries, config.collectionName);
|
|
1438
|
-
logger.info("Chat session saved to PayloadCMS", { conversationId });
|
|
1086
|
+
logger$1.info("Chat session saved to PayloadCMS", { conversationId });
|
|
1439
1087
|
}
|
|
1440
1088
|
|
|
1441
1089
|
//#endregion
|
|
@@ -1448,7 +1096,7 @@ async function checkTokenLimitsIfNeeded(config, payload, userId, userEmail, user
|
|
|
1448
1096
|
const estimatedTotalTokens = config.estimateTokensFromText(userMessage) + config.estimateTokensFromText(userMessage) * 10;
|
|
1449
1097
|
const limitCheck = await config.checkTokenLimit(payload, userId, estimatedTotalTokens);
|
|
1450
1098
|
if (!limitCheck.allowed) {
|
|
1451
|
-
logger.warn("Token limit exceeded for user", {
|
|
1099
|
+
logger$1.warn("Token limit exceeded for user", {
|
|
1452
1100
|
userId,
|
|
1453
1101
|
limit: limitCheck.limit,
|
|
1454
1102
|
used: limitCheck.used,
|
|
@@ -1464,7 +1112,7 @@ async function checkTokenLimitsIfNeeded(config, payload, userId, userEmail, user
|
|
|
1464
1112
|
}
|
|
1465
1113
|
}, { status: 429 });
|
|
1466
1114
|
}
|
|
1467
|
-
logger.info("Chat request started with token limit check passed", {
|
|
1115
|
+
logger$1.info("Chat request started with token limit check passed", {
|
|
1468
1116
|
userId,
|
|
1469
1117
|
userEmail,
|
|
1470
1118
|
limit: limitCheck.limit,
|
|
@@ -1482,7 +1130,7 @@ async function checkTokenLimitsIfNeeded(config, payload, userId, userEmail, user
|
|
|
1482
1130
|
function calculateTotalUsage(spendingEntries) {
|
|
1483
1131
|
const totalTokensUsed = spendingEntries.reduce((sum, entry) => sum + entry.tokens.total, 0);
|
|
1484
1132
|
const totalCostUSD = spendingEntries.reduce((sum, entry) => sum + (entry.cost_usd || 0), 0);
|
|
1485
|
-
logger.info("Total token usage calculated", {
|
|
1133
|
+
logger$1.info("Total token usage calculated", {
|
|
1486
1134
|
totalTokens: totalTokensUsed,
|
|
1487
1135
|
totalCostUsd: totalCostUSD
|
|
1488
1136
|
});
|
|
@@ -1544,7 +1192,7 @@ function createChatPOSTHandler(config) {
|
|
|
1544
1192
|
} else return new Response(JSON.stringify({ error: "No RAG configuration available" }), { status: 500 });
|
|
1545
1193
|
const tokenLimitError = await checkTokenLimitsIfNeeded(config, payload, userId, userEmail, userMessage);
|
|
1546
1194
|
if (tokenLimitError) return tokenLimitError;
|
|
1547
|
-
logger.info("Processing chat message", {
|
|
1195
|
+
logger$1.info("Processing chat message", {
|
|
1548
1196
|
userId,
|
|
1549
1197
|
chatId: body.chatId || "new",
|
|
1550
1198
|
agentSlug: agentSlug || "default",
|
|
@@ -1576,14 +1224,14 @@ function createChatPOSTHandler(config) {
|
|
|
1576
1224
|
const { totalTokens: totalTokensUsed, totalCostUSD } = calculateTotalUsage(spendingEntries);
|
|
1577
1225
|
await sendUsageStatsIfNeeded(config, payload, userId, totalTokensUsed, totalCostUSD, sendEvent);
|
|
1578
1226
|
await saveChatSessionIfNeeded(config, payload, userId, conversationIdCapture, userMessage, fullAssistantMessage, sourcesCapture, spendingEntries);
|
|
1579
|
-
logger.info("Chat request completed successfully", {
|
|
1227
|
+
logger$1.info("Chat request completed successfully", {
|
|
1580
1228
|
userId,
|
|
1581
1229
|
conversationId: conversationIdCapture,
|
|
1582
1230
|
totalTokens: totalTokensUsed
|
|
1583
1231
|
});
|
|
1584
1232
|
controller.close();
|
|
1585
1233
|
} catch (error) {
|
|
1586
|
-
logger.error("Fatal error in chat stream", error, {
|
|
1234
|
+
logger$1.error("Fatal error in chat stream", error, {
|
|
1587
1235
|
userId,
|
|
1588
1236
|
chatId: body.chatId
|
|
1589
1237
|
});
|
|
@@ -1600,7 +1248,7 @@ function createChatPOSTHandler(config) {
|
|
|
1600
1248
|
Connection: "keep-alive"
|
|
1601
1249
|
} });
|
|
1602
1250
|
} catch (error) {
|
|
1603
|
-
logger.error("Error in chat API endpoint", error, { userId: request.user?.id });
|
|
1251
|
+
logger$1.error("Error in chat API endpoint", error, { userId: request.user?.id });
|
|
1604
1252
|
return new Response(JSON.stringify({
|
|
1605
1253
|
error: "Error al procesar tu mensaje. Por favor, inténtalo de nuevo.",
|
|
1606
1254
|
details: error instanceof Error ? error.message : "Error desconocido"
|
|
@@ -1647,7 +1295,7 @@ function estimateTokensFromText(text) {
|
|
|
1647
1295
|
* Default implementation for handling streaming responses
|
|
1648
1296
|
*/
|
|
1649
1297
|
async function defaultHandleStreamingResponse(response, controller, encoder) {
|
|
1650
|
-
logger.debug("Starting streaming response handling");
|
|
1298
|
+
logger$1.debug("Starting streaming response handling");
|
|
1651
1299
|
if (!response.body) throw new Error("Response body is null");
|
|
1652
1300
|
const reader = response.body.getReader();
|
|
1653
1301
|
const decoder = new TextDecoder();
|
|
@@ -1661,7 +1309,7 @@ async function defaultHandleStreamingResponse(response, controller, encoder) {
|
|
|
1661
1309
|
while (true) {
|
|
1662
1310
|
const { done, value } = await reader.read();
|
|
1663
1311
|
if (done) {
|
|
1664
|
-
logger.debug("Streaming response completed");
|
|
1312
|
+
logger$1.debug("Streaming response completed");
|
|
1665
1313
|
break;
|
|
1666
1314
|
}
|
|
1667
1315
|
buffer += decoder.decode(value, { stream: true });
|
|
@@ -1679,7 +1327,7 @@ async function defaultHandleStreamingResponse(response, controller, encoder) {
|
|
|
1679
1327
|
}
|
|
1680
1328
|
if (!conversationId && event.conversationId) {
|
|
1681
1329
|
conversationId = event.conversationId;
|
|
1682
|
-
logger.debug("Conversation ID captured", { conversationId });
|
|
1330
|
+
logger$1.debug("Conversation ID captured", { conversationId });
|
|
1683
1331
|
sendSSEEvent(controller, encoder, {
|
|
1684
1332
|
type: "conversation_id",
|
|
1685
1333
|
data: conversationId
|
|
@@ -1719,7 +1367,7 @@ async function defaultHandleStreamingResponse(response, controller, encoder) {
|
|
|
1719
1367
|
cost_usd: llmInputTokens * 15e-8 + llmOutputTokens * 6e-7,
|
|
1720
1368
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1721
1369
|
};
|
|
1722
|
-
logger.info("LLM cost calculated", {
|
|
1370
|
+
logger$1.info("LLM cost calculated", {
|
|
1723
1371
|
inputTokens: llmInputTokens,
|
|
1724
1372
|
outputTokens: llmOutputTokens,
|
|
1725
1373
|
totalTokens: llmSpending.tokens.total,
|
|
@@ -1744,7 +1392,7 @@ async function defaultHandleStreamingResponse(response, controller, encoder) {
|
|
|
1744
1392
|
* Default implementation for handling non-streaming responses
|
|
1745
1393
|
*/
|
|
1746
1394
|
async function defaultHandleNonStreamingResponse(data, controller, encoder) {
|
|
1747
|
-
logger.debug("Using non-streaming fallback for response handling");
|
|
1395
|
+
logger$1.debug("Using non-streaming fallback for response handling");
|
|
1748
1396
|
const typedData = data;
|
|
1749
1397
|
let conversationId = null;
|
|
1750
1398
|
if (typedData.conversation?.conversation_id) conversationId = typedData.conversation.conversation_id;
|
|
@@ -1826,7 +1474,7 @@ function createSessionGETHandler(config) {
|
|
|
1826
1474
|
if (!session) return jsonResponse({ error: "Sesión de chat no encontrada." }, { status: 404 });
|
|
1827
1475
|
return jsonResponse(session);
|
|
1828
1476
|
} catch (error) {
|
|
1829
|
-
logger.error("Error retrieving chat session", error, { userId: request.user?.id });
|
|
1477
|
+
logger$1.error("Error retrieving chat session", error, { userId: request.user?.id });
|
|
1830
1478
|
return jsonResponse({
|
|
1831
1479
|
error: "Error al recuperar la sesión.",
|
|
1832
1480
|
details: error instanceof Error ? error.message : "Error desconocido"
|
|
@@ -1850,13 +1498,13 @@ function createSessionDELETEHandler(config) {
|
|
|
1850
1498
|
const conversationId = searchParams.get("conversationId");
|
|
1851
1499
|
if (!conversationId) return jsonResponse({ error: "Se requiere un conversationId válido." }, { status: 400 });
|
|
1852
1500
|
const payload = await config.getPayload();
|
|
1853
|
-
logger.info("Closing chat session", {
|
|
1501
|
+
logger$1.info("Closing chat session", {
|
|
1854
1502
|
conversationId,
|
|
1855
1503
|
userId
|
|
1856
1504
|
});
|
|
1857
1505
|
const session = await closeSession(payload, userId, conversationId, config.sessionConfig);
|
|
1858
1506
|
if (!session) return jsonResponse({ error: "Sesión de chat no encontrada o no tienes permisos." }, { status: 404 });
|
|
1859
|
-
logger.info("Chat session closed successfully", {
|
|
1507
|
+
logger$1.info("Chat session closed successfully", {
|
|
1860
1508
|
conversationId,
|
|
1861
1509
|
totalTokens: session.total_tokens,
|
|
1862
1510
|
totalCost: session.total_cost
|
|
@@ -1872,7 +1520,7 @@ function createSessionDELETEHandler(config) {
|
|
|
1872
1520
|
}
|
|
1873
1521
|
});
|
|
1874
1522
|
} catch (error) {
|
|
1875
|
-
logger.error("Error closing chat session", error, {
|
|
1523
|
+
logger$1.error("Error closing chat session", error, {
|
|
1876
1524
|
conversationId: request.url ? new URL(request.url).searchParams.get("conversationId") : void 0,
|
|
1877
1525
|
userId: request.user?.id
|
|
1878
1526
|
});
|
|
@@ -1910,7 +1558,7 @@ function createChunksGETHandler(config) {
|
|
|
1910
1558
|
validCollections: config.validCollections
|
|
1911
1559
|
}));
|
|
1912
1560
|
} catch (error) {
|
|
1913
|
-
logger.error("Error fetching chunk", error, {
|
|
1561
|
+
logger$1.error("Error fetching chunk", error, {
|
|
1914
1562
|
chunkId: request.routeParams?.id,
|
|
1915
1563
|
collection: request.url ? new URL(request.url).searchParams.get("collection") : void 0
|
|
1916
1564
|
});
|
|
@@ -1948,55 +1596,64 @@ function createAgentsGETHandler(config) {
|
|
|
1948
1596
|
//#region src/features/rag/endpoints.ts
|
|
1949
1597
|
/**
|
|
1950
1598
|
* Creates Payload handlers for RAG endpoints
|
|
1599
|
+
*
|
|
1600
|
+
* @param config - RAG plugin configuration (composable, doesn't depend on ModularPluginConfig)
|
|
1951
1601
|
*/
|
|
1952
|
-
function createRAGPayloadHandlers(
|
|
1602
|
+
function createRAGPayloadHandlers(config) {
|
|
1953
1603
|
const endpoints = [];
|
|
1954
|
-
if (!
|
|
1955
|
-
const
|
|
1956
|
-
const
|
|
1957
|
-
const agentCollections = ragConfig.agents?.flatMap((agent) => agent.searchCollections) || [];
|
|
1604
|
+
if (!config.agents || config.agents.length === 0) return endpoints;
|
|
1605
|
+
const { agents, callbacks, typesense } = config;
|
|
1606
|
+
const agentCollections = agents.flatMap((agent) => agent.searchCollections) || [];
|
|
1958
1607
|
const validCollections = Array.from(new Set(agentCollections));
|
|
1608
|
+
const ragFeatureConfig = {
|
|
1609
|
+
enabled: true,
|
|
1610
|
+
agents,
|
|
1611
|
+
callbacks,
|
|
1612
|
+
hybrid: config.hybrid,
|
|
1613
|
+
hnsw: config.hnsw,
|
|
1614
|
+
advanced: config.advanced
|
|
1615
|
+
};
|
|
1959
1616
|
endpoints.push({
|
|
1960
1617
|
path: "/chat",
|
|
1961
1618
|
method: "post",
|
|
1962
1619
|
handler: createChatPOSTHandler({
|
|
1963
1620
|
collectionName: "chat-sessions",
|
|
1964
|
-
checkPermissions:
|
|
1965
|
-
typesense
|
|
1966
|
-
rag:
|
|
1967
|
-
getPayload:
|
|
1968
|
-
checkTokenLimit:
|
|
1969
|
-
getUserUsageStats:
|
|
1970
|
-
saveChatSession:
|
|
1621
|
+
checkPermissions: callbacks.checkPermissions,
|
|
1622
|
+
typesense,
|
|
1623
|
+
rag: ragFeatureConfig,
|
|
1624
|
+
getPayload: callbacks.getPayload,
|
|
1625
|
+
checkTokenLimit: callbacks.checkTokenLimit,
|
|
1626
|
+
getUserUsageStats: callbacks.getUserUsageStats,
|
|
1627
|
+
saveChatSession: callbacks.saveChatSession,
|
|
1971
1628
|
handleStreamingResponse: defaultHandleStreamingResponse,
|
|
1972
1629
|
handleNonStreamingResponse: defaultHandleNonStreamingResponse,
|
|
1973
|
-
createEmbeddingSpending:
|
|
1974
|
-
estimateTokensFromText:
|
|
1975
|
-
embeddingConfig:
|
|
1630
|
+
createEmbeddingSpending: callbacks.createEmbeddingSpending,
|
|
1631
|
+
estimateTokensFromText: callbacks.estimateTokensFromText,
|
|
1632
|
+
embeddingConfig: config.embeddingConfig
|
|
1976
1633
|
})
|
|
1977
1634
|
});
|
|
1978
1635
|
endpoints.push({
|
|
1979
1636
|
path: "/chat/session",
|
|
1980
1637
|
method: "get",
|
|
1981
1638
|
handler: createSessionGETHandler({
|
|
1982
|
-
getPayload:
|
|
1983
|
-
checkPermissions:
|
|
1639
|
+
getPayload: callbacks.getPayload,
|
|
1640
|
+
checkPermissions: callbacks.checkPermissions
|
|
1984
1641
|
})
|
|
1985
1642
|
});
|
|
1986
1643
|
endpoints.push({
|
|
1987
1644
|
path: "/chat/session",
|
|
1988
1645
|
method: "delete",
|
|
1989
1646
|
handler: createSessionDELETEHandler({
|
|
1990
|
-
getPayload:
|
|
1991
|
-
checkPermissions:
|
|
1647
|
+
getPayload: callbacks.getPayload,
|
|
1648
|
+
checkPermissions: callbacks.checkPermissions
|
|
1992
1649
|
})
|
|
1993
1650
|
});
|
|
1994
1651
|
endpoints.push({
|
|
1995
1652
|
path: "/chat/chunks/:id",
|
|
1996
1653
|
method: "get",
|
|
1997
1654
|
handler: createChunksGETHandler({
|
|
1998
|
-
typesense
|
|
1999
|
-
checkPermissions:
|
|
1655
|
+
typesense,
|
|
1656
|
+
checkPermissions: callbacks.checkPermissions,
|
|
2000
1657
|
validCollections
|
|
2001
1658
|
})
|
|
2002
1659
|
});
|
|
@@ -2004,8 +1661,8 @@ function createRAGPayloadHandlers(pluginOptions) {
|
|
|
2004
1661
|
path: "/chat/agents",
|
|
2005
1662
|
method: "get",
|
|
2006
1663
|
handler: createAgentsGETHandler({
|
|
2007
|
-
ragConfig,
|
|
2008
|
-
checkPermissions:
|
|
1664
|
+
ragConfig: ragFeatureConfig,
|
|
1665
|
+
checkPermissions: callbacks.checkPermissions
|
|
2009
1666
|
})
|
|
2010
1667
|
});
|
|
2011
1668
|
return endpoints;
|
|
@@ -2045,23 +1702,6 @@ const createCollectionsHandler = (pluginOptions) => {
|
|
|
2045
1702
|
};
|
|
2046
1703
|
};
|
|
2047
1704
|
|
|
2048
|
-
//#endregion
|
|
2049
|
-
//#region src/core/utils/naming.ts
|
|
2050
|
-
/**
|
|
2051
|
-
* Generates the Typesense collection name based on the configuration.
|
|
2052
|
-
*
|
|
2053
|
-
* Priority:
|
|
2054
|
-
* 1. Explicit `tableName` if provided.
|
|
2055
|
-
* 2. `collectionSlug` (fallback).
|
|
2056
|
-
*
|
|
2057
|
-
* @param collectionSlug The slug of the Payload collection
|
|
2058
|
-
* @param tableConfig The configuration for the specific table
|
|
2059
|
-
* @returns The generated Typesense collection name
|
|
2060
|
-
*/
|
|
2061
|
-
const getTypesenseCollectionName = (collectionSlug, tableConfig) => {
|
|
2062
|
-
return tableConfig.tableName ?? collectionSlug;
|
|
2063
|
-
};
|
|
2064
|
-
|
|
2065
1705
|
//#endregion
|
|
2066
1706
|
//#region src/shared/cache/cache.ts
|
|
2067
1707
|
var SearchCache = class {
|
|
@@ -2289,7 +1929,7 @@ const searchTraditionalCollection = async (typesenseClient, collectionName, conf
|
|
|
2289
1929
|
//#endregion
|
|
2290
1930
|
//#region src/features/search/endpoints/handlers/executors/traditional-multi-collection-search.ts
|
|
2291
1931
|
const performTraditionalMultiCollectionSearch = async (typesenseClient, enabledCollections, query, options) => {
|
|
2292
|
-
logger.info("Performing traditional multi-collection search", {
|
|
1932
|
+
logger$1.info("Performing traditional multi-collection search", {
|
|
2293
1933
|
query,
|
|
2294
1934
|
collections: enabledCollections.map(([name]) => name)
|
|
2295
1935
|
});
|
|
@@ -2311,7 +1951,7 @@ const performTraditionalMultiCollectionSearch = async (typesenseClient, enabledC
|
|
|
2311
1951
|
...options.exclude_fields && { exclude_fields: options.exclude_fields }
|
|
2312
1952
|
});
|
|
2313
1953
|
} catch (error) {
|
|
2314
|
-
logger.error("Error searching collection", error, {
|
|
1954
|
+
logger$1.error("Error searching collection", error, {
|
|
2315
1955
|
collection: collectionName,
|
|
2316
1956
|
query
|
|
2317
1957
|
});
|
|
@@ -2479,7 +2119,7 @@ var SearchService = class {
|
|
|
2479
2119
|
searchCache.set(query, results, cacheKey, options);
|
|
2480
2120
|
return results;
|
|
2481
2121
|
} catch (error) {
|
|
2482
|
-
logger.error("Vector search failed, falling back to traditional", error);
|
|
2122
|
+
logger$1.error("Vector search failed, falling back to traditional", error);
|
|
2483
2123
|
return this.performTraditionalSearch(query, targetCollections, options);
|
|
2484
2124
|
}
|
|
2485
2125
|
}
|
|
@@ -2531,21 +2171,38 @@ function resolveDocumentType(collectionName) {
|
|
|
2531
2171
|
* Transform search response to simplified format
|
|
2532
2172
|
*/
|
|
2533
2173
|
function transformToSimpleFormat(data) {
|
|
2534
|
-
if (!data
|
|
2174
|
+
if (!data.hits) return { documents: [] };
|
|
2535
2175
|
return { documents: data.hits.map((hit) => {
|
|
2536
2176
|
const doc = hit.document || {};
|
|
2537
2177
|
const collectionValue = hit.collection || doc.collection;
|
|
2538
2178
|
const collection = typeof collectionValue === "string" ? collectionValue : "";
|
|
2539
2179
|
return {
|
|
2540
|
-
id: doc.id || "",
|
|
2541
|
-
title: doc.title || "Sin título",
|
|
2542
|
-
slug: doc.slug || "",
|
|
2180
|
+
id: String(doc.id || ""),
|
|
2181
|
+
title: String(doc.title || "Sin título"),
|
|
2182
|
+
slug: String(doc.slug || ""),
|
|
2543
2183
|
type: resolveDocumentType(collection),
|
|
2544
2184
|
collection
|
|
2545
2185
|
};
|
|
2546
2186
|
}) };
|
|
2547
2187
|
}
|
|
2548
2188
|
|
|
2189
|
+
//#endregion
|
|
2190
|
+
//#region src/core/utils/naming.ts
|
|
2191
|
+
/**
|
|
2192
|
+
* Generates the Typesense collection name based on the configuration.
|
|
2193
|
+
*
|
|
2194
|
+
* Priority:
|
|
2195
|
+
* 1. Explicit `tableName` if provided.
|
|
2196
|
+
* 2. `collectionSlug` (fallback).
|
|
2197
|
+
*
|
|
2198
|
+
* @param collectionSlug The slug of the Payload collection
|
|
2199
|
+
* @param tableConfig The configuration for the specific table
|
|
2200
|
+
* @returns The generated Typesense collection name
|
|
2201
|
+
*/
|
|
2202
|
+
const getTypesenseCollectionName = (collectionSlug, tableConfig) => {
|
|
2203
|
+
return tableConfig.tableName ?? collectionSlug;
|
|
2204
|
+
};
|
|
2205
|
+
|
|
2549
2206
|
//#endregion
|
|
2550
2207
|
//#region src/features/search/endpoints/handlers/utils/target-resolver.ts
|
|
2551
2208
|
var TargetCollectionResolver = class {
|
|
@@ -2768,12 +2425,6 @@ function validateSearchRequest(request) {
|
|
|
2768
2425
|
//#endregion
|
|
2769
2426
|
//#region src/features/search/endpoints/handlers/search-handler.ts
|
|
2770
2427
|
/**
|
|
2771
|
-
* Helper type guard to check if a result is a valid search response
|
|
2772
|
-
*/
|
|
2773
|
-
function isValidSearchResponse(result) {
|
|
2774
|
-
return typeof result === "object" && result !== null && "hits" in result && Array.isArray(result.hits);
|
|
2775
|
-
}
|
|
2776
|
-
/**
|
|
2777
2428
|
* Creates a handler for standard search requests
|
|
2778
2429
|
*/
|
|
2779
2430
|
const createSearchHandler = (typesenseClient, pluginOptions) => {
|
|
@@ -2801,8 +2452,7 @@ const createSearchHandler = (typesenseClient, pluginOptions) => {
|
|
|
2801
2452
|
exclude_fields: searchParams.exclude_fields,
|
|
2802
2453
|
query_by: searchParams.query_by
|
|
2803
2454
|
});
|
|
2804
|
-
if (
|
|
2805
|
-
if (searchParams.simple && isValidSearchResponse(searchResult)) return Response.json(transformToSimpleFormat(searchResult));
|
|
2455
|
+
if (searchParams.simple) return Response.json(transformToSimpleFormat(searchResult));
|
|
2806
2456
|
return Response.json(searchResult);
|
|
2807
2457
|
} catch (error) {
|
|
2808
2458
|
return Response.json({
|
|
@@ -2836,479 +2486,43 @@ const createSearchEndpoints = (typesenseClient, pluginOptions) => {
|
|
|
2836
2486
|
};
|
|
2837
2487
|
|
|
2838
2488
|
//#endregion
|
|
2839
|
-
//#region src/
|
|
2489
|
+
//#region src/core/config/constants.ts
|
|
2840
2490
|
/**
|
|
2841
|
-
*
|
|
2491
|
+
* Constants for payload-typesense plugin
|
|
2492
|
+
* Centralizes all magic numbers and configuration defaults
|
|
2842
2493
|
*/
|
|
2843
|
-
const getValueByPath = (obj, path) => {
|
|
2844
|
-
if (!obj || typeof obj !== "object") return void 0;
|
|
2845
|
-
return path.split(".").reduce((acc, part) => {
|
|
2846
|
-
if (acc && typeof acc === "object" && part in acc) return acc[part];
|
|
2847
|
-
}, obj);
|
|
2848
|
-
};
|
|
2849
|
-
/**
|
|
2850
|
-
* Maps a Payload document to a Typesense document based on field configuration
|
|
2851
|
-
*/
|
|
2852
|
-
const mapPayloadDocumentToTypesense = async (doc, fields) => {
|
|
2853
|
-
const result = {};
|
|
2854
|
-
for (const field of fields) {
|
|
2855
|
-
let value = getValueByPath(doc, field.payloadField || field.name);
|
|
2856
|
-
if (field.transform) value = await field.transform(value);
|
|
2857
|
-
else {
|
|
2858
|
-
if (value === void 0 || value === null) {
|
|
2859
|
-
if (field.optional) continue;
|
|
2860
|
-
if (field.type === "string") value = "";
|
|
2861
|
-
else if (field.type === "string[]") value = [];
|
|
2862
|
-
else if (field.type === "bool") value = false;
|
|
2863
|
-
else if (field.type.startsWith("int") || field.type === "float") value = 0;
|
|
2864
|
-
}
|
|
2865
|
-
if (field.type === "string" && typeof value !== "string") if (typeof value === "object" && value !== null) value = JSON.stringify(value);
|
|
2866
|
-
else value = String(value);
|
|
2867
|
-
else if (field.type === "string[]" && !Array.isArray(value)) value = [String(value)];
|
|
2868
|
-
else if (field.type === "bool") value = Boolean(value);
|
|
2869
|
-
}
|
|
2870
|
-
result[field.name] = value;
|
|
2871
|
-
}
|
|
2872
|
-
return result;
|
|
2873
|
-
};
|
|
2874
|
-
|
|
2875
|
-
//#endregion
|
|
2876
|
-
//#region src/features/embedding/chunking/strategies/markdown-based/markdown-chunker.ts
|
|
2877
2494
|
/**
|
|
2878
|
-
*
|
|
2879
|
-
* Splits markdown text respecting markdown structure and preserves header metadata
|
|
2495
|
+
* Default dimensions for OpenAI text-embedding-3-large model
|
|
2880
2496
|
*/
|
|
2497
|
+
const DEFAULT_EMBEDDING_DIMENSIONS$1 = 3072;
|
|
2881
2498
|
/**
|
|
2882
|
-
*
|
|
2499
|
+
* Default alpha value for hybrid search (0 = pure semantic, 1 = pure keyword)
|
|
2883
2500
|
*/
|
|
2884
|
-
const
|
|
2885
|
-
const headerRegex = /^(#{1,6})\s+(.+)$/gm;
|
|
2886
|
-
const headers = [];
|
|
2887
|
-
let match;
|
|
2888
|
-
while ((match = headerRegex.exec(text)) !== null) headers.push({
|
|
2889
|
-
level: match[1]?.length ?? 0,
|
|
2890
|
-
text: match[2]?.trim() ?? "",
|
|
2891
|
-
position: match.index
|
|
2892
|
-
});
|
|
2893
|
-
return headers;
|
|
2894
|
-
};
|
|
2895
|
-
/**
|
|
2896
|
-
* Finds the headers that apply to a given chunk based on its content
|
|
2897
|
-
*/
|
|
2898
|
-
const findChunkHeaders = (chunkText$1, allHeaders, fullText) => {
|
|
2899
|
-
const chunkPosition = fullText.indexOf(chunkText$1.substring(0, Math.min(50, chunkText$1.length)));
|
|
2900
|
-
if (chunkPosition === -1) return {};
|
|
2901
|
-
const applicableHeaders = allHeaders.filter((h) => h.position <= chunkPosition);
|
|
2902
|
-
if (applicableHeaders.length === 0) return {};
|
|
2903
|
-
const metadata = {};
|
|
2904
|
-
const currentHierarchy = Array(6).fill(null);
|
|
2905
|
-
for (const header of applicableHeaders) {
|
|
2906
|
-
currentHierarchy[header.level - 1] = header;
|
|
2907
|
-
for (let i = header.level; i < 6; i++) currentHierarchy[i] = null;
|
|
2908
|
-
}
|
|
2909
|
-
for (let i = 0; i < 6; i++) if (currentHierarchy[i]) metadata[`Header ${i + 1}`] = currentHierarchy[i].text;
|
|
2910
|
-
return metadata;
|
|
2911
|
-
};
|
|
2912
|
-
/**
|
|
2913
|
-
* Chunks markdown text using LangChain's MarkdownTextSplitter
|
|
2914
|
-
* Respects markdown structure and extracts header metadata for each chunk
|
|
2915
|
-
*/
|
|
2916
|
-
const chunkMarkdown = async (text, options = {}) => {
|
|
2917
|
-
const { maxChunkSize = DEFAULT_CHUNK_SIZE, overlap = DEFAULT_OVERLAP } = options;
|
|
2918
|
-
if (!text || text.trim().length === 0) return [];
|
|
2919
|
-
const headers = extractHeaders(text);
|
|
2920
|
-
return (await new MarkdownTextSplitter({
|
|
2921
|
-
chunkSize: maxChunkSize,
|
|
2922
|
-
chunkOverlap: overlap
|
|
2923
|
-
}).createDocuments([text])).map((chunk, index) => {
|
|
2924
|
-
const metadata = findChunkHeaders(chunk.pageContent, headers, text);
|
|
2925
|
-
return {
|
|
2926
|
-
text: chunk.pageContent,
|
|
2927
|
-
index,
|
|
2928
|
-
startIndex: 0,
|
|
2929
|
-
endIndex: chunk.pageContent.length,
|
|
2930
|
-
metadata: Object.keys(metadata).length > 0 ? metadata : void 0
|
|
2931
|
-
};
|
|
2932
|
-
});
|
|
2933
|
-
};
|
|
2934
|
-
|
|
2935
|
-
//#endregion
|
|
2936
|
-
//#region src/features/embedding/chunking/index.ts
|
|
2937
|
-
/**
|
|
2938
|
-
* Text chunking module - provides utilities for splitting text into optimal chunks
|
|
2939
|
-
*
|
|
2940
|
-
* Available strategies:
|
|
2941
|
-
* - Simple: Uses LangChain's RecursiveCharacterTextSplitter
|
|
2942
|
-
* - Markdown-based: Uses LangChain's MarkdownTextSplitter for markdown documents
|
|
2943
|
-
*
|
|
2944
|
-
* Future strategies can be added in ./strategies/
|
|
2945
|
-
*/
|
|
2946
|
-
/**
|
|
2947
|
-
* Splits text into chunks using LangChain's RecursiveCharacterTextSplitter
|
|
2948
|
-
* Main entry point for simple text chunking
|
|
2949
|
-
*/
|
|
2950
|
-
const chunkText = async (text, options = {}) => {
|
|
2951
|
-
const { maxChunkSize = DEFAULT_CHUNK_SIZE, overlap = DEFAULT_OVERLAP } = options;
|
|
2952
|
-
if (!text || text.trim().length === 0) return [];
|
|
2953
|
-
if (text.length <= maxChunkSize) return [{
|
|
2954
|
-
text: text.trim(),
|
|
2955
|
-
index: 0,
|
|
2956
|
-
startIndex: 0,
|
|
2957
|
-
endIndex: text.length
|
|
2958
|
-
}];
|
|
2959
|
-
return (await new RecursiveCharacterTextSplitter({
|
|
2960
|
-
chunkSize: maxChunkSize,
|
|
2961
|
-
chunkOverlap: overlap
|
|
2962
|
-
}).createDocuments([text])).map((chunk, index) => ({
|
|
2963
|
-
text: chunk.pageContent,
|
|
2964
|
-
index,
|
|
2965
|
-
startIndex: 0,
|
|
2966
|
-
endIndex: chunk.pageContent.length
|
|
2967
|
-
}));
|
|
2968
|
-
};
|
|
2969
|
-
|
|
2970
|
-
//#endregion
|
|
2971
|
-
//#region src/core/utils/header-utils.ts
|
|
2972
|
-
/**
|
|
2973
|
-
* Builds a hierarchical path array from markdown header metadata.
|
|
2974
|
-
*
|
|
2975
|
-
* @param metadata - The metadata object from LangChain's MarkdownHeaderTextSplitter
|
|
2976
|
-
* @returns An array of header paths showing the hierarchy
|
|
2977
|
-
*
|
|
2978
|
-
* @example
|
|
2979
|
-
* // Input: { 'Header 1': 'Introduction', 'Header 2': 'Getting Started', 'Header 3': 'Installation' }
|
|
2980
|
-
* // Output: ['Introduction', 'Introduction > Getting Started', 'Introduction > Getting Started > Installation']
|
|
2981
|
-
*/
|
|
2982
|
-
const buildHeaderHierarchy = (metadata) => {
|
|
2983
|
-
if (!metadata || Object.keys(metadata).length === 0) return [];
|
|
2984
|
-
const headers = [];
|
|
2985
|
-
const headerLevels = Object.keys(metadata).filter((key) => key.startsWith("Header ")).sort((a, b) => {
|
|
2986
|
-
return parseInt(a.replace("Header ", "")) - parseInt(b.replace("Header ", ""));
|
|
2987
|
-
});
|
|
2988
|
-
let currentPath = [];
|
|
2989
|
-
for (const headerKey of headerLevels) {
|
|
2990
|
-
const headerValue = metadata[headerKey];
|
|
2991
|
-
if (!headerValue) continue;
|
|
2992
|
-
currentPath.push(headerValue);
|
|
2993
|
-
headers.push(currentPath.join(" > "));
|
|
2994
|
-
}
|
|
2995
|
-
return headers;
|
|
2996
|
-
};
|
|
2997
|
-
|
|
2998
|
-
//#endregion
|
|
2999
|
-
//#region src/core/utils/chunk-format-utils.ts
|
|
2501
|
+
const DEFAULT_HYBRID_SEARCH_ALPHA = .5;
|
|
3000
2502
|
/**
|
|
3001
|
-
*
|
|
2503
|
+
* Default number of search results to return
|
|
3002
2504
|
*/
|
|
2505
|
+
const DEFAULT_SEARCH_LIMIT = 10;
|
|
3003
2506
|
/**
|
|
3004
|
-
*
|
|
2507
|
+
* Default TTL for cache entries (in milliseconds) - 5 minutes
|
|
3005
2508
|
*/
|
|
3006
|
-
const
|
|
3007
|
-
/**
|
|
3008
|
-
* Formats chunk text with header metadata at the end
|
|
3009
|
-
*
|
|
3010
|
-
* @param content - The chunk content
|
|
3011
|
-
* @param headers - Hierarchical array of headers (e.g., ['Introduction', 'Introduction > Getting Started'])
|
|
3012
|
-
* @returns Formatted chunk text with content + separator + key-value metadata
|
|
3013
|
-
*
|
|
3014
|
-
* @example
|
|
3015
|
-
* const formatted = formatChunkWithHeaders(
|
|
3016
|
-
* 'To install the package...',
|
|
3017
|
-
* ['Introduction', 'Introduction > Getting Started', 'Introduction > Getting Started > Installation']
|
|
3018
|
-
* );
|
|
3019
|
-
* // Result:
|
|
3020
|
-
* // To install the package...
|
|
3021
|
-
* // ._________________________________________.
|
|
3022
|
-
* // section: Installation | path: Introduction > Getting Started > Installation
|
|
3023
|
-
*/
|
|
3024
|
-
const formatChunkWithHeaders = (content, headers) => {
|
|
3025
|
-
if (!headers || headers.length === 0) return content;
|
|
3026
|
-
const fullPath = headers[headers.length - 1];
|
|
3027
|
-
return `${content}\n${CHUNK_HEADER_SEPARATOR}\n${`section: ${fullPath && fullPath.split(" > ").pop() || fullPath || ""} | path: ${fullPath}`}`;
|
|
3028
|
-
};
|
|
3029
|
-
/**
|
|
3030
|
-
* Parses chunk text to extract header metadata and content separately
|
|
3031
|
-
*
|
|
3032
|
-
* @param chunkText - The formatted chunk text
|
|
3033
|
-
* @returns Object with separated metadata and content
|
|
3034
|
-
*
|
|
3035
|
-
* @example
|
|
3036
|
-
* const parsed = parseChunkText('Content here\\n._________________________________________.\\nsection: Installation | path: Introduction > Getting Started > Installation');
|
|
3037
|
-
* console.log(parsed.metadata.section); // "Installation"
|
|
3038
|
-
* console.log(parsed.content); // "Content here"
|
|
3039
|
-
*/
|
|
3040
|
-
const parseChunkText = (chunkText$1) => {
|
|
3041
|
-
if (!chunkText$1.includes(CHUNK_HEADER_SEPARATOR)) return { content: chunkText$1 };
|
|
3042
|
-
const [contentPart, ...metadataParts] = chunkText$1.split(CHUNK_HEADER_SEPARATOR);
|
|
3043
|
-
const content = contentPart ? contentPart.trim() : "";
|
|
3044
|
-
const metadataLine = metadataParts.join(CHUNK_HEADER_SEPARATOR).trim();
|
|
3045
|
-
try {
|
|
3046
|
-
const pairs = metadataLine.split(" | ");
|
|
3047
|
-
const metadata = {
|
|
3048
|
-
section: "",
|
|
3049
|
-
path: ""
|
|
3050
|
-
};
|
|
3051
|
-
for (const pair of pairs) {
|
|
3052
|
-
const [key, ...valueParts] = pair.split(": ");
|
|
3053
|
-
const value = valueParts.join(": ").trim();
|
|
3054
|
-
if (key?.trim() === "section") metadata.section = value;
|
|
3055
|
-
else if (key?.trim() === "path") metadata.path = value;
|
|
3056
|
-
}
|
|
3057
|
-
if (metadata.section || metadata.path) return {
|
|
3058
|
-
metadata,
|
|
3059
|
-
content
|
|
3060
|
-
};
|
|
3061
|
-
return { content: chunkText$1 };
|
|
3062
|
-
} catch (error) {
|
|
3063
|
-
return { content: chunkText$1 };
|
|
3064
|
-
}
|
|
3065
|
-
};
|
|
2509
|
+
const DEFAULT_CACHE_TTL_MS = 300 * 1e3;
|
|
3066
2510
|
/**
|
|
3067
|
-
*
|
|
3068
|
-
*
|
|
3069
|
-
* @param chunkText - The formatted chunk text
|
|
3070
|
-
* @returns Just the content without header metadata
|
|
2511
|
+
* Default maximum tokens for RAG responses
|
|
3071
2512
|
*/
|
|
3072
|
-
const
|
|
3073
|
-
return parseChunkText(chunkText$1).content;
|
|
3074
|
-
};
|
|
2513
|
+
const DEFAULT_RAG_MAX_TOKENS = 1e3;
|
|
3075
2514
|
/**
|
|
3076
|
-
*
|
|
3077
|
-
*
|
|
3078
|
-
* @param chunkText - The formatted chunk text
|
|
3079
|
-
* @returns Header metadata or undefined if not present
|
|
2515
|
+
* Default number of search results to use for RAG context
|
|
3080
2516
|
*/
|
|
3081
|
-
const
|
|
3082
|
-
return parseChunkText(chunkText$1).metadata;
|
|
3083
|
-
};
|
|
3084
|
-
|
|
3085
|
-
//#endregion
|
|
3086
|
-
//#region src/features/sync/services/document-syncer.ts
|
|
2517
|
+
const DEFAULT_RAG_CONTEXT_LIMIT = 5;
|
|
3087
2518
|
/**
|
|
3088
|
-
*
|
|
3089
|
-
* Uses Strategy pattern to handle both chunked and full document approaches
|
|
2519
|
+
* Default session TTL (in seconds) - 30 minutes
|
|
3090
2520
|
*/
|
|
3091
|
-
const
|
|
3092
|
-
try {
|
|
3093
|
-
const tableName = tableConfig.tableName || getTypesenseCollectionName(collectionSlug, tableConfig);
|
|
3094
|
-
logger.debug("Syncing document to Typesense", {
|
|
3095
|
-
documentId: doc.id,
|
|
3096
|
-
collection: collectionSlug,
|
|
3097
|
-
tableName,
|
|
3098
|
-
operation
|
|
3099
|
-
});
|
|
3100
|
-
await new DocumentSyncer(typesenseClient, collectionSlug, tableName, tableConfig, embeddingService).sync(doc, operation);
|
|
3101
|
-
logger.info("Document synced successfully to Typesense", {
|
|
3102
|
-
documentId: doc.id,
|
|
3103
|
-
collection: collectionSlug,
|
|
3104
|
-
operation
|
|
3105
|
-
});
|
|
3106
|
-
} catch (error) {
|
|
3107
|
-
const isValidationError = (error instanceof Error ? error.message : String(error)).toLowerCase().includes("validation");
|
|
3108
|
-
logger.error(`Failed to sync document to Typesense`, error, {
|
|
3109
|
-
documentId: doc.id,
|
|
3110
|
-
collection: collectionSlug,
|
|
3111
|
-
operation,
|
|
3112
|
-
isValidationError
|
|
3113
|
-
});
|
|
3114
|
-
}
|
|
3115
|
-
};
|
|
3116
|
-
var DocumentSyncer = class {
|
|
3117
|
-
constructor(client, collectionSlug, tableName, config, embeddingService) {
|
|
3118
|
-
this.client = client;
|
|
3119
|
-
this.collectionSlug = collectionSlug;
|
|
3120
|
-
this.tableName = tableName;
|
|
3121
|
-
this.config = config;
|
|
3122
|
-
this.embeddingService = embeddingService;
|
|
3123
|
-
}
|
|
3124
|
-
async sync(doc, operation) {
|
|
3125
|
-
logger.debug(`Syncing document ${doc.id} to table ${this.tableName}`);
|
|
3126
|
-
if (this.config.embedding?.chunking) await this.syncChunked(doc, operation);
|
|
3127
|
-
else await this.syncDocument(doc, operation);
|
|
3128
|
-
}
|
|
3129
|
-
async syncDocument(doc, operation) {
|
|
3130
|
-
const typesenseDoc = await mapPayloadDocumentToTypesense(doc, this.config.fields);
|
|
3131
|
-
typesenseDoc.id = String(doc.id);
|
|
3132
|
-
typesenseDoc.slug = doc.slug || "";
|
|
3133
|
-
typesenseDoc.createdAt = new Date(doc.createdAt).getTime();
|
|
3134
|
-
typesenseDoc.updatedAt = new Date(doc.updatedAt).getTime();
|
|
3135
|
-
if (doc.publishedAt) typesenseDoc.publishedAt = new Date(doc.publishedAt).getTime();
|
|
3136
|
-
if (this.config.embedding?.fields && this.embeddingService) {
|
|
3137
|
-
const sourceText = await this.extractSourceText(doc);
|
|
3138
|
-
if (sourceText) {
|
|
3139
|
-
const embedding = await this.embeddingService.getEmbedding(sourceText);
|
|
3140
|
-
if (embedding) typesenseDoc.embedding = embedding;
|
|
3141
|
-
}
|
|
3142
|
-
}
|
|
3143
|
-
await this.client.collections(this.tableName).documents().upsert(typesenseDoc);
|
|
3144
|
-
logger.info(`Synced document ${doc.id} to ${this.tableName}`);
|
|
3145
|
-
}
|
|
3146
|
-
async syncChunked(doc, operation) {
|
|
3147
|
-
const sourceText = await this.extractSourceText(doc);
|
|
3148
|
-
if (!sourceText) {
|
|
3149
|
-
logger.warn(`No source text found for document ${doc.id}`);
|
|
3150
|
-
return;
|
|
3151
|
-
}
|
|
3152
|
-
const chunks = await this.generateChunks(sourceText);
|
|
3153
|
-
const fields = this.config.fields ? await mapPayloadDocumentToTypesense(doc, this.config.fields) : {};
|
|
3154
|
-
fields.slug = doc.slug || "";
|
|
3155
|
-
fields.publishedAt = doc.publishedAt ? new Date(doc.publishedAt).getTime() : void 0;
|
|
3156
|
-
if (operation === "update") await this.client.collections(this.tableName).documents().delete({ filter_by: `parent_doc_id:${doc.id}` });
|
|
3157
|
-
for (const chunk of chunks) {
|
|
3158
|
-
const headers = buildHeaderHierarchy(chunk.metadata);
|
|
3159
|
-
let formattedText = formatChunkWithHeaders(chunk.text, headers);
|
|
3160
|
-
if (this.config.embedding?.chunking?.interceptResult) formattedText = this.config.embedding.chunking.interceptResult({
|
|
3161
|
-
...chunk,
|
|
3162
|
-
headers,
|
|
3163
|
-
formattedText
|
|
3164
|
-
}, doc);
|
|
3165
|
-
let embedding = [];
|
|
3166
|
-
if (this.embeddingService) {
|
|
3167
|
-
const result = await this.embeddingService.getEmbedding(formattedText);
|
|
3168
|
-
if (result) embedding = result;
|
|
3169
|
-
}
|
|
3170
|
-
const chunkDoc = {
|
|
3171
|
-
id: `${doc.id}_chunk_${chunk.index}`,
|
|
3172
|
-
parent_doc_id: String(doc.id),
|
|
3173
|
-
chunk_index: chunk.index,
|
|
3174
|
-
chunk_text: formattedText,
|
|
3175
|
-
is_chunk: true,
|
|
3176
|
-
headers,
|
|
3177
|
-
embedding,
|
|
3178
|
-
createdAt: new Date(doc.createdAt).getTime(),
|
|
3179
|
-
updatedAt: new Date(doc.updatedAt).getTime(),
|
|
3180
|
-
...fields
|
|
3181
|
-
};
|
|
3182
|
-
await this.client.collections(this.tableName).documents().upsert(chunkDoc);
|
|
3183
|
-
}
|
|
3184
|
-
logger.info(`Synced ${chunks.length} chunks for document ${doc.id} to ${this.tableName}`);
|
|
3185
|
-
}
|
|
3186
|
-
/**
|
|
3187
|
-
* Extract and transform source fields for embedding generation
|
|
3188
|
-
*/
|
|
3189
|
-
async extractSourceText(doc) {
|
|
3190
|
-
if (!this.config.embedding?.fields) return "";
|
|
3191
|
-
const textParts = [];
|
|
3192
|
-
for (const sourceField of this.config.embedding.fields) {
|
|
3193
|
-
let fieldName;
|
|
3194
|
-
let transform;
|
|
3195
|
-
if (typeof sourceField === "string") fieldName = sourceField;
|
|
3196
|
-
else {
|
|
3197
|
-
fieldName = sourceField.field;
|
|
3198
|
-
transform = sourceField.transform;
|
|
3199
|
-
}
|
|
3200
|
-
let val = doc[fieldName];
|
|
3201
|
-
if (transform) val = await transform(val);
|
|
3202
|
-
else if (typeof val === "object" && val !== null && "root" in val) val = JSON.stringify(val);
|
|
3203
|
-
textParts.push(String(val || ""));
|
|
3204
|
-
}
|
|
3205
|
-
return textParts.join("\n\n");
|
|
3206
|
-
}
|
|
3207
|
-
async generateChunks(text) {
|
|
3208
|
-
if (!this.config.embedding?.chunking) return [];
|
|
3209
|
-
const { strategy, size, overlap } = this.config.embedding.chunking;
|
|
3210
|
-
const options = {
|
|
3211
|
-
maxChunkSize: size,
|
|
3212
|
-
overlap
|
|
3213
|
-
};
|
|
3214
|
-
if (strategy === "markdown") return await chunkMarkdown(text, options);
|
|
3215
|
-
else return await chunkText(text, options);
|
|
3216
|
-
}
|
|
3217
|
-
};
|
|
3218
|
-
|
|
3219
|
-
//#endregion
|
|
3220
|
-
//#region src/features/sync/services/document-delete.ts
|
|
2521
|
+
const DEFAULT_SESSION_TTL_SEC = 1800;
|
|
3221
2522
|
/**
|
|
3222
|
-
*
|
|
3223
|
-
* Handles both direct document deletion and chunk deletion
|
|
2523
|
+
* Default OpenAI model for RAG chat
|
|
3224
2524
|
*/
|
|
3225
|
-
const
|
|
3226
|
-
try {
|
|
3227
|
-
const tableName = getTypesenseCollectionName(collectionSlug, tableConfig);
|
|
3228
|
-
logger.debug("Attempting to delete document from Typesense", {
|
|
3229
|
-
documentId: docId,
|
|
3230
|
-
collection: collectionSlug,
|
|
3231
|
-
tableName
|
|
3232
|
-
});
|
|
3233
|
-
try {
|
|
3234
|
-
await typesenseClient.collections(tableName).documents(docId).delete();
|
|
3235
|
-
logger.info("Document deleted from Typesense", {
|
|
3236
|
-
documentId: docId,
|
|
3237
|
-
tableName
|
|
3238
|
-
});
|
|
3239
|
-
} catch (docDeleteError) {
|
|
3240
|
-
if (docDeleteError.httpStatus === 404) {
|
|
3241
|
-
logger.debug("Document not found, attempting to delete chunks", {
|
|
3242
|
-
documentId: docId,
|
|
3243
|
-
tableName
|
|
3244
|
-
});
|
|
3245
|
-
try {
|
|
3246
|
-
await typesenseClient.collections(tableName).documents().delete({ filter_by: `parent_doc_id:${docId}` });
|
|
3247
|
-
logger.info("All chunks deleted for document", {
|
|
3248
|
-
documentId: docId,
|
|
3249
|
-
tableName
|
|
3250
|
-
});
|
|
3251
|
-
} catch (chunkDeleteError) {
|
|
3252
|
-
if (chunkDeleteError.httpStatus !== 404) logger.error("Failed to delete chunks for document", chunkDeleteError, {
|
|
3253
|
-
documentId: docId,
|
|
3254
|
-
tableName
|
|
3255
|
-
});
|
|
3256
|
-
else logger.debug("No chunks found to delete", { documentId: docId });
|
|
3257
|
-
}
|
|
3258
|
-
} else throw docDeleteError;
|
|
3259
|
-
}
|
|
3260
|
-
} catch (error) {
|
|
3261
|
-
const tableName = getTypesenseCollectionName(collectionSlug, tableConfig);
|
|
3262
|
-
logger.error("Failed to delete document from Typesense", error, {
|
|
3263
|
-
documentId: docId,
|
|
3264
|
-
collection: collectionSlug,
|
|
3265
|
-
tableName
|
|
3266
|
-
});
|
|
3267
|
-
}
|
|
3268
|
-
};
|
|
3269
|
-
|
|
3270
|
-
//#endregion
|
|
3271
|
-
//#region src/features/sync/hooks.ts
|
|
3272
|
-
/**
|
|
3273
|
-
* Applies sync hooks to Payload collections
|
|
3274
|
-
*/
|
|
3275
|
-
const applySyncHooks = (config, pluginOptions, typesenseClient, embeddingService) => {
|
|
3276
|
-
if (!pluginOptions.features.sync?.enabled || pluginOptions.features.sync.autoSync === false || !pluginOptions.collections) return config;
|
|
3277
|
-
return (config || []).map((collection) => {
|
|
3278
|
-
const tableConfigs = pluginOptions.collections?.[collection.slug];
|
|
3279
|
-
if (tableConfigs && Array.isArray(tableConfigs) && tableConfigs.some((tableConfig) => tableConfig.enabled)) {
|
|
3280
|
-
logger.debug("Registering sync hooks for collection", {
|
|
3281
|
-
collection: collection.slug,
|
|
3282
|
-
tableCount: tableConfigs?.length || 0
|
|
3283
|
-
});
|
|
3284
|
-
return {
|
|
3285
|
-
...collection,
|
|
3286
|
-
hooks: {
|
|
3287
|
-
...collection.hooks,
|
|
3288
|
-
afterChange: [...collection.hooks?.afterChange || [], async ({ doc, operation, req: _req }) => {
|
|
3289
|
-
if (tableConfigs && Array.isArray(tableConfigs)) {
|
|
3290
|
-
for (const tableConfig of tableConfigs) if (tableConfig.enabled) {
|
|
3291
|
-
if (tableConfig.shouldIndex) {
|
|
3292
|
-
if (!await tableConfig.shouldIndex(doc)) {
|
|
3293
|
-
await deleteDocumentFromTypesense(typesenseClient, collection.slug, doc.id, tableConfig);
|
|
3294
|
-
continue;
|
|
3295
|
-
}
|
|
3296
|
-
}
|
|
3297
|
-
await syncDocumentToTypesense(typesenseClient, collection.slug, doc, operation, tableConfig, embeddingService);
|
|
3298
|
-
}
|
|
3299
|
-
}
|
|
3300
|
-
}],
|
|
3301
|
-
afterDelete: [...collection.hooks?.afterDelete || [], async ({ doc, req: _req }) => {
|
|
3302
|
-
if (tableConfigs && Array.isArray(tableConfigs)) {
|
|
3303
|
-
for (const tableConfig of tableConfigs) if (tableConfig.enabled) await deleteDocumentFromTypesense(typesenseClient, collection.slug, doc.id, tableConfig);
|
|
3304
|
-
}
|
|
3305
|
-
}]
|
|
3306
|
-
}
|
|
3307
|
-
};
|
|
3308
|
-
}
|
|
3309
|
-
return collection;
|
|
3310
|
-
});
|
|
3311
|
-
};
|
|
2525
|
+
const DEFAULT_RAG_LLM_MODEL = "gpt-4o-mini";
|
|
3312
2526
|
|
|
3313
2527
|
//#endregion
|
|
3314
2528
|
//#region src/shared/schema/collection-schemas.ts
|
|
@@ -3338,7 +2552,7 @@ const getBaseFields = () => [
|
|
|
3338
2552
|
* @param optional - Whether the embedding field is optional
|
|
3339
2553
|
* @param dimensions - Number of dimensions for the embedding vector (default: 1536)
|
|
3340
2554
|
*/
|
|
3341
|
-
const getEmbeddingField = (optional = true, dimensions = DEFAULT_EMBEDDING_DIMENSIONS) => ({
|
|
2555
|
+
const getEmbeddingField = (optional = true, dimensions = DEFAULT_EMBEDDING_DIMENSIONS$1) => ({
|
|
3342
2556
|
name: "embedding",
|
|
3343
2557
|
type: "float[]",
|
|
3344
2558
|
num_dim: dimensions,
|
|
@@ -3387,7 +2601,7 @@ const getChunkFields = () => [
|
|
|
3387
2601
|
/**
|
|
3388
2602
|
* Creates a complete schema for a chunk collection
|
|
3389
2603
|
*/
|
|
3390
|
-
const getChunkCollectionSchema = (collectionSlug, tableConfig, embeddingDimensions = DEFAULT_EMBEDDING_DIMENSIONS) => {
|
|
2604
|
+
const getChunkCollectionSchema = (collectionSlug, tableConfig, embeddingDimensions = DEFAULT_EMBEDDING_DIMENSIONS$1) => {
|
|
3391
2605
|
const fields = tableConfig.fields ? mapFieldMappingsToSchema(tableConfig.fields) : [];
|
|
3392
2606
|
const userFieldNames = new Set([...fields.map((f) => f.name), ...getChunkFields().map((f) => f.name)]);
|
|
3393
2607
|
return {
|
|
@@ -3403,7 +2617,7 @@ const getChunkCollectionSchema = (collectionSlug, tableConfig, embeddingDimensio
|
|
|
3403
2617
|
/**
|
|
3404
2618
|
* Creates a complete schema for a full document collection
|
|
3405
2619
|
*/
|
|
3406
|
-
const getFullDocumentCollectionSchema = (collectionSlug, tableConfig, embeddingDimensions = DEFAULT_EMBEDDING_DIMENSIONS) => {
|
|
2620
|
+
const getFullDocumentCollectionSchema = (collectionSlug, tableConfig, embeddingDimensions = DEFAULT_EMBEDDING_DIMENSIONS$1) => {
|
|
3407
2621
|
const mappedFields = mapFieldMappingsToSchema(tableConfig.fields);
|
|
3408
2622
|
const userFieldNames = new Set(mappedFields.map((f) => f.name));
|
|
3409
2623
|
return {
|
|
@@ -3428,7 +2642,7 @@ var SchemaManager = class {
|
|
|
3428
2642
|
*/
|
|
3429
2643
|
async syncCollections() {
|
|
3430
2644
|
if (!this.config.collections) return;
|
|
3431
|
-
logger.info("Starting schema synchronization...");
|
|
2645
|
+
logger$1.info("Starting schema synchronization...");
|
|
3432
2646
|
const embeddingDimensions = this.getEmbeddingDimensions();
|
|
3433
2647
|
for (const [collectionSlug, tableConfigs] of Object.entries(this.config.collections)) {
|
|
3434
2648
|
if (!tableConfigs) continue;
|
|
@@ -3437,7 +2651,7 @@ var SchemaManager = class {
|
|
|
3437
2651
|
await this.syncTable(collectionSlug, tableConfig, embeddingDimensions);
|
|
3438
2652
|
}
|
|
3439
2653
|
}
|
|
3440
|
-
logger.info("Schema synchronization completed.");
|
|
2654
|
+
logger$1.info("Schema synchronization completed.");
|
|
3441
2655
|
}
|
|
3442
2656
|
/**
|
|
3443
2657
|
* Syncs a single table configuration
|
|
@@ -3452,10 +2666,10 @@ var SchemaManager = class {
|
|
|
3452
2666
|
await this.updateCollectionSchema(tableName, collection, targetSchema);
|
|
3453
2667
|
} catch (error) {
|
|
3454
2668
|
if (error?.httpStatus === 404) {
|
|
3455
|
-
logger.info(`Creating collection: ${tableName}`);
|
|
2669
|
+
logger$1.info(`Creating collection: ${tableName}`);
|
|
3456
2670
|
await this.client.collections().create(targetSchema);
|
|
3457
2671
|
} else {
|
|
3458
|
-
logger.error(`Error checking collection ${tableName}`, error);
|
|
2672
|
+
logger$1.error(`Error checking collection ${tableName}`, error);
|
|
3459
2673
|
throw error;
|
|
3460
2674
|
}
|
|
3461
2675
|
}
|
|
@@ -3465,17 +2679,17 @@ var SchemaManager = class {
|
|
|
3465
2679
|
const currentFields = new Set(currentSchema.fields.map((f) => f.name));
|
|
3466
2680
|
const newFields = targetSchema.fields?.filter((f) => !currentFields.has(f.name) && f.name !== "id") || [];
|
|
3467
2681
|
if (newFields.length > 0) {
|
|
3468
|
-
logger.info(`Updating collection ${tableName} with ${newFields.length} new fields`, { fields: newFields.map((f) => f.name) });
|
|
2682
|
+
logger$1.info(`Updating collection ${tableName} with ${newFields.length} new fields`, { fields: newFields.map((f) => f.name) });
|
|
3469
2683
|
try {
|
|
3470
2684
|
await this.client.collections(tableName).update({ fields: newFields });
|
|
3471
2685
|
} catch (error) {
|
|
3472
|
-
logger.error(`Failed to update collection ${tableName}`, error);
|
|
2686
|
+
logger$1.error(`Failed to update collection ${tableName}`, error);
|
|
3473
2687
|
}
|
|
3474
2688
|
}
|
|
3475
2689
|
}
|
|
3476
2690
|
getEmbeddingDimensions() {
|
|
3477
2691
|
if (this.config.features.embedding?.dimensions) {}
|
|
3478
|
-
return DEFAULT_EMBEDDING_DIMENSIONS;
|
|
2692
|
+
return DEFAULT_EMBEDDING_DIMENSIONS$1;
|
|
3479
2693
|
}
|
|
3480
2694
|
};
|
|
3481
2695
|
|
|
@@ -3490,14 +2704,13 @@ var AgentManager = class {
|
|
|
3490
2704
|
* Synchronizes all configured RAG agents with Typesense
|
|
3491
2705
|
*/
|
|
3492
2706
|
async syncAgents() {
|
|
3493
|
-
|
|
3494
|
-
const agents = this.config.features.rag.agents || [];
|
|
2707
|
+
const agents = this.config.agents || [];
|
|
3495
2708
|
if (agents.length === 0) return;
|
|
3496
|
-
logger.info(`Starting synchronization of ${agents.length} RAG agents...`);
|
|
2709
|
+
logger$1.info(`Starting synchronization of ${agents.length} RAG agents...`);
|
|
3497
2710
|
const historyCollections = new Set(agents.map((a) => a.historyCollection || "conversation_history"));
|
|
3498
2711
|
for (const collectionName of historyCollections) await ensureConversationCollection(this.client, collectionName);
|
|
3499
2712
|
for (const agent of agents) await this.syncAgentModel(agent);
|
|
3500
|
-
logger.info("Agent synchronization completed.");
|
|
2713
|
+
logger$1.info("Agent synchronization completed.");
|
|
3501
2714
|
}
|
|
3502
2715
|
async syncAgentModel(agent) {
|
|
3503
2716
|
try {
|
|
@@ -3513,14 +2726,14 @@ var AgentManager = class {
|
|
|
3513
2726
|
};
|
|
3514
2727
|
return await this.upsertConversationModel(modelConfig);
|
|
3515
2728
|
} catch (error) {
|
|
3516
|
-
logger.error(`Failed to sync agent ${agent.slug}`, error);
|
|
2729
|
+
logger$1.error(`Failed to sync agent ${agent.slug}`, error);
|
|
3517
2730
|
return false;
|
|
3518
2731
|
}
|
|
3519
2732
|
}
|
|
3520
2733
|
async upsertConversationModel(modelConfig) {
|
|
3521
2734
|
const configuration = this.client.configuration;
|
|
3522
2735
|
if (!configuration || !configuration.nodes || configuration.nodes.length === 0) {
|
|
3523
|
-
logger.error("Invalid Typesense client configuration");
|
|
2736
|
+
logger$1.error("Invalid Typesense client configuration");
|
|
3524
2737
|
return false;
|
|
3525
2738
|
}
|
|
3526
2739
|
const node = configuration.nodes[0];
|
|
@@ -3536,11 +2749,11 @@ var AgentManager = class {
|
|
|
3536
2749
|
body: JSON.stringify(modelConfig)
|
|
3537
2750
|
});
|
|
3538
2751
|
if (createResponse.ok) {
|
|
3539
|
-
logger.info(`Agent model created: ${modelConfig.id}`);
|
|
2752
|
+
logger$1.info(`Agent model created: ${modelConfig.id}`);
|
|
3540
2753
|
return true;
|
|
3541
2754
|
}
|
|
3542
2755
|
if (createResponse.status === 409) {
|
|
3543
|
-
logger.debug(`Agent model ${modelConfig.id} exists, updating...`);
|
|
2756
|
+
logger$1.debug(`Agent model ${modelConfig.id} exists, updating...`);
|
|
3544
2757
|
const updateResponse = await fetch(`${baseUrl}/conversations/models/${modelConfig.id}`, {
|
|
3545
2758
|
method: "PUT",
|
|
3546
2759
|
headers: {
|
|
@@ -3550,96 +2763,403 @@ var AgentManager = class {
|
|
|
3550
2763
|
body: JSON.stringify(modelConfig)
|
|
3551
2764
|
});
|
|
3552
2765
|
if (updateResponse.ok) {
|
|
3553
|
-
logger.info(`Agent model updated: ${modelConfig.id}`);
|
|
2766
|
+
logger$1.info(`Agent model updated: ${modelConfig.id}`);
|
|
3554
2767
|
return true;
|
|
3555
2768
|
} else {
|
|
3556
2769
|
const err$1 = await updateResponse.text();
|
|
3557
|
-
logger.error(`Failed to update agent ${modelConfig.id}: ${err$1}`);
|
|
2770
|
+
logger$1.error(`Failed to update agent ${modelConfig.id}: ${err$1}`);
|
|
3558
2771
|
return false;
|
|
3559
2772
|
}
|
|
3560
2773
|
}
|
|
3561
2774
|
const err = await createResponse.text();
|
|
3562
|
-
logger.error(`Failed to create agent ${modelConfig.id}: ${err}`);
|
|
2775
|
+
logger$1.error(`Failed to create agent ${modelConfig.id}: ${err}`);
|
|
3563
2776
|
return false;
|
|
3564
2777
|
} catch (networkError) {
|
|
3565
|
-
logger.error("Network error syncing agent model", networkError);
|
|
2778
|
+
logger$1.error("Network error syncing agent model", networkError);
|
|
3566
2779
|
return false;
|
|
3567
2780
|
}
|
|
3568
2781
|
}
|
|
3569
2782
|
};
|
|
3570
2783
|
|
|
3571
2784
|
//#endregion
|
|
3572
|
-
//#region src/plugin/
|
|
2785
|
+
//#region src/plugin/create-rag-plugin.ts
|
|
3573
2786
|
/**
|
|
3574
|
-
* Typesense
|
|
2787
|
+
* Creates a composable Typesense RAG plugin for Payload CMS
|
|
3575
2788
|
*
|
|
3576
|
-
*
|
|
3577
|
-
*
|
|
2789
|
+
* This plugin handles all Typesense-specific features:
|
|
2790
|
+
* - Search endpoints (semantic, hybrid, keyword)
|
|
2791
|
+
* - RAG endpoints (chat, session management)
|
|
2792
|
+
* - Schema synchronization
|
|
2793
|
+
* - Agent synchronization
|
|
3578
2794
|
*
|
|
3579
|
-
* @param
|
|
2795
|
+
* @param config - Typesense RAG plugin configuration
|
|
3580
2796
|
* @returns Payload config modifier function
|
|
3581
2797
|
*/
|
|
3582
|
-
|
|
3583
|
-
const
|
|
3584
|
-
const logger$1 = new Logger({
|
|
2798
|
+
function createTypesenseRAGPlugin(config) {
|
|
2799
|
+
const logger$2 = new Logger({
|
|
3585
2800
|
enabled: true,
|
|
3586
2801
|
prefix: "[payload-typesense]"
|
|
3587
2802
|
});
|
|
3588
|
-
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
|
|
3592
|
-
|
|
3593
|
-
|
|
3594
|
-
|
|
3595
|
-
|
|
3596
|
-
|
|
3597
|
-
|
|
3598
|
-
|
|
3599
|
-
|
|
3600
|
-
|
|
3601
|
-
|
|
3602
|
-
|
|
3603
|
-
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
|
|
3608
|
-
|
|
2803
|
+
return (payloadConfig) => {
|
|
2804
|
+
const typesenseClient = createTypesenseClient(config.typesense);
|
|
2805
|
+
if (config.search?.enabled) {
|
|
2806
|
+
const searchEndpoints = createSearchEndpoints(typesenseClient, {
|
|
2807
|
+
typesense: config.typesense,
|
|
2808
|
+
features: {
|
|
2809
|
+
embedding: config.embeddingConfig,
|
|
2810
|
+
search: config.search
|
|
2811
|
+
},
|
|
2812
|
+
collections: config.collections || {}
|
|
2813
|
+
});
|
|
2814
|
+
payloadConfig.endpoints = [...payloadConfig.endpoints || [], ...searchEndpoints];
|
|
2815
|
+
logger$2.debug("Search endpoints registered", { endpointsCount: searchEndpoints.length });
|
|
2816
|
+
}
|
|
2817
|
+
if (config.agents && config.agents.length > 0 && config.callbacks) {
|
|
2818
|
+
const ragEndpoints = createRAGPayloadHandlers({
|
|
2819
|
+
typesense: config.typesense,
|
|
2820
|
+
embeddingConfig: config.embeddingConfig,
|
|
2821
|
+
agents: config.agents,
|
|
2822
|
+
callbacks: config.callbacks,
|
|
2823
|
+
hybrid: config.hybrid,
|
|
2824
|
+
hnsw: config.hnsw,
|
|
2825
|
+
advanced: config.advanced
|
|
2826
|
+
});
|
|
2827
|
+
payloadConfig.endpoints = [...payloadConfig.endpoints || [], ...ragEndpoints];
|
|
2828
|
+
logger$2.debug("RAG endpoints registered", {
|
|
2829
|
+
endpointsCount: ragEndpoints.length,
|
|
2830
|
+
agentsCount: config.agents.length
|
|
2831
|
+
});
|
|
2832
|
+
}
|
|
2833
|
+
const incomingOnInit = payloadConfig.onInit;
|
|
2834
|
+
payloadConfig.onInit = async (payload) => {
|
|
2835
|
+
if (incomingOnInit) await incomingOnInit(payload);
|
|
2836
|
+
try {
|
|
2837
|
+
if (config.collections && Object.keys(config.collections).length > 0) {
|
|
2838
|
+
logger$2.info("Syncing Typesense collections schema...");
|
|
2839
|
+
await new SchemaManager(typesenseClient, {
|
|
2840
|
+
typesense: config.typesense,
|
|
2841
|
+
features: { embedding: config.embeddingConfig },
|
|
2842
|
+
collections: config.collections
|
|
2843
|
+
}).syncCollections();
|
|
2844
|
+
}
|
|
2845
|
+
if (config.agents && config.agents.length > 0) {
|
|
2846
|
+
logger$2.info("Initializing RAG agents...");
|
|
2847
|
+
await new AgentManager(typesenseClient, { agents: config.agents }).syncAgents();
|
|
2848
|
+
}
|
|
2849
|
+
} catch (error) {
|
|
2850
|
+
logger$2.error("Error initializing Typesense resources", error);
|
|
2851
|
+
}
|
|
2852
|
+
};
|
|
2853
|
+
return payloadConfig;
|
|
2854
|
+
};
|
|
2855
|
+
}
|
|
2856
|
+
|
|
2857
|
+
//#endregion
|
|
2858
|
+
//#region src/adapter/typesense-adapter.ts
|
|
2859
|
+
/**
|
|
2860
|
+
* Typesense implementation of the IndexerAdapter interface
|
|
2861
|
+
*
|
|
2862
|
+
* This adapter provides type-safe field definitions for Typesense.
|
|
2863
|
+
* When used with createIndexerPlugin, TypeScript will validate that
|
|
2864
|
+
* all field mappings in your collection config are valid TypesenseFieldMapping.
|
|
2865
|
+
*
|
|
2866
|
+
* @example
|
|
2867
|
+
* ```typescript
|
|
2868
|
+
* const adapter = createTypesenseAdapter(config);
|
|
2869
|
+
*
|
|
2870
|
+
* // TypeScript infers TFieldMapping = TypesenseFieldMapping
|
|
2871
|
+
* const { plugin } = createIndexerPlugin({
|
|
2872
|
+
* adapter,
|
|
2873
|
+
* collections: {
|
|
2874
|
+
* posts: [{
|
|
2875
|
+
* enabled: true,
|
|
2876
|
+
* fields: [
|
|
2877
|
+
* { name: 'title', type: 'string' }, // ✅ Valid
|
|
2878
|
+
* { name: 'views', type: 'int64' }, // ✅ Valid
|
|
2879
|
+
* { name: 'tags', type: 'string[]', facet: true }, // ✅ With faceting
|
|
2880
|
+
* ]
|
|
2881
|
+
* }]
|
|
2882
|
+
* }
|
|
2883
|
+
* });
|
|
2884
|
+
* ```
|
|
2885
|
+
*/
|
|
2886
|
+
var TypesenseAdapter = class {
|
|
2887
|
+
name = "typesense";
|
|
2888
|
+
constructor(client) {
|
|
2889
|
+
this.client = client;
|
|
2890
|
+
}
|
|
2891
|
+
/**
|
|
2892
|
+
* Test connection to Typesense
|
|
2893
|
+
*/
|
|
2894
|
+
async testConnection() {
|
|
2895
|
+
try {
|
|
2896
|
+
await this.client.health.retrieve();
|
|
2897
|
+
return true;
|
|
2898
|
+
} catch (error) {
|
|
2899
|
+
logger$1.error("Typesense connection test failed", error);
|
|
2900
|
+
return false;
|
|
2901
|
+
}
|
|
2902
|
+
}
|
|
2903
|
+
/**
|
|
2904
|
+
* Create or update a collection schema
|
|
2905
|
+
*/
|
|
2906
|
+
async ensureCollection(schema) {
|
|
2907
|
+
const typesenseSchema = this.convertToTypesenseSchema(schema);
|
|
3609
2908
|
try {
|
|
3610
|
-
|
|
3611
|
-
await
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
2909
|
+
const existing = await this.client.collections(schema.name).retrieve();
|
|
2910
|
+
await this.updateCollectionIfNeeded(schema.name, existing, typesenseSchema);
|
|
2911
|
+
} catch (error) {
|
|
2912
|
+
if (error?.httpStatus === 404) {
|
|
2913
|
+
logger$1.info(`Creating collection: ${schema.name}`);
|
|
2914
|
+
await this.client.collections().create(typesenseSchema);
|
|
2915
|
+
} else throw error;
|
|
2916
|
+
}
|
|
2917
|
+
}
|
|
2918
|
+
/**
|
|
2919
|
+
* Check if a collection exists
|
|
2920
|
+
*/
|
|
2921
|
+
async collectionExists(collectionName) {
|
|
2922
|
+
try {
|
|
2923
|
+
await this.client.collections(collectionName).retrieve();
|
|
2924
|
+
return true;
|
|
2925
|
+
} catch (error) {
|
|
2926
|
+
if (error?.httpStatus === 404) return false;
|
|
2927
|
+
throw error;
|
|
2928
|
+
}
|
|
2929
|
+
}
|
|
2930
|
+
/**
|
|
2931
|
+
* Delete a collection
|
|
2932
|
+
*/
|
|
2933
|
+
async deleteCollection(collectionName) {
|
|
2934
|
+
try {
|
|
2935
|
+
await this.client.collections(collectionName).delete();
|
|
2936
|
+
logger$1.info(`Deleted collection: ${collectionName}`);
|
|
2937
|
+
} catch (error) {
|
|
2938
|
+
if (error?.httpStatus !== 404) throw error;
|
|
2939
|
+
}
|
|
2940
|
+
}
|
|
2941
|
+
/**
|
|
2942
|
+
* Upsert a single document
|
|
2943
|
+
*/
|
|
2944
|
+
async upsertDocument(collectionName, document) {
|
|
2945
|
+
try {
|
|
2946
|
+
await this.client.collections(collectionName).documents().upsert(document);
|
|
2947
|
+
} catch (error) {
|
|
2948
|
+
logger$1.error(`Failed to upsert document ${document.id} to ${collectionName}`, error);
|
|
2949
|
+
throw error;
|
|
2950
|
+
}
|
|
2951
|
+
}
|
|
2952
|
+
/**
|
|
2953
|
+
* Upsert multiple documents (batch)
|
|
2954
|
+
*/
|
|
2955
|
+
async upsertDocuments(collectionName, documents) {
|
|
2956
|
+
if (documents.length === 0) return;
|
|
2957
|
+
try {
|
|
2958
|
+
await this.client.collections(collectionName).documents().import(documents, { action: "upsert" });
|
|
2959
|
+
} catch (error) {
|
|
2960
|
+
logger$1.error(`Failed to batch upsert ${documents.length} documents to ${collectionName}`, error);
|
|
2961
|
+
throw error;
|
|
2962
|
+
}
|
|
2963
|
+
}
|
|
2964
|
+
/**
|
|
2965
|
+
* Delete a document by ID
|
|
2966
|
+
*/
|
|
2967
|
+
async deleteDocument(collectionName, documentId) {
|
|
2968
|
+
try {
|
|
2969
|
+
await this.client.collections(collectionName).documents(documentId).delete();
|
|
2970
|
+
} catch (error) {
|
|
2971
|
+
if (error?.httpStatus !== 404) {
|
|
2972
|
+
logger$1.error(`Failed to delete document ${documentId} from ${collectionName}`, error);
|
|
2973
|
+
throw error;
|
|
3615
2974
|
}
|
|
2975
|
+
}
|
|
2976
|
+
}
|
|
2977
|
+
/**
|
|
2978
|
+
* Delete documents matching a filter
|
|
2979
|
+
* Returns the number of deleted documents
|
|
2980
|
+
*/
|
|
2981
|
+
async deleteDocumentsByFilter(collectionName, filter) {
|
|
2982
|
+
const filterStr = this.buildFilterString(filter);
|
|
2983
|
+
try {
|
|
2984
|
+
return (await this.client.collections(collectionName).documents().delete({ filter_by: filterStr })).num_deleted || 0;
|
|
3616
2985
|
} catch (error) {
|
|
3617
|
-
logger$1.error(
|
|
2986
|
+
logger$1.error(`Failed to delete documents by filter from ${collectionName}`, error, { filter });
|
|
2987
|
+
throw error;
|
|
3618
2988
|
}
|
|
3619
|
-
}
|
|
3620
|
-
|
|
2989
|
+
}
|
|
2990
|
+
/**
|
|
2991
|
+
* Perform a vector search
|
|
2992
|
+
* @typeParam TDoc - The document type to return in results
|
|
2993
|
+
*/
|
|
2994
|
+
async vectorSearch(collectionName, vector, options = {}) {
|
|
2995
|
+
const { limit = 10, filter, includeFields, excludeFields } = options;
|
|
2996
|
+
try {
|
|
2997
|
+
const searchParams = {
|
|
2998
|
+
q: "*",
|
|
2999
|
+
vector_query: `embedding:([${vector.join(",")}], k:${limit})`
|
|
3000
|
+
};
|
|
3001
|
+
if (filter) searchParams["filter_by"] = this.buildFilterString(filter);
|
|
3002
|
+
if (includeFields) searchParams["include_fields"] = includeFields.join(",");
|
|
3003
|
+
if (excludeFields) searchParams["exclude_fields"] = excludeFields.join(",");
|
|
3004
|
+
return ((await this.client.collections(collectionName).documents().search(searchParams)).hits || []).map((hit) => ({
|
|
3005
|
+
id: String(hit.document?.id || ""),
|
|
3006
|
+
score: hit.vector_distance ?? 0,
|
|
3007
|
+
document: hit.document
|
|
3008
|
+
}));
|
|
3009
|
+
} catch (error) {
|
|
3010
|
+
logger$1.error(`Vector search failed on ${collectionName}`, error);
|
|
3011
|
+
throw error;
|
|
3012
|
+
}
|
|
3013
|
+
}
|
|
3014
|
+
/**
|
|
3015
|
+
* Convert generic schema to Typesense-specific schema
|
|
3016
|
+
*/
|
|
3017
|
+
convertToTypesenseSchema(schema) {
|
|
3018
|
+
return {
|
|
3019
|
+
name: schema.name,
|
|
3020
|
+
fields: schema.fields.map((field) => this.convertField(field)),
|
|
3021
|
+
default_sorting_field: schema.defaultSortingField
|
|
3022
|
+
};
|
|
3023
|
+
}
|
|
3024
|
+
/**
|
|
3025
|
+
* Convert a single field schema to Typesense format
|
|
3026
|
+
*/
|
|
3027
|
+
convertField(field) {
|
|
3028
|
+
const typesenseField = {
|
|
3029
|
+
name: field.name,
|
|
3030
|
+
type: field.type,
|
|
3031
|
+
facet: field.facet,
|
|
3032
|
+
index: field.index,
|
|
3033
|
+
optional: field.optional
|
|
3034
|
+
};
|
|
3035
|
+
if (field.type === "float[]" && field.vectorDimensions) typesenseField.num_dim = field.vectorDimensions;
|
|
3036
|
+
return typesenseField;
|
|
3037
|
+
}
|
|
3038
|
+
/**
|
|
3039
|
+
* Update collection with new fields if needed
|
|
3040
|
+
*/
|
|
3041
|
+
async updateCollectionIfNeeded(collectionName, currentSchema, targetSchema) {
|
|
3042
|
+
if (!currentSchema?.fields) return;
|
|
3043
|
+
const currentFields = new Set(currentSchema.fields.map((f) => f.name));
|
|
3044
|
+
const newFields = targetSchema.fields?.filter((f) => !currentFields.has(f.name) && f.name !== "id") || [];
|
|
3045
|
+
if (newFields.length > 0) {
|
|
3046
|
+
logger$1.info(`Updating collection ${collectionName} with ${newFields.length} new fields`, { fields: newFields.map((f) => f.name) });
|
|
3047
|
+
try {
|
|
3048
|
+
await this.client.collections(collectionName).update({ fields: newFields });
|
|
3049
|
+
} catch (error) {
|
|
3050
|
+
logger$1.error(`Failed to update collection ${collectionName}`, error);
|
|
3051
|
+
}
|
|
3052
|
+
}
|
|
3053
|
+
}
|
|
3054
|
+
/**
|
|
3055
|
+
* Build a Typesense filter string from a filter object
|
|
3056
|
+
*/
|
|
3057
|
+
buildFilterString(filter) {
|
|
3058
|
+
const parts = [];
|
|
3059
|
+
for (const [key, value] of Object.entries(filter)) if (Array.isArray(value)) parts.push(`${key}:[${value.map((v) => String(v)).join(",")}]`);
|
|
3060
|
+
else if (typeof value === "string") parts.push(`${key}:=${value}`);
|
|
3061
|
+
else if (typeof value === "number") parts.push(`${key}:${value}`);
|
|
3062
|
+
else if (typeof value === "boolean") parts.push(`${key}:${value}`);
|
|
3063
|
+
return parts.join(" && ");
|
|
3064
|
+
}
|
|
3621
3065
|
};
|
|
3622
3066
|
|
|
3623
3067
|
//#endregion
|
|
3624
|
-
//#region src/
|
|
3068
|
+
//#region src/adapter/create-adapter.ts
|
|
3625
3069
|
/**
|
|
3626
|
-
*
|
|
3627
|
-
* @param value - The serialized editor state
|
|
3628
|
-
* @param config - Optional Payload config. If provided, it will be used to generate the editor config.
|
|
3070
|
+
* Factory function for creating a TypesenseAdapter
|
|
3629
3071
|
*/
|
|
3630
|
-
|
|
3631
|
-
|
|
3072
|
+
/**
|
|
3073
|
+
* Creates a TypesenseAdapter instance with the provided configuration
|
|
3074
|
+
*
|
|
3075
|
+
* @param config - Typesense connection configuration
|
|
3076
|
+
* @returns A configured TypesenseAdapter instance
|
|
3077
|
+
*
|
|
3078
|
+
* @example
|
|
3079
|
+
* ```typescript
|
|
3080
|
+
* import { createTypesenseAdapter } from '@nexo-labs/payload-typesense';
|
|
3081
|
+
*
|
|
3082
|
+
* const adapter = createTypesenseAdapter({
|
|
3083
|
+
* apiKey: process.env.TYPESENSE_API_KEY!,
|
|
3084
|
+
* nodes: [{
|
|
3085
|
+
* host: 'localhost',
|
|
3086
|
+
* port: 8108,
|
|
3087
|
+
* protocol: 'http'
|
|
3088
|
+
* }]
|
|
3089
|
+
* });
|
|
3090
|
+
* ```
|
|
3091
|
+
*/
|
|
3092
|
+
function createTypesenseAdapter(config) {
|
|
3093
|
+
return new TypesenseAdapter(new Client({
|
|
3094
|
+
apiKey: config.apiKey,
|
|
3095
|
+
nodes: config.nodes,
|
|
3096
|
+
connectionTimeoutSeconds: config.connectionTimeoutSeconds ?? 10,
|
|
3097
|
+
retryIntervalSeconds: config.retryIntervalSeconds,
|
|
3098
|
+
numRetries: config.numRetries
|
|
3099
|
+
}));
|
|
3100
|
+
}
|
|
3101
|
+
/**
|
|
3102
|
+
* Creates a TypesenseAdapter from an existing Typesense Client
|
|
3103
|
+
* Useful when you already have a configured client instance
|
|
3104
|
+
*
|
|
3105
|
+
* @param client - Existing Typesense Client instance
|
|
3106
|
+
* @returns A TypesenseAdapter instance wrapping the provided client
|
|
3107
|
+
*/
|
|
3108
|
+
function createTypesenseAdapterFromClient(client) {
|
|
3109
|
+
return new TypesenseAdapter(client);
|
|
3110
|
+
}
|
|
3111
|
+
|
|
3112
|
+
//#endregion
|
|
3113
|
+
//#region src/features/sync/services/document-delete.ts
|
|
3114
|
+
/**
|
|
3115
|
+
* Deletes a document from Typesense
|
|
3116
|
+
* Handles both direct document deletion and chunk deletion
|
|
3117
|
+
*/
|
|
3118
|
+
const deleteDocumentFromTypesense = async (typesenseClient, collectionSlug, docId, tableConfig) => {
|
|
3632
3119
|
try {
|
|
3633
|
-
|
|
3634
|
-
|
|
3635
|
-
|
|
3120
|
+
const tableName = getTypesenseCollectionName(collectionSlug, tableConfig);
|
|
3121
|
+
logger$1.debug("Attempting to delete document from Typesense", {
|
|
3122
|
+
documentId: docId,
|
|
3123
|
+
collection: collectionSlug,
|
|
3124
|
+
tableName
|
|
3636
3125
|
});
|
|
3126
|
+
try {
|
|
3127
|
+
await typesenseClient.collections(tableName).documents(docId).delete();
|
|
3128
|
+
logger$1.info("Document deleted from Typesense", {
|
|
3129
|
+
documentId: docId,
|
|
3130
|
+
tableName
|
|
3131
|
+
});
|
|
3132
|
+
} catch (docDeleteError) {
|
|
3133
|
+
if (docDeleteError.httpStatus === 404) {
|
|
3134
|
+
logger$1.debug("Document not found, attempting to delete chunks", {
|
|
3135
|
+
documentId: docId,
|
|
3136
|
+
tableName
|
|
3137
|
+
});
|
|
3138
|
+
try {
|
|
3139
|
+
await typesenseClient.collections(tableName).documents().delete({ filter_by: `parent_doc_id:${docId}` });
|
|
3140
|
+
logger$1.info("All chunks deleted for document", {
|
|
3141
|
+
documentId: docId,
|
|
3142
|
+
tableName
|
|
3143
|
+
});
|
|
3144
|
+
} catch (chunkDeleteError) {
|
|
3145
|
+
if (chunkDeleteError.httpStatus !== 404) logger$1.error("Failed to delete chunks for document", chunkDeleteError, {
|
|
3146
|
+
documentId: docId,
|
|
3147
|
+
tableName
|
|
3148
|
+
});
|
|
3149
|
+
else logger$1.debug("No chunks found to delete", { documentId: docId });
|
|
3150
|
+
}
|
|
3151
|
+
} else throw docDeleteError;
|
|
3152
|
+
}
|
|
3637
3153
|
} catch (error) {
|
|
3638
|
-
|
|
3639
|
-
|
|
3154
|
+
const tableName = getTypesenseCollectionName(collectionSlug, tableConfig);
|
|
3155
|
+
logger$1.error("Failed to delete document from Typesense", error, {
|
|
3156
|
+
documentId: docId,
|
|
3157
|
+
collection: collectionSlug,
|
|
3158
|
+
tableName
|
|
3159
|
+
});
|
|
3640
3160
|
}
|
|
3641
3161
|
};
|
|
3642
3162
|
|
|
3643
3163
|
//#endregion
|
|
3644
|
-
export {
|
|
3164
|
+
export { DEFAULT_CACHE_TTL_MS, DEFAULT_HYBRID_SEARCH_ALPHA, DEFAULT_RAG_CONTEXT_LIMIT, DEFAULT_RAG_LLM_MODEL, DEFAULT_RAG_MAX_TOKENS, DEFAULT_SEARCH_LIMIT, DEFAULT_SESSION_TTL_SEC, TypesenseAdapter, buildContextText, buildConversationalUrl, buildHybridSearchParams, buildMultiSearchRequestBody, buildMultiSearchRequests, closeSession, createRAGPayloadHandlers, createSSEForwardStream, createSearchEndpoints, createTypesenseAdapter, createTypesenseAdapterFromClient, createTypesenseClient, createTypesenseRAGPlugin, deleteDocumentFromTypesense, ensureConversationCollection, executeRAGSearch, extractSourcesFromResults, fetchChunkById, formatSSEEvent, generateEmbedding, generateEmbeddingWithUsage, generateEmbeddingsBatchWithUsage, getActiveSession, getDefaultRAGConfig, getSessionByConversationId, jsonResponse, mergeRAGConfigWithDefaults, parseConversationEvent, processConversationStream, saveChatSession, sendSSEEvent, testTypesenseConnection };
|
|
3645
3165
|
//# sourceMappingURL=index.mjs.map
|