@elizaos/plugin-knowledge 1.0.8 → 1.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -38
- package/dist/.vite/manifest.json +2 -2
- package/dist/assets/index-CBT93PUU.css +1 -0
- package/dist/assets/index-ChiOWvZU.js +165 -0
- package/dist/{chunk-536BD2UA.js → chunk-UOE4LEMH.js} +176 -5
- package/dist/chunk-UOE4LEMH.js.map +1 -0
- package/dist/{docs-loader-IBTEOAYT.js → docs-loader-PF5X4UMB.js} +2 -2
- package/dist/index.html +2 -2
- package/dist/index.js +173 -189
- package/dist/index.js.map +1 -1
- package/package.json +8 -6
- package/dist/assets/index-BMDX6vvo.js +0 -160
- package/dist/assets/index-o1rKIvUo.css +0 -1
- package/dist/chunk-536BD2UA.js.map +0 -1
- /package/dist/{docs-loader-IBTEOAYT.js.map → docs-loader-PF5X4UMB.js.map} +0 -0
package/dist/index.js
CHANGED
|
@@ -2,11 +2,13 @@ import {
|
|
|
2
2
|
convertPdfToTextFromBuffer,
|
|
3
3
|
extractTextFromFileBuffer,
|
|
4
4
|
fetchUrlContent,
|
|
5
|
+
generateContentBasedId,
|
|
5
6
|
isBinaryContentType,
|
|
6
7
|
loadDocsFromPath,
|
|
7
8
|
looksLikeBase64,
|
|
8
|
-
normalizeS3Url
|
|
9
|
-
|
|
9
|
+
normalizeS3Url,
|
|
10
|
+
v4_default
|
|
11
|
+
} from "./chunk-UOE4LEMH.js";
|
|
10
12
|
|
|
11
13
|
// src/index.ts
|
|
12
14
|
import { logger as logger7 } from "@elizaos/core";
|
|
@@ -206,51 +208,6 @@ import {
|
|
|
206
208
|
splitChunks
|
|
207
209
|
} from "@elizaos/core";
|
|
208
210
|
|
|
209
|
-
// node_modules/uuid/dist/esm/stringify.js
|
|
210
|
-
var byteToHex = [];
|
|
211
|
-
for (let i = 0; i < 256; ++i) {
|
|
212
|
-
byteToHex.push((i + 256).toString(16).slice(1));
|
|
213
|
-
}
|
|
214
|
-
function unsafeStringify(arr, offset = 0) {
|
|
215
|
-
return (byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + "-" + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + "-" + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + "-" + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + "-" + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]]).toLowerCase();
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
// node_modules/uuid/dist/esm/rng.js
|
|
219
|
-
import { randomFillSync } from "crypto";
|
|
220
|
-
var rnds8Pool = new Uint8Array(256);
|
|
221
|
-
var poolPtr = rnds8Pool.length;
|
|
222
|
-
function rng() {
|
|
223
|
-
if (poolPtr > rnds8Pool.length - 16) {
|
|
224
|
-
randomFillSync(rnds8Pool);
|
|
225
|
-
poolPtr = 0;
|
|
226
|
-
}
|
|
227
|
-
return rnds8Pool.slice(poolPtr, poolPtr += 16);
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
// node_modules/uuid/dist/esm/native.js
|
|
231
|
-
import { randomUUID } from "crypto";
|
|
232
|
-
var native_default = { randomUUID };
|
|
233
|
-
|
|
234
|
-
// node_modules/uuid/dist/esm/v4.js
|
|
235
|
-
function v4(options, buf, offset) {
|
|
236
|
-
if (native_default.randomUUID && !buf && !options) {
|
|
237
|
-
return native_default.randomUUID();
|
|
238
|
-
}
|
|
239
|
-
options = options || {};
|
|
240
|
-
const rnds = options.random || (options.rng || rng)();
|
|
241
|
-
rnds[6] = rnds[6] & 15 | 64;
|
|
242
|
-
rnds[8] = rnds[8] & 63 | 128;
|
|
243
|
-
if (buf) {
|
|
244
|
-
offset = offset || 0;
|
|
245
|
-
for (let i = 0; i < 16; ++i) {
|
|
246
|
-
buf[offset + i] = rnds[i];
|
|
247
|
-
}
|
|
248
|
-
return buf;
|
|
249
|
-
}
|
|
250
|
-
return unsafeStringify(rnds);
|
|
251
|
-
}
|
|
252
|
-
var v4_default = v4;
|
|
253
|
-
|
|
254
211
|
// src/ctx-embeddings.ts
|
|
255
212
|
var DEFAULT_CHUNK_TOKEN_SIZE = 500;
|
|
256
213
|
var DEFAULT_CHUNK_OVERLAP_TOKENS = 100;
|
|
@@ -674,14 +631,6 @@ function getChunkWithContext(chunkContent, generatedContext) {
|
|
|
674
631
|
);
|
|
675
632
|
return chunkContent;
|
|
676
633
|
}
|
|
677
|
-
if (!generatedContext.includes(chunkContent)) {
|
|
678
|
-
console.warn(
|
|
679
|
-
"Generated context does not contain the original chunk. Appending original to ensure data integrity."
|
|
680
|
-
);
|
|
681
|
-
return `${generatedContext.trim()}
|
|
682
|
-
|
|
683
|
-
${chunkContent}`;
|
|
684
|
-
}
|
|
685
634
|
return generatedContext.trim();
|
|
686
635
|
}
|
|
687
636
|
|
|
@@ -1015,7 +964,9 @@ var useCustomLLM = shouldUseCustomLLM();
|
|
|
1015
964
|
if (ctxKnowledgeEnabled) {
|
|
1016
965
|
logger3.info(`Document processor starting with Contextual Knowledge ENABLED`);
|
|
1017
966
|
if (useCustomLLM) {
|
|
1018
|
-
logger3.info(
|
|
967
|
+
logger3.info(
|
|
968
|
+
`Using Custom LLM with provider: ${process.env.TEXT_PROVIDER}, model: ${process.env.TEXT_MODEL}`
|
|
969
|
+
);
|
|
1019
970
|
} else {
|
|
1020
971
|
logger3.info(`Using ElizaOS Runtime LLM (default behavior)`);
|
|
1021
972
|
}
|
|
@@ -1307,6 +1258,8 @@ async function getContextualizedChunks(runtime, fullDocumentText, chunks, conten
|
|
|
1307
1258
|
}
|
|
1308
1259
|
}
|
|
1309
1260
|
async function generateContextsInBatch(runtime, fullDocumentText, chunks, contentType, batchIndices) {
|
|
1261
|
+
console.log("####### generateContextsInBatch FULLL DOCUMENT", fullDocumentText);
|
|
1262
|
+
console.log("####### generateContextsInBatch CHUNKS", chunks);
|
|
1310
1263
|
if (!chunks || chunks.length === 0) {
|
|
1311
1264
|
return [];
|
|
1312
1265
|
}
|
|
@@ -1315,6 +1268,9 @@ async function generateContextsInBatch(runtime, fullDocumentText, chunks, conten
|
|
|
1315
1268
|
const config = validateModelConfig();
|
|
1316
1269
|
const isUsingOpenRouter = config.TEXT_PROVIDER === "openrouter";
|
|
1317
1270
|
const isUsingCacheCapableModel = isUsingOpenRouter && (config.TEXT_MODEL?.toLowerCase().includes("claude") || config.TEXT_MODEL?.toLowerCase().includes("gemini"));
|
|
1271
|
+
logger3.info(
|
|
1272
|
+
`Using provider: ${config.TEXT_PROVIDER}, model: ${config.TEXT_MODEL}, caching capability: ${isUsingCacheCapableModel}`
|
|
1273
|
+
);
|
|
1318
1274
|
const promptConfigs = prepareContextPrompts(
|
|
1319
1275
|
chunks,
|
|
1320
1276
|
fullDocumentText,
|
|
@@ -1337,15 +1293,11 @@ async function generateContextsInBatch(runtime, fullDocumentText, chunks, conten
|
|
|
1337
1293
|
const generateTextOperation = async () => {
|
|
1338
1294
|
if (useCustomLLM) {
|
|
1339
1295
|
if (item.usesCaching) {
|
|
1340
|
-
return await generateText(
|
|
1341
|
-
item.
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
cacheOptions: { type: "ephemeral" },
|
|
1346
|
-
autoCacheContextualRetrieval: true
|
|
1347
|
-
}
|
|
1348
|
-
);
|
|
1296
|
+
return await generateText(item.promptText, item.systemPrompt, {
|
|
1297
|
+
cacheDocument: item.fullDocumentTextForContext,
|
|
1298
|
+
cacheOptions: { type: "ephemeral" },
|
|
1299
|
+
autoCacheContextualRetrieval: true
|
|
1300
|
+
});
|
|
1349
1301
|
} else {
|
|
1350
1302
|
return await generateText(item.prompt);
|
|
1351
1303
|
}
|
|
@@ -1366,7 +1318,7 @@ async function generateContextsInBatch(runtime, fullDocumentText, chunks, conten
|
|
|
1366
1318
|
generateTextOperation,
|
|
1367
1319
|
`context generation for chunk ${item.originalIndex}`
|
|
1368
1320
|
);
|
|
1369
|
-
const generatedContext = llmResponse.text;
|
|
1321
|
+
const generatedContext = typeof llmResponse === "string" ? llmResponse : llmResponse.text;
|
|
1370
1322
|
const contextualizedText = getChunkWithContext(item.chunkText, generatedContext);
|
|
1371
1323
|
logger3.debug(
|
|
1372
1324
|
`Context added for chunk ${item.originalIndex}. New length: ${contextualizedText.length}`
|
|
@@ -1630,36 +1582,42 @@ var KnowledgeService = class _KnowledgeService extends Service {
|
|
|
1630
1582
|
*/
|
|
1631
1583
|
async addKnowledge(options) {
|
|
1632
1584
|
const agentId = options.agentId || this.runtime.agentId;
|
|
1585
|
+
const contentBasedId = generateContentBasedId(options.content, agentId, {
|
|
1586
|
+
includeFilename: options.originalFilename,
|
|
1587
|
+
contentType: options.contentType,
|
|
1588
|
+
maxChars: 2e3
|
|
1589
|
+
// Use first 2KB of content for ID generation
|
|
1590
|
+
});
|
|
1633
1591
|
logger4.info(
|
|
1634
|
-
`KnowledgeService processing document for agent: ${agentId}, file: ${options.originalFilename}, type: ${options.contentType}`
|
|
1592
|
+
`KnowledgeService processing document for agent: ${agentId}, file: ${options.originalFilename}, type: ${options.contentType}, generated ID: ${contentBasedId}`
|
|
1635
1593
|
);
|
|
1636
1594
|
try {
|
|
1637
|
-
const existingDocument = await this.runtime.getMemoryById(
|
|
1595
|
+
const existingDocument = await this.runtime.getMemoryById(contentBasedId);
|
|
1638
1596
|
if (existingDocument && existingDocument.metadata?.type === MemoryType2.DOCUMENT) {
|
|
1639
1597
|
logger4.info(
|
|
1640
|
-
`Document ${options.originalFilename} with ID ${
|
|
1598
|
+
`Document ${options.originalFilename} with ID ${contentBasedId} already exists. Skipping processing.`
|
|
1641
1599
|
);
|
|
1642
1600
|
const fragments = await this.runtime.getMemories({
|
|
1643
1601
|
tableName: "knowledge"
|
|
1644
|
-
// Assuming fragments store original documentId in metadata.documentId
|
|
1645
|
-
// This query might need adjustment based on actual fragment metadata structure.
|
|
1646
|
-
// A more robust way would be to query where metadata.documentId === options.clientDocumentId
|
|
1647
1602
|
});
|
|
1648
1603
|
const relatedFragments = fragments.filter(
|
|
1649
|
-
(f) => f.metadata?.type === MemoryType2.FRAGMENT && f.metadata.documentId ===
|
|
1604
|
+
(f) => f.metadata?.type === MemoryType2.FRAGMENT && f.metadata.documentId === contentBasedId
|
|
1650
1605
|
);
|
|
1651
1606
|
return {
|
|
1652
|
-
clientDocumentId:
|
|
1607
|
+
clientDocumentId: contentBasedId,
|
|
1653
1608
|
storedDocumentMemoryId: existingDocument.id,
|
|
1654
1609
|
fragmentCount: relatedFragments.length
|
|
1655
1610
|
};
|
|
1656
1611
|
}
|
|
1657
1612
|
} catch (error) {
|
|
1658
1613
|
logger4.debug(
|
|
1659
|
-
`Document ${
|
|
1614
|
+
`Document ${contentBasedId} not found or error checking existence, proceeding with processing: ${error instanceof Error ? error.message : String(error)}`
|
|
1660
1615
|
);
|
|
1661
1616
|
}
|
|
1662
|
-
return this.processDocument(
|
|
1617
|
+
return this.processDocument({
|
|
1618
|
+
...options,
|
|
1619
|
+
clientDocumentId: contentBasedId
|
|
1620
|
+
});
|
|
1663
1621
|
}
|
|
1664
1622
|
/**
|
|
1665
1623
|
* Process a document regardless of type - Called by public addKnowledge
|
|
@@ -1851,7 +1809,12 @@ var KnowledgeService = class _KnowledgeService extends Service {
|
|
|
1851
1809
|
const processingPromises = items.map(async (item) => {
|
|
1852
1810
|
await this.knowledgeProcessingSemaphore.acquire();
|
|
1853
1811
|
try {
|
|
1854
|
-
const knowledgeId =
|
|
1812
|
+
const knowledgeId = generateContentBasedId(item, this.runtime.agentId, {
|
|
1813
|
+
maxChars: 2e3,
|
|
1814
|
+
// Use first 2KB of content
|
|
1815
|
+
includeFilename: "character-knowledge"
|
|
1816
|
+
// A constant identifier for character knowledge
|
|
1817
|
+
});
|
|
1855
1818
|
if (await this.checkExistingKnowledge(knowledgeId)) {
|
|
1856
1819
|
logger4.debug(
|
|
1857
1820
|
`KnowledgeService: Character knowledge item with ID ${knowledgeId} already exists. Skipping.`
|
|
@@ -1888,7 +1851,7 @@ var KnowledgeService = class _KnowledgeService extends Service {
|
|
|
1888
1851
|
await this._internalAddKnowledge(
|
|
1889
1852
|
{
|
|
1890
1853
|
id: knowledgeId,
|
|
1891
|
-
// Use the content-
|
|
1854
|
+
// Use the content-based ID
|
|
1892
1855
|
content: {
|
|
1893
1856
|
text: item
|
|
1894
1857
|
},
|
|
@@ -3270,6 +3233,41 @@ var knowledgeActions = [processKnowledgeAction, searchKnowledgeAction];
|
|
|
3270
3233
|
import { createUniqueUuid as createUniqueUuid2, logger as logger6, ModelType as ModelType4 } from "@elizaos/core";
|
|
3271
3234
|
import fs3 from "fs";
|
|
3272
3235
|
import path3 from "path";
|
|
3236
|
+
import multer from "multer";
|
|
3237
|
+
var createUploadMiddleware = (runtime) => {
|
|
3238
|
+
const uploadDir = runtime.getSetting("KNOWLEDGE_UPLOAD_DIR") || "/tmp/uploads/";
|
|
3239
|
+
const maxFileSize = parseInt(runtime.getSetting("KNOWLEDGE_MAX_FILE_SIZE") || "52428800");
|
|
3240
|
+
const maxFiles = parseInt(runtime.getSetting("KNOWLEDGE_MAX_FILES") || "10");
|
|
3241
|
+
const allowedMimeTypes = runtime.getSetting("KNOWLEDGE_ALLOWED_MIME_TYPES")?.split(",") || [
|
|
3242
|
+
"text/plain",
|
|
3243
|
+
"text/markdown",
|
|
3244
|
+
"application/pdf",
|
|
3245
|
+
"application/msword",
|
|
3246
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
3247
|
+
"text/html",
|
|
3248
|
+
"application/json",
|
|
3249
|
+
"application/xml",
|
|
3250
|
+
"text/csv"
|
|
3251
|
+
];
|
|
3252
|
+
return multer({
|
|
3253
|
+
dest: uploadDir,
|
|
3254
|
+
limits: {
|
|
3255
|
+
fileSize: maxFileSize,
|
|
3256
|
+
files: maxFiles
|
|
3257
|
+
},
|
|
3258
|
+
fileFilter: (req, file, cb) => {
|
|
3259
|
+
if (allowedMimeTypes.includes(file.mimetype)) {
|
|
3260
|
+
cb(null, true);
|
|
3261
|
+
} else {
|
|
3262
|
+
cb(
|
|
3263
|
+
new Error(
|
|
3264
|
+
`File type ${file.mimetype} not allowed. Allowed types: ${allowedMimeTypes.join(", ")}`
|
|
3265
|
+
)
|
|
3266
|
+
);
|
|
3267
|
+
}
|
|
3268
|
+
}
|
|
3269
|
+
});
|
|
3270
|
+
};
|
|
3273
3271
|
function sendSuccess(res, data, status = 200) {
|
|
3274
3272
|
res.writeHead(status, { "Content-Type": "application/json" });
|
|
3275
3273
|
res.end(JSON.stringify({ success: true, data }));
|
|
@@ -3289,11 +3287,7 @@ var cleanupFile = (filePath) => {
|
|
|
3289
3287
|
};
|
|
3290
3288
|
var cleanupFiles = (files) => {
|
|
3291
3289
|
if (files) {
|
|
3292
|
-
files.forEach((file) =>
|
|
3293
|
-
if (file.tempFilePath) {
|
|
3294
|
-
cleanupFile(file.tempFilePath);
|
|
3295
|
-
}
|
|
3296
|
-
});
|
|
3290
|
+
files.forEach((file) => cleanupFile(file.path));
|
|
3297
3291
|
}
|
|
3298
3292
|
};
|
|
3299
3293
|
async function uploadKnowledgeHandler(req, res, runtime) {
|
|
@@ -3301,56 +3295,45 @@ async function uploadKnowledgeHandler(req, res, runtime) {
|
|
|
3301
3295
|
if (!service) {
|
|
3302
3296
|
return sendError(res, 500, "SERVICE_NOT_FOUND", "KnowledgeService not found");
|
|
3303
3297
|
}
|
|
3304
|
-
const hasUploadedFiles = req.files &&
|
|
3298
|
+
const hasUploadedFiles = req.files && req.files.length > 0;
|
|
3305
3299
|
const isJsonRequest = !hasUploadedFiles && req.body && (req.body.fileUrl || req.body.fileUrls);
|
|
3306
3300
|
if (!hasUploadedFiles && !isJsonRequest) {
|
|
3307
3301
|
return sendError(res, 400, "INVALID_REQUEST", "Request must contain either files or URLs");
|
|
3308
3302
|
}
|
|
3309
3303
|
try {
|
|
3310
3304
|
if (hasUploadedFiles) {
|
|
3311
|
-
|
|
3312
|
-
if (req.files.files) {
|
|
3313
|
-
if (Array.isArray(req.files.files)) {
|
|
3314
|
-
files = req.files.files;
|
|
3315
|
-
} else {
|
|
3316
|
-
files = [req.files.files];
|
|
3317
|
-
}
|
|
3318
|
-
} else if (req.files.file) {
|
|
3319
|
-
files = [req.files.file];
|
|
3320
|
-
} else {
|
|
3321
|
-
files = Object.values(req.files).flat();
|
|
3322
|
-
}
|
|
3305
|
+
const files = req.files;
|
|
3323
3306
|
if (!files || files.length === 0) {
|
|
3324
3307
|
return sendError(res, 400, "NO_FILES", "No files uploaded");
|
|
3325
3308
|
}
|
|
3326
3309
|
const invalidFiles = files.filter((file) => {
|
|
3327
|
-
if (file.truncated) {
|
|
3328
|
-
logger6.warn(`File ${file.name} was truncated during upload`);
|
|
3329
|
-
return true;
|
|
3330
|
-
}
|
|
3331
3310
|
if (file.size === 0) {
|
|
3332
|
-
logger6.warn(`File ${file.
|
|
3311
|
+
logger6.warn(`File ${file.originalname} is empty`);
|
|
3333
3312
|
return true;
|
|
3334
3313
|
}
|
|
3335
|
-
if (!file.
|
|
3314
|
+
if (!file.originalname || file.originalname.trim() === "") {
|
|
3336
3315
|
logger6.warn(`File has no name`);
|
|
3337
3316
|
return true;
|
|
3338
3317
|
}
|
|
3339
|
-
if (!file.
|
|
3340
|
-
logger6.warn(`File ${file.
|
|
3318
|
+
if (!file.path) {
|
|
3319
|
+
logger6.warn(`File ${file.originalname} has no path`);
|
|
3341
3320
|
return true;
|
|
3342
3321
|
}
|
|
3343
3322
|
return false;
|
|
3344
3323
|
});
|
|
3345
3324
|
if (invalidFiles.length > 0) {
|
|
3346
3325
|
cleanupFiles(files);
|
|
3347
|
-
const invalidFileNames = invalidFiles.map((f) => f.
|
|
3348
|
-
return sendError(
|
|
3326
|
+
const invalidFileNames = invalidFiles.map((f) => f.originalname || "unnamed").join(", ");
|
|
3327
|
+
return sendError(
|
|
3328
|
+
res,
|
|
3329
|
+
400,
|
|
3330
|
+
"INVALID_FILES",
|
|
3331
|
+
`Invalid or corrupted files: ${invalidFileNames}`
|
|
3332
|
+
);
|
|
3349
3333
|
}
|
|
3350
3334
|
const agentId = req.body.agentId || req.query.agentId;
|
|
3351
3335
|
if (!agentId) {
|
|
3352
3336
|
logger6.error("[KNOWLEDGE UPLOAD HANDLER] No agent ID provided in request");
|
|
3353
|
-
cleanupFiles(files);
|
|
3354
3337
|
return sendError(
|
|
3355
3338
|
res,
|
|
3356
3339
|
400,
|
|
@@ -3361,51 +3344,23 @@ async function uploadKnowledgeHandler(req, res, runtime) {
|
|
|
3361
3344
|
const worldId = req.body.worldId || agentId;
|
|
3362
3345
|
logger6.info(`[KNOWLEDGE UPLOAD HANDLER] Processing upload for agent: ${agentId}`);
|
|
3363
3346
|
const processingPromises = files.map(async (file, index) => {
|
|
3364
|
-
|
|
3365
|
-
const
|
|
3366
|
-
const filePath = file.tempFilePath;
|
|
3367
|
-
knowledgeId = req.body?.documentIds && req.body.documentIds[index] || req.body?.documentId || createUniqueUuid2(runtime, `knowledge-${originalFilename}-${Date.now()}`);
|
|
3347
|
+
const originalFilename = file.originalname;
|
|
3348
|
+
const filePath = file.path;
|
|
3368
3349
|
logger6.debug(
|
|
3369
|
-
`[KNOWLEDGE UPLOAD HANDLER] File: ${originalFilename}, Agent ID: ${agentId}, World ID: ${worldId}
|
|
3350
|
+
`[KNOWLEDGE UPLOAD HANDLER] File: ${originalFilename}, Agent ID: ${agentId}, World ID: ${worldId}`
|
|
3370
3351
|
);
|
|
3371
3352
|
try {
|
|
3372
|
-
|
|
3373
|
-
if (filePath && fs3.existsSync(filePath)) {
|
|
3374
|
-
try {
|
|
3375
|
-
const stats = await fs3.promises.stat(filePath);
|
|
3376
|
-
if (stats.size === 0) {
|
|
3377
|
-
throw new Error("Temporary file is empty");
|
|
3378
|
-
}
|
|
3379
|
-
fileBuffer = await fs3.promises.readFile(filePath);
|
|
3380
|
-
logger6.debug(`[KNOWLEDGE UPLOAD] Read ${fileBuffer.length} bytes from temp file: ${filePath}`);
|
|
3381
|
-
} catch (fsError) {
|
|
3382
|
-
throw new Error(`Failed to read temporary file: ${fsError.message}`);
|
|
3383
|
-
}
|
|
3384
|
-
} else if (file.data && Buffer.isBuffer(file.data)) {
|
|
3385
|
-
fileBuffer = file.data;
|
|
3386
|
-
logger6.debug(`[KNOWLEDGE UPLOAD] Using in-memory buffer of ${fileBuffer.length} bytes`);
|
|
3387
|
-
} else {
|
|
3388
|
-
throw new Error("No file data available - neither temp file nor buffer found");
|
|
3389
|
-
}
|
|
3390
|
-
if (!Buffer.isBuffer(fileBuffer) || fileBuffer.length === 0) {
|
|
3391
|
-
throw new Error("Invalid or empty file buffer");
|
|
3392
|
-
}
|
|
3393
|
-
if (fileBuffer.length !== file.size) {
|
|
3394
|
-
logger6.warn(`File size mismatch for ${originalFilename}: expected ${file.size}, got ${fileBuffer.length}`);
|
|
3395
|
-
}
|
|
3353
|
+
const fileBuffer = await fs3.promises.readFile(filePath);
|
|
3396
3354
|
const base64Content = fileBuffer.toString("base64");
|
|
3397
|
-
if (!base64Content || base64Content.length === 0) {
|
|
3398
|
-
throw new Error("Failed to convert file to base64");
|
|
3399
|
-
}
|
|
3400
3355
|
const addKnowledgeOpts = {
|
|
3401
3356
|
agentId,
|
|
3402
3357
|
// Pass the agent ID from frontend
|
|
3403
|
-
clientDocumentId:
|
|
3404
|
-
// This
|
|
3358
|
+
clientDocumentId: "",
|
|
3359
|
+
// This will be ignored by the service
|
|
3405
3360
|
contentType: file.mimetype,
|
|
3406
|
-
// Directly from
|
|
3361
|
+
// Directly from multer file object
|
|
3407
3362
|
originalFilename,
|
|
3408
|
-
// Directly from
|
|
3363
|
+
// Directly from multer file object
|
|
3409
3364
|
content: base64Content,
|
|
3410
3365
|
// The base64 string of the file
|
|
3411
3366
|
worldId,
|
|
@@ -3414,12 +3369,11 @@ async function uploadKnowledgeHandler(req, res, runtime) {
|
|
|
3414
3369
|
entityId: agentId
|
|
3415
3370
|
// Use the correct agent ID
|
|
3416
3371
|
};
|
|
3417
|
-
await service.addKnowledge(addKnowledgeOpts);
|
|
3418
|
-
|
|
3419
|
-
cleanupFile(filePath);
|
|
3420
|
-
}
|
|
3372
|
+
const result = await service.addKnowledge(addKnowledgeOpts);
|
|
3373
|
+
cleanupFile(filePath);
|
|
3421
3374
|
return {
|
|
3422
|
-
id:
|
|
3375
|
+
id: result.clientDocumentId,
|
|
3376
|
+
// Use the content-based ID returned by the service
|
|
3423
3377
|
filename: originalFilename,
|
|
3424
3378
|
type: file.mimetype,
|
|
3425
3379
|
size: file.size,
|
|
@@ -3428,13 +3382,12 @@ async function uploadKnowledgeHandler(req, res, runtime) {
|
|
|
3428
3382
|
};
|
|
3429
3383
|
} catch (fileError) {
|
|
3430
3384
|
logger6.error(
|
|
3431
|
-
`[KNOWLEDGE UPLOAD HANDLER] Error processing file ${file.
|
|
3385
|
+
`[KNOWLEDGE UPLOAD HANDLER] Error processing file ${file.originalname}: ${fileError}`
|
|
3432
3386
|
);
|
|
3433
|
-
|
|
3434
|
-
cleanupFile(filePath);
|
|
3435
|
-
}
|
|
3387
|
+
cleanupFile(filePath);
|
|
3436
3388
|
return {
|
|
3437
|
-
id:
|
|
3389
|
+
id: "",
|
|
3390
|
+
// No ID since processing failed
|
|
3438
3391
|
filename: originalFilename,
|
|
3439
3392
|
status: "error_processing",
|
|
3440
3393
|
error: fileError.message
|
|
@@ -3462,7 +3415,6 @@ async function uploadKnowledgeHandler(req, res, runtime) {
|
|
|
3462
3415
|
const processingPromises = fileUrls.map(async (fileUrl) => {
|
|
3463
3416
|
try {
|
|
3464
3417
|
const normalizedUrl = normalizeS3Url(fileUrl);
|
|
3465
|
-
const knowledgeId = createUniqueUuid2(runtime, normalizedUrl);
|
|
3466
3418
|
const urlObject = new URL(fileUrl);
|
|
3467
3419
|
const pathSegments = urlObject.pathname.split("/");
|
|
3468
3420
|
const encodedFilename = pathSegments[pathSegments.length - 1] || "document.pdf";
|
|
@@ -3493,7 +3445,8 @@ async function uploadKnowledgeHandler(req, res, runtime) {
|
|
|
3493
3445
|
const addKnowledgeOpts = {
|
|
3494
3446
|
agentId,
|
|
3495
3447
|
// Pass the agent ID from frontend
|
|
3496
|
-
clientDocumentId:
|
|
3448
|
+
clientDocumentId: "",
|
|
3449
|
+
// This will be ignored by the service
|
|
3497
3450
|
contentType,
|
|
3498
3451
|
originalFilename,
|
|
3499
3452
|
content,
|
|
@@ -3512,6 +3465,7 @@ async function uploadKnowledgeHandler(req, res, runtime) {
|
|
|
3512
3465
|
const result = await service.addKnowledge(addKnowledgeOpts);
|
|
3513
3466
|
return {
|
|
3514
3467
|
id: result.clientDocumentId,
|
|
3468
|
+
// Use the content-based ID returned by the service
|
|
3515
3469
|
fileUrl,
|
|
3516
3470
|
filename: originalFilename,
|
|
3517
3471
|
message: "Knowledge created successfully",
|
|
@@ -3533,9 +3487,8 @@ async function uploadKnowledgeHandler(req, res, runtime) {
|
|
|
3533
3487
|
}
|
|
3534
3488
|
} catch (error) {
|
|
3535
3489
|
logger6.error("[KNOWLEDGE HANDLER] Error processing knowledge:", error);
|
|
3536
|
-
if (hasUploadedFiles
|
|
3537
|
-
|
|
3538
|
-
cleanupFiles(allFiles);
|
|
3490
|
+
if (hasUploadedFiles) {
|
|
3491
|
+
cleanupFiles(req.files);
|
|
3539
3492
|
}
|
|
3540
3493
|
sendError(res, 500, "PROCESSING_ERROR", "Failed to process knowledge", error.message);
|
|
3541
3494
|
}
|
|
@@ -3794,19 +3747,56 @@ async function getKnowledgeChunksHandler(req, res, runtime) {
|
|
|
3794
3747
|
return sendError(res, 500, "SERVICE_NOT_FOUND", "KnowledgeService not found");
|
|
3795
3748
|
}
|
|
3796
3749
|
try {
|
|
3797
|
-
const limit = req.query.limit ? Number.parseInt(req.query.limit, 10) : 100;
|
|
3798
|
-
const before = req.query.before ? Number.parseInt(req.query.before, 10) : Date.now();
|
|
3799
3750
|
const documentId = req.query.documentId;
|
|
3800
|
-
const
|
|
3801
|
-
const
|
|
3802
|
-
tableName: "
|
|
3803
|
-
count:
|
|
3804
|
-
|
|
3751
|
+
const documentsOnly = req.query.documentsOnly === "true";
|
|
3752
|
+
const documents = await service.getMemories({
|
|
3753
|
+
tableName: "documents",
|
|
3754
|
+
count: 1e3,
|
|
3755
|
+
// Reasonable limit for documents
|
|
3756
|
+
end: Date.now()
|
|
3757
|
+
});
|
|
3758
|
+
if (documentsOnly) {
|
|
3759
|
+
sendSuccess(res, {
|
|
3760
|
+
chunks: documents,
|
|
3761
|
+
stats: {
|
|
3762
|
+
documents: documents.length,
|
|
3763
|
+
fragments: 0,
|
|
3764
|
+
mode: "documents-only"
|
|
3765
|
+
}
|
|
3766
|
+
});
|
|
3767
|
+
return;
|
|
3768
|
+
}
|
|
3769
|
+
if (documentId) {
|
|
3770
|
+
const allFragments = await service.getMemories({
|
|
3771
|
+
tableName: "knowledge",
|
|
3772
|
+
count: 1e5
|
|
3773
|
+
// Very high limit to get all fragments
|
|
3774
|
+
});
|
|
3775
|
+
const documentFragments = allFragments.filter((fragment) => {
|
|
3776
|
+
const metadata = fragment.metadata;
|
|
3777
|
+
return metadata?.documentId === documentId;
|
|
3778
|
+
});
|
|
3779
|
+
const specificDocument = documents.find((d) => d.id === documentId);
|
|
3780
|
+
const results = specificDocument ? [specificDocument, ...documentFragments] : documentFragments;
|
|
3781
|
+
sendSuccess(res, {
|
|
3782
|
+
chunks: results,
|
|
3783
|
+
stats: {
|
|
3784
|
+
documents: specificDocument ? 1 : 0,
|
|
3785
|
+
fragments: documentFragments.length,
|
|
3786
|
+
mode: "single-document",
|
|
3787
|
+
documentId
|
|
3788
|
+
}
|
|
3789
|
+
});
|
|
3790
|
+
return;
|
|
3791
|
+
}
|
|
3792
|
+
sendSuccess(res, {
|
|
3793
|
+
chunks: documents,
|
|
3794
|
+
stats: {
|
|
3795
|
+
documents: documents.length,
|
|
3796
|
+
fragments: 0,
|
|
3797
|
+
mode: "documents-only"
|
|
3798
|
+
}
|
|
3805
3799
|
});
|
|
3806
|
-
const filteredChunks = documentId ? chunks.filter(
|
|
3807
|
-
(chunk) => chunk.metadata && typeof chunk.metadata === "object" && "documentId" in chunk.metadata && chunk.metadata.documentId === documentId
|
|
3808
|
-
) : chunks;
|
|
3809
|
-
sendSuccess(res, { chunks: filteredChunks });
|
|
3810
3800
|
} catch (error) {
|
|
3811
3801
|
logger6.error("[KNOWLEDGE CHUNKS GET HANDLER] Error retrieving chunks:", error);
|
|
3812
3802
|
sendError(res, 500, "RETRIEVAL_ERROR", "Failed to retrieve knowledge chunks", error.message);
|
|
@@ -3893,25 +3883,19 @@ async function searchKnowledgeHandler(req, res, runtime) {
|
|
|
3893
3883
|
sendError(res, 500, "SEARCH_ERROR", "Failed to search knowledge", error.message);
|
|
3894
3884
|
}
|
|
3895
3885
|
}
|
|
3896
|
-
async function
|
|
3897
|
-
|
|
3898
|
-
|
|
3899
|
-
|
|
3900
|
-
|
|
3901
|
-
|
|
3902
|
-
|
|
3903
|
-
|
|
3904
|
-
|
|
3905
|
-
|
|
3906
|
-
try {
|
|
3907
|
-
logger6.debug("[KNOWLEDGE UPLOAD] Using files parsed by global middleware");
|
|
3908
|
-
await uploadKnowledgeHandler(req, res, runtime);
|
|
3909
|
-
} catch (handlerError) {
|
|
3910
|
-
logger6.error("[KNOWLEDGE UPLOAD] Handler error:", handlerError);
|
|
3911
|
-
if (!res.headersSent) {
|
|
3912
|
-
sendError(res, 500, "HANDLER_ERROR", "Failed to process upload");
|
|
3886
|
+
async function uploadKnowledgeWithMulter(req, res, runtime) {
|
|
3887
|
+
const upload = createUploadMiddleware(runtime);
|
|
3888
|
+
const uploadArray = upload.array(
|
|
3889
|
+
"files",
|
|
3890
|
+
parseInt(runtime.getSetting("KNOWLEDGE_MAX_FILES") || "10")
|
|
3891
|
+
);
|
|
3892
|
+
uploadArray(req, res, (err) => {
|
|
3893
|
+
if (err) {
|
|
3894
|
+
logger6.error("[KNOWLEDGE UPLOAD] Multer error:", err);
|
|
3895
|
+
return sendError(res, 400, "UPLOAD_ERROR", err.message);
|
|
3913
3896
|
}
|
|
3914
|
-
|
|
3897
|
+
uploadKnowledgeHandler(req, res, runtime);
|
|
3898
|
+
});
|
|
3915
3899
|
}
|
|
3916
3900
|
var knowledgeRoutes = [
|
|
3917
3901
|
{
|
|
@@ -3929,7 +3913,7 @@ var knowledgeRoutes = [
|
|
|
3929
3913
|
{
|
|
3930
3914
|
type: "POST",
|
|
3931
3915
|
path: "/documents",
|
|
3932
|
-
handler:
|
|
3916
|
+
handler: uploadKnowledgeWithMulter
|
|
3933
3917
|
},
|
|
3934
3918
|
{
|
|
3935
3919
|
type: "GET",
|
|
@@ -4004,7 +3988,7 @@ var knowledgePlugin = {
|
|
|
4004
3988
|
try {
|
|
4005
3989
|
const service = runtime.getService(KnowledgeService.serviceType);
|
|
4006
3990
|
if (service instanceof KnowledgeService) {
|
|
4007
|
-
const { loadDocsFromPath: loadDocsFromPath2 } = await import("./docs-loader-
|
|
3991
|
+
const { loadDocsFromPath: loadDocsFromPath2 } = await import("./docs-loader-PF5X4UMB.js");
|
|
4008
3992
|
const result = await loadDocsFromPath2(service, runtime.agentId);
|
|
4009
3993
|
if (result.successful > 0) {
|
|
4010
3994
|
logger7.info(`Loaded ${result.successful} documents from docs folder on startup`);
|