@botpress/adk 1.10.4 → 1.10.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bot-generator/generator.d.ts.map +1 -1
- package/dist/index.js +159 -77
- package/dist/index.js.map +6 -6
- package/dist/knowledge/manager.d.ts +19 -6
- package/dist/knowledge/manager.d.ts.map +1 -1
- package/dist/knowledge/types.d.ts +7 -5
- package/dist/knowledge/types.d.ts.map +1 -1
- package/package.json +2 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generator.d.ts","sourceRoot":"","sources":["../../src/bot-generator/generator.ts"],"names":[],"mappings":"AA4BA,MAAM,WAAW,mBAAmB;IAClC,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,SAAS,GAAG,WAAW,GAAG,YAAY,CAAA;CACpD;AAkBD,qBAAa,YAAY;IACvB,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,UAAU,CAAQ;gBAEd,OAAO,EAAE,mBAAmB;YAK1B,kBAAkB;YAuBlB,sBAAsB;IAsB9B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;YA+BjB,yBAAyB;YAkEzB,kBAAkB;YAKlB,oBAAoB;YAKpB,gBAAgB;YAKhB,kBAAkB;YAKlB,0BAA0B;YAK1B,kBAAkB;YAKlB,qBAAqB;YAKrB,yBAAyB;YAKzB,mBAAmB;YAMnB,8BAA8B;YAM9B,oBAAoB;YAyEpB,uBAAuB;YA8FvB,8BAA8B;YAiC9B,4BAA4B;YA2B5B,qBAAqB;
|
|
1
|
+
{"version":3,"file":"generator.d.ts","sourceRoot":"","sources":["../../src/bot-generator/generator.ts"],"names":[],"mappings":"AA4BA,MAAM,WAAW,mBAAmB;IAClC,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,SAAS,GAAG,WAAW,GAAG,YAAY,CAAA;CACpD;AAkBD,qBAAa,YAAY;IACvB,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,UAAU,CAAQ;gBAEd,OAAO,EAAE,mBAAmB;YAK1B,kBAAkB;YAuBlB,sBAAsB;IAsB9B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;YA+BjB,yBAAyB;YAkEzB,kBAAkB;YAKlB,oBAAoB;YAKpB,gBAAgB;YAKhB,kBAAkB;YAKlB,0BAA0B;YAK1B,kBAAkB;YAKlB,qBAAqB;YAKrB,yBAAyB;YAKzB,mBAAmB;YAMnB,8BAA8B;YAM9B,oBAAoB;YAyEpB,uBAAuB;YA8FvB,8BAA8B;YAiC9B,4BAA4B;YA2B5B,qBAAqB;YAgXrB,gBAAgB;YAoChB,mBAAmB;YAqBnB,gBAAgB;YAiChB,mBAAmB;YAyBnB,UAAU;YAWV,aAAa;IAgBrB,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC;IA6ZnC,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;CAOzC;AAED,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CA0CpF"}
|
package/dist/index.js
CHANGED
|
@@ -654,7 +654,7 @@ var PRETTIER_CONFIG, formatCode = async (code, filepath) => {
|
|
|
654
654
|
`));
|
|
655
655
|
return code;
|
|
656
656
|
}
|
|
657
|
-
}, ADK_VERSION = "1.10.
|
|
657
|
+
}, ADK_VERSION = "1.10.5", relative2 = (from, to) => {
|
|
658
658
|
const fromDir = path10.dirname(from);
|
|
659
659
|
const relative3 = path10.relative(fromDir, to);
|
|
660
660
|
return relative3.startsWith(".") ? relative3 : `./${relative3}`;
|
|
@@ -797,7 +797,7 @@ var init_integration_action_types = __esm(() => {
|
|
|
797
797
|
var require_package = __commonJS((exports, module) => {
|
|
798
798
|
module.exports = {
|
|
799
799
|
name: "@botpress/adk",
|
|
800
|
-
version: "1.10.
|
|
800
|
+
version: "1.10.5",
|
|
801
801
|
description: "Core ADK library for building AI agents on Botpress",
|
|
802
802
|
type: "module",
|
|
803
803
|
main: "dist/index.js",
|
|
@@ -844,7 +844,7 @@ var require_package = __commonJS((exports, module) => {
|
|
|
844
844
|
"@botpress/cli": "^4.27.3",
|
|
845
845
|
"@botpress/client": "^1.27.2",
|
|
846
846
|
"@botpress/cognitive": "^0.2.0",
|
|
847
|
-
"@botpress/runtime": "^1.10.
|
|
847
|
+
"@botpress/runtime": "^1.10.5",
|
|
848
848
|
"@botpress/sdk": "^4.18.1",
|
|
849
849
|
"@bpinternal/jex": "^1.2.4",
|
|
850
850
|
"@bpinternal/yargs-extra": "^0.0.21",
|
|
@@ -4866,7 +4866,7 @@ class AgentProjectGenerator {
|
|
|
4866
4866
|
deploy: "adk deploy"
|
|
4867
4867
|
},
|
|
4868
4868
|
dependencies: {
|
|
4869
|
-
"@botpress/runtime": `^${"1.10.
|
|
4869
|
+
"@botpress/runtime": `^${"1.10.5"}`
|
|
4870
4870
|
},
|
|
4871
4871
|
devDependencies: {
|
|
4872
4872
|
typescript: "^5.9.3"
|
|
@@ -7108,6 +7108,14 @@ configuration: {
|
|
|
7108
7108
|
type: "workflow",
|
|
7109
7109
|
schema: TrackedStateSchema,
|
|
7110
7110
|
},
|
|
7111
|
+
|
|
7112
|
+
/**
|
|
7113
|
+
* Data source metadata for dashboard visibility (knowledge base sources)
|
|
7114
|
+
*/
|
|
7115
|
+
[BUILT_IN_STATES.dsData]: {
|
|
7116
|
+
type: "bot",
|
|
7117
|
+
schema: z.record(z.any()),
|
|
7118
|
+
},
|
|
7111
7119
|
},
|
|
7112
7120
|
});
|
|
7113
7121
|
|
|
@@ -8089,13 +8097,14 @@ var KBSyncOperation;
|
|
|
8089
8097
|
|
|
8090
8098
|
// src/knowledge/manager.ts
|
|
8091
8099
|
var WellKnownTags = {
|
|
8092
|
-
KNOWLEDGE: "
|
|
8100
|
+
KNOWLEDGE: "source",
|
|
8101
|
+
KNOWLEDGE_BASE_ID: "kbId",
|
|
8093
8102
|
KNOWLEDGE_BASE_NAME: "kbName",
|
|
8094
|
-
KNOWLEDGE_SOURCE_ID: "
|
|
8095
|
-
KNOWLEDGE_SOURCE_TYPE: "
|
|
8103
|
+
KNOWLEDGE_SOURCE_ID: "dsId",
|
|
8104
|
+
KNOWLEDGE_SOURCE_TYPE: "dsType"
|
|
8096
8105
|
};
|
|
8097
|
-
function sourceTag(
|
|
8098
|
-
const sanitizedId =
|
|
8106
|
+
function sourceTag(dsId, field) {
|
|
8107
|
+
const sanitizedId = dsId.replace(/[^a-zA-Z0-9]/g, "").toLowerCase();
|
|
8099
8108
|
return `source${sanitizedId}${field}`;
|
|
8100
8109
|
}
|
|
8101
8110
|
var WellKnownMetadata = {
|
|
@@ -8194,10 +8203,10 @@ class KnowledgeManager {
|
|
|
8194
8203
|
const remoteKbs = await this.listRemoteKnowledgeBases();
|
|
8195
8204
|
return remoteKbs.filter((kb) => !localKbNames.includes(kb.name));
|
|
8196
8205
|
}
|
|
8197
|
-
async getOrphanedSources(kbName,
|
|
8206
|
+
async getOrphanedSources(kbName, localDsIds) {
|
|
8198
8207
|
const client = await this.getClient();
|
|
8199
8208
|
const tags = {
|
|
8200
|
-
[WellKnownTags.KNOWLEDGE]: "
|
|
8209
|
+
[WellKnownTags.KNOWLEDGE]: "knowledge-base",
|
|
8201
8210
|
[WellKnownTags.KNOWLEDGE_BASE_NAME]: kbName
|
|
8202
8211
|
};
|
|
8203
8212
|
const files = [];
|
|
@@ -8207,40 +8216,40 @@ class KnowledgeManager {
|
|
|
8207
8216
|
files.push(...response.files);
|
|
8208
8217
|
nextToken = response.meta.nextToken;
|
|
8209
8218
|
} while (nextToken);
|
|
8210
|
-
const
|
|
8219
|
+
const filesByDsId = new Map;
|
|
8211
8220
|
for (const file of files) {
|
|
8212
|
-
const
|
|
8213
|
-
if (
|
|
8214
|
-
|
|
8221
|
+
const dsId = file.tags?.[WellKnownTags.KNOWLEDGE_SOURCE_ID] || file.metadata?.dsId;
|
|
8222
|
+
if (dsId) {
|
|
8223
|
+
filesByDsId.set(dsId, (filesByDsId.get(dsId) || 0) + 1);
|
|
8215
8224
|
}
|
|
8216
8225
|
}
|
|
8217
8226
|
const orphaned = [];
|
|
8218
|
-
for (const [
|
|
8219
|
-
if (!
|
|
8220
|
-
orphaned.push({
|
|
8227
|
+
for (const [dsId, fileCount] of filesByDsId) {
|
|
8228
|
+
if (!localDsIds.includes(dsId)) {
|
|
8229
|
+
orphaned.push({ dsId, fileCount });
|
|
8221
8230
|
}
|
|
8222
8231
|
}
|
|
8223
8232
|
return orphaned;
|
|
8224
8233
|
}
|
|
8225
|
-
async deleteOrphanedSource(kbName,
|
|
8234
|
+
async deleteOrphanedSource(kbName, dsId) {
|
|
8226
8235
|
const client = await this.getClient();
|
|
8227
8236
|
const tags = {
|
|
8228
|
-
[WellKnownTags.KNOWLEDGE]: "
|
|
8237
|
+
[WellKnownTags.KNOWLEDGE]: "knowledge-base",
|
|
8229
8238
|
[WellKnownTags.KNOWLEDGE_BASE_NAME]: kbName,
|
|
8230
|
-
[WellKnownTags.KNOWLEDGE_SOURCE_ID]:
|
|
8239
|
+
[WellKnownTags.KNOWLEDGE_SOURCE_ID]: dsId
|
|
8231
8240
|
};
|
|
8232
8241
|
const files = await this.listExistingFiles(client, tags);
|
|
8233
|
-
console.log(` Deleting ${files.length} files from orphaned source "${
|
|
8242
|
+
console.log(` Deleting ${files.length} files from orphaned source "${dsId}"...`);
|
|
8234
8243
|
const results = await Promise.allSettled(files.map((f) => client.deleteFile({ id: f.id })));
|
|
8235
8244
|
const deletedFiles = results.filter((r) => r.status === "fulfilled").length;
|
|
8236
8245
|
const errors = results.filter((r) => r.status === "rejected").map((r) => String(r.reason));
|
|
8237
8246
|
return { deletedFiles, errors };
|
|
8238
8247
|
}
|
|
8239
|
-
async cleanupSourceTags(kbId, kbName,
|
|
8248
|
+
async cleanupSourceTags(kbId, kbName, dsIds, existingTags) {
|
|
8240
8249
|
const client = await this.getClient();
|
|
8241
8250
|
const newTags = { ...existingTags };
|
|
8242
|
-
for (const
|
|
8243
|
-
const sanitizedId =
|
|
8251
|
+
for (const dsId of dsIds) {
|
|
8252
|
+
const sanitizedId = dsId.replace(/[^a-zA-Z0-9]/g, "").toLowerCase();
|
|
8244
8253
|
newTags[`source${sanitizedId}hash`] = "";
|
|
8245
8254
|
newTags[`source${sanitizedId}lastupdatedat`] = "";
|
|
8246
8255
|
}
|
|
@@ -8249,7 +8258,7 @@ class KnowledgeManager {
|
|
|
8249
8258
|
async deleteKnowledgeBase(kbId, kbName) {
|
|
8250
8259
|
const client = await this.getClient();
|
|
8251
8260
|
const tags = {
|
|
8252
|
-
[WellKnownTags.KNOWLEDGE]: "
|
|
8261
|
+
[WellKnownTags.KNOWLEDGE]: "knowledge-base",
|
|
8253
8262
|
[WellKnownTags.KNOWLEDGE_BASE_NAME]: kbName
|
|
8254
8263
|
};
|
|
8255
8264
|
console.log(` Listing files for KB "${kbName}"...`);
|
|
@@ -8261,18 +8270,18 @@ class KnowledgeManager {
|
|
|
8261
8270
|
await client.deleteKnowledgeBase({ id: kbId });
|
|
8262
8271
|
return { deletedFiles };
|
|
8263
8272
|
}
|
|
8264
|
-
getRemoteSourceHash(kb,
|
|
8265
|
-
return kb.tags?.[sourceTag(
|
|
8273
|
+
getRemoteSourceHash(kb, dsId) {
|
|
8274
|
+
return kb.tags?.[sourceTag(dsId, "hash")];
|
|
8266
8275
|
}
|
|
8267
|
-
async updateSourceHash(kbId, kbName,
|
|
8276
|
+
async updateSourceHash(kbId, kbName, dsId, hash, existingTags) {
|
|
8268
8277
|
const client = await this.getClient();
|
|
8269
8278
|
await client.updateKnowledgeBase({
|
|
8270
8279
|
id: kbId,
|
|
8271
8280
|
name: kbName,
|
|
8272
8281
|
tags: {
|
|
8273
8282
|
...existingTags,
|
|
8274
|
-
[sourceTag(
|
|
8275
|
-
[sourceTag(
|
|
8283
|
+
[sourceTag(dsId, "hash")]: hash,
|
|
8284
|
+
[sourceTag(dsId, "lastupdatedat")]: new Date().toISOString()
|
|
8276
8285
|
}
|
|
8277
8286
|
});
|
|
8278
8287
|
}
|
|
@@ -8280,16 +8289,43 @@ class KnowledgeManager {
|
|
|
8280
8289
|
const sortedConfig = JSON.stringify(config, Object.keys(config).sort());
|
|
8281
8290
|
return crypto4.createHash("sha256").update(sortedConfig).digest("hex");
|
|
8282
8291
|
}
|
|
8283
|
-
async syncWebsiteSource(kbName, force) {
|
|
8292
|
+
async syncWebsiteSource(kbName, kbId, force) {
|
|
8284
8293
|
const client = await this.getClient();
|
|
8285
8294
|
const response = await client.createWorkflow({
|
|
8286
8295
|
name: "builtin_knowledge_indexing",
|
|
8287
|
-
input: { kbName, force },
|
|
8296
|
+
input: { kbName, kbId, force },
|
|
8288
8297
|
status: "pending",
|
|
8289
8298
|
timeoutAt: new Date(Date.now() + 180 * 60 * 1000).toISOString()
|
|
8290
8299
|
});
|
|
8291
8300
|
return { workflowId: response.workflow.id };
|
|
8292
8301
|
}
|
|
8302
|
+
async detectLegacyWebsiteFiles(kbName, sourceId) {
|
|
8303
|
+
const client = await this.getClient();
|
|
8304
|
+
const legacyTags = {
|
|
8305
|
+
knowledge: "true",
|
|
8306
|
+
sourceId,
|
|
8307
|
+
kbName
|
|
8308
|
+
};
|
|
8309
|
+
const legacyFiles = await this.listExistingFiles(client, legacyTags);
|
|
8310
|
+
return legacyFiles.length;
|
|
8311
|
+
}
|
|
8312
|
+
async deleteLegacyWebsiteFiles(kbName, sourceId) {
|
|
8313
|
+
const client = await this.getClient();
|
|
8314
|
+
const legacyTags = {
|
|
8315
|
+
knowledge: "true",
|
|
8316
|
+
sourceId,
|
|
8317
|
+
kbName
|
|
8318
|
+
};
|
|
8319
|
+
const legacyFiles = await this.listExistingFiles(client, legacyTags);
|
|
8320
|
+
if (legacyFiles.length === 0) {
|
|
8321
|
+
return { deletedFiles: 0 };
|
|
8322
|
+
}
|
|
8323
|
+
console.log(` Found ${legacyFiles.length} legacy website files to migrate`);
|
|
8324
|
+
const results = await Promise.allSettled(legacyFiles.map((f) => client.deleteFile({ id: f.id })));
|
|
8325
|
+
const deletedFiles = results.filter((r) => r.status === "fulfilled").length;
|
|
8326
|
+
console.log(` ✅ Deleted ${deletedFiles} legacy website files`);
|
|
8327
|
+
return { deletedFiles };
|
|
8328
|
+
}
|
|
8293
8329
|
hasWebsiteSources(kbName) {
|
|
8294
8330
|
const kbRef = this.project.knowledge.find((k) => k.definition.name === kbName);
|
|
8295
8331
|
if (!kbRef)
|
|
@@ -8314,8 +8350,8 @@ class KnowledgeManager {
|
|
|
8314
8350
|
const remoteKb = remoteKbs.find((r) => r.name === kb.name);
|
|
8315
8351
|
if (!remoteKb) {
|
|
8316
8352
|
const sources2 = (kbRef.definition.sources || []).map((source) => ({
|
|
8317
|
-
|
|
8318
|
-
|
|
8353
|
+
dsId: source.id,
|
|
8354
|
+
dsType: DataSource.isDirectory(source) ? "document" : "web-page",
|
|
8319
8355
|
needsSync: true,
|
|
8320
8356
|
reason: "New KB"
|
|
8321
8357
|
}));
|
|
@@ -8336,8 +8372,8 @@ class KnowledgeManager {
|
|
|
8336
8372
|
const localHash = await this.computeDirectorySourceHash(source.directoryPath, source.filterFn);
|
|
8337
8373
|
if (!remoteHash) {
|
|
8338
8374
|
sources.push({
|
|
8339
|
-
|
|
8340
|
-
|
|
8375
|
+
dsId: source.id,
|
|
8376
|
+
dsType: "document",
|
|
8341
8377
|
needsSync: true,
|
|
8342
8378
|
reason: "First-time sync"
|
|
8343
8379
|
});
|
|
@@ -8345,8 +8381,8 @@ class KnowledgeManager {
|
|
|
8345
8381
|
} else if (localHash !== remoteHash) {
|
|
8346
8382
|
const fileChanges = await this.detectDirectorySourceChanges(kb.name, source);
|
|
8347
8383
|
sources.push({
|
|
8348
|
-
|
|
8349
|
-
|
|
8384
|
+
dsId: source.id,
|
|
8385
|
+
dsType: "document",
|
|
8350
8386
|
needsSync: true,
|
|
8351
8387
|
reason: "Content changed",
|
|
8352
8388
|
fileChanges
|
|
@@ -8354,8 +8390,8 @@ class KnowledgeManager {
|
|
|
8354
8390
|
hasChanges = true;
|
|
8355
8391
|
} else {
|
|
8356
8392
|
sources.push({
|
|
8357
|
-
|
|
8358
|
-
|
|
8393
|
+
dsId: source.id,
|
|
8394
|
+
dsType: "document",
|
|
8359
8395
|
needsSync: false,
|
|
8360
8396
|
reason: "No changes"
|
|
8361
8397
|
});
|
|
@@ -8365,34 +8401,46 @@ class KnowledgeManager {
|
|
|
8365
8401
|
const localHash = this.computeConfigHash(config);
|
|
8366
8402
|
if (!remoteHash) {
|
|
8367
8403
|
sources.push({
|
|
8368
|
-
|
|
8369
|
-
|
|
8404
|
+
dsId: source.id,
|
|
8405
|
+
dsType: "web-page",
|
|
8370
8406
|
needsSync: true,
|
|
8371
8407
|
reason: "First-time crawl"
|
|
8372
8408
|
});
|
|
8373
8409
|
hasChanges = true;
|
|
8374
8410
|
} else if (localHash !== remoteHash) {
|
|
8375
8411
|
sources.push({
|
|
8376
|
-
|
|
8377
|
-
|
|
8412
|
+
dsId: source.id,
|
|
8413
|
+
dsType: "web-page",
|
|
8378
8414
|
needsSync: true,
|
|
8379
8415
|
reason: "Config changed - needs recrawl"
|
|
8380
8416
|
});
|
|
8381
8417
|
hasChanges = true;
|
|
8382
8418
|
} else {
|
|
8383
|
-
|
|
8384
|
-
|
|
8385
|
-
|
|
8386
|
-
|
|
8387
|
-
|
|
8388
|
-
|
|
8419
|
+
const legacyFileCount = await this.detectLegacyWebsiteFiles(kb.name, source.id);
|
|
8420
|
+
if (legacyFileCount > 0) {
|
|
8421
|
+
sources.push({
|
|
8422
|
+
dsId: source.id,
|
|
8423
|
+
dsType: "web-page",
|
|
8424
|
+
needsSync: true,
|
|
8425
|
+
reason: `${legacyFileCount} legacy files to migrate`,
|
|
8426
|
+
legacyFileCount
|
|
8427
|
+
});
|
|
8428
|
+
hasChanges = true;
|
|
8429
|
+
} else {
|
|
8430
|
+
sources.push({
|
|
8431
|
+
dsId: source.id,
|
|
8432
|
+
dsType: "web-page",
|
|
8433
|
+
needsSync: false,
|
|
8434
|
+
reason: "No config changes"
|
|
8435
|
+
});
|
|
8436
|
+
}
|
|
8389
8437
|
}
|
|
8390
8438
|
}
|
|
8391
8439
|
}
|
|
8392
|
-
const
|
|
8393
|
-
const orphaned = await this.getOrphanedSources(kb.name,
|
|
8440
|
+
const localDsIds = (kbRef.definition.sources || []).map((s) => s.id);
|
|
8441
|
+
const orphaned = await this.getOrphanedSources(kb.name, localDsIds);
|
|
8394
8442
|
const orphanedSourceStatuses = orphaned.map((o) => ({
|
|
8395
|
-
|
|
8443
|
+
dsId: o.dsId,
|
|
8396
8444
|
fileCount: o.fileCount,
|
|
8397
8445
|
willDelete: true
|
|
8398
8446
|
}));
|
|
@@ -8436,11 +8484,17 @@ class KnowledgeManager {
|
|
|
8436
8484
|
const deleted = [];
|
|
8437
8485
|
const modified = [];
|
|
8438
8486
|
const tags = {
|
|
8439
|
-
[WellKnownTags.KNOWLEDGE]: "
|
|
8487
|
+
[WellKnownTags.KNOWLEDGE]: "knowledge-base",
|
|
8440
8488
|
[WellKnownTags.KNOWLEDGE_SOURCE_ID]: source.id,
|
|
8441
8489
|
[WellKnownTags.KNOWLEDGE_BASE_NAME]: kbName
|
|
8442
8490
|
};
|
|
8443
8491
|
const remoteFiles = await this.listExistingFiles(client, tags);
|
|
8492
|
+
const legacyTags = {
|
|
8493
|
+
knowledge: "true",
|
|
8494
|
+
sourceId: source.id,
|
|
8495
|
+
kbName
|
|
8496
|
+
};
|
|
8497
|
+
const legacyFiles = await this.listExistingFiles(client, legacyTags);
|
|
8444
8498
|
const remoteHashes = {};
|
|
8445
8499
|
for (const file of remoteFiles) {
|
|
8446
8500
|
if (isFileMetadata(file.metadata)) {
|
|
@@ -8460,6 +8514,20 @@ class KnowledgeManager {
|
|
|
8460
8514
|
deleted.push(relPath);
|
|
8461
8515
|
}
|
|
8462
8516
|
}
|
|
8517
|
+
if (legacyFiles.length > 0) {
|
|
8518
|
+
for (const file of legacyFiles) {
|
|
8519
|
+
const meta = file.metadata;
|
|
8520
|
+
const relPath = meta?.relPath;
|
|
8521
|
+
if (relPath) {
|
|
8522
|
+
if (!deleted.includes(`${relPath} (legacy)`)) {
|
|
8523
|
+
deleted.push(`${relPath} (legacy)`);
|
|
8524
|
+
}
|
|
8525
|
+
if (localHashes[relPath] && !added.includes(relPath) && !remoteHashes[relPath]) {
|
|
8526
|
+
added.push(relPath);
|
|
8527
|
+
}
|
|
8528
|
+
}
|
|
8529
|
+
}
|
|
8530
|
+
}
|
|
8463
8531
|
return { added, deleted, modified };
|
|
8464
8532
|
}
|
|
8465
8533
|
async scanLocalFileHashes(directoryPath, filterFn) {
|
|
@@ -8507,10 +8575,10 @@ class KnowledgeManager {
|
|
|
8507
8575
|
console.log(` Removing ${item.orphanedSources.length} orphaned source(s)...`);
|
|
8508
8576
|
const deletedSourceIds = [];
|
|
8509
8577
|
for (const orphaned of item.orphanedSources) {
|
|
8510
|
-
const { deletedFiles, errors } = await this.deleteOrphanedSource(item.kb.name, orphaned.
|
|
8511
|
-
console.log(` ✕ ${orphaned.
|
|
8578
|
+
const { deletedFiles, errors } = await this.deleteOrphanedSource(item.kb.name, orphaned.dsId);
|
|
8579
|
+
console.log(` ✕ ${orphaned.dsId}: ${deletedFiles} files deleted`);
|
|
8512
8580
|
if (errors.length === 0) {
|
|
8513
|
-
deletedSourceIds.push(orphaned.
|
|
8581
|
+
deletedSourceIds.push(orphaned.dsId);
|
|
8514
8582
|
} else {
|
|
8515
8583
|
console.warn(` Warning: ${errors.length} errors during deletion`);
|
|
8516
8584
|
}
|
|
@@ -8531,14 +8599,14 @@ class KnowledgeManager {
|
|
|
8531
8599
|
errors: []
|
|
8532
8600
|
};
|
|
8533
8601
|
const sourcesToSync = options.force ? item.sources || [] : (item.sources || []).filter((s) => s.needsSync);
|
|
8534
|
-
const directorySourcesToSync = sourcesToSync.filter((s) => s.
|
|
8535
|
-
const websiteSourcesToSync = sourcesToSync.filter((s) => s.
|
|
8602
|
+
const directorySourcesToSync = sourcesToSync.filter((s) => s.dsType === "document");
|
|
8603
|
+
const websiteSourcesToSync = sourcesToSync.filter((s) => s.dsType === "web-page");
|
|
8536
8604
|
for (const sourceStatus of directorySourcesToSync) {
|
|
8537
|
-
const source = kbRef.definition.sources?.find((s) => s.id === sourceStatus.
|
|
8605
|
+
const source = kbRef.definition.sources?.find((s) => s.id === sourceStatus.dsId);
|
|
8538
8606
|
if (!source || !DataSource.isDirectory(source))
|
|
8539
8607
|
continue;
|
|
8540
8608
|
console.log(` Syncing directory source "${source.id}"...`);
|
|
8541
|
-
const sourceOutput = await this.syncDirectorySource(client, item.kb.name, source.id, source.directoryPath, source.filterFn, options.force || false);
|
|
8609
|
+
const sourceOutput = await this.syncDirectorySource(client, item.kb.name, remoteKb.id, source.id, source.directoryPath, source.filterFn, options.force || false);
|
|
8542
8610
|
syncOutput.processed += sourceOutput.processed;
|
|
8543
8611
|
syncOutput.added.push(...sourceOutput.added);
|
|
8544
8612
|
syncOutput.updated.push(...sourceOutput.updated);
|
|
@@ -8549,11 +8617,14 @@ class KnowledgeManager {
|
|
|
8549
8617
|
}
|
|
8550
8618
|
if (websiteSourcesToSync.length > 0) {
|
|
8551
8619
|
try {
|
|
8620
|
+
for (const sourceStatus of websiteSourcesToSync) {
|
|
8621
|
+
await this.deleteLegacyWebsiteFiles(item.kb.name, sourceStatus.dsId);
|
|
8622
|
+
}
|
|
8552
8623
|
console.log(` Triggering website sync workflow for ${websiteSourcesToSync.length} source(s)...`);
|
|
8553
|
-
const { workflowId } = await this.syncWebsiteSource(item.kb.name, options.force || false);
|
|
8624
|
+
const { workflowId } = await this.syncWebsiteSource(item.kb.name, remoteKb.id, options.force || false);
|
|
8554
8625
|
result.websiteSyncs.push({ kbName: item.kb.name, workflowId });
|
|
8555
8626
|
for (const sourceStatus of websiteSourcesToSync) {
|
|
8556
|
-
const source = kbRef.definition.sources?.find((s) => s.id === sourceStatus.
|
|
8627
|
+
const source = kbRef.definition.sources?.find((s) => s.id === sourceStatus.dsId);
|
|
8557
8628
|
if (!source || !DataSource.isWebsite(source))
|
|
8558
8629
|
continue;
|
|
8559
8630
|
const config = source.getConfig();
|
|
@@ -8574,16 +8645,17 @@ class KnowledgeManager {
|
|
|
8574
8645
|
}
|
|
8575
8646
|
return result;
|
|
8576
8647
|
}
|
|
8577
|
-
async syncDirectorySource(client, kbName,
|
|
8648
|
+
async syncDirectorySource(client, kbName, kbId, dsId, directoryPath, filterFn, force) {
|
|
8578
8649
|
const projectDir = this.project.path;
|
|
8579
8650
|
const directory = path33.resolve(projectDir, directoryPath);
|
|
8580
8651
|
if (!directory.startsWith(projectDir)) {
|
|
8581
8652
|
throw new Error("Directory path must be within the agent's directory");
|
|
8582
8653
|
}
|
|
8583
8654
|
const tags = {
|
|
8584
|
-
[WellKnownTags.KNOWLEDGE]: "
|
|
8585
|
-
[WellKnownTags.
|
|
8586
|
-
[WellKnownTags.
|
|
8655
|
+
[WellKnownTags.KNOWLEDGE]: "knowledge-base",
|
|
8656
|
+
[WellKnownTags.KNOWLEDGE_BASE_ID]: kbId,
|
|
8657
|
+
[WellKnownTags.KNOWLEDGE_SOURCE_ID]: dsId,
|
|
8658
|
+
[WellKnownTags.KNOWLEDGE_SOURCE_TYPE]: "document",
|
|
8587
8659
|
[WellKnownTags.KNOWLEDGE_BASE_NAME]: kbName
|
|
8588
8660
|
};
|
|
8589
8661
|
let allFiles = glob.sync(directory + "/**/*.*", { absolute: true, nodir: true }).filter((file) => {
|
|
@@ -8604,7 +8676,17 @@ class KnowledgeManager {
|
|
|
8604
8676
|
const cachedHashes = await this.scanLocalFileHashes(directoryPath, filterFn);
|
|
8605
8677
|
const existingFiles = await this.listExistingFiles(client, tags);
|
|
8606
8678
|
console.log(` Found ${existingFiles.length} existing files in Botpress`);
|
|
8679
|
+
const legacyTags = {
|
|
8680
|
+
knowledge: "true",
|
|
8681
|
+
sourceId: dsId,
|
|
8682
|
+
kbName
|
|
8683
|
+
};
|
|
8684
|
+
const legacyFiles = await this.listExistingFiles(client, legacyTags);
|
|
8685
|
+
if (legacyFiles.length > 0) {
|
|
8686
|
+
console.log(` Found ${legacyFiles.length} legacy files to migrate`);
|
|
8687
|
+
}
|
|
8607
8688
|
const toRemove = existingFiles.filter((f) => !allFiles.find((af) => af.rel === f.metadata?.relPath));
|
|
8689
|
+
toRemove.push(...legacyFiles);
|
|
8608
8690
|
const toAdd = allFiles.filter((af) => !existingFiles.find((f) => f.metadata?.relPath === af.rel));
|
|
8609
8691
|
const toUpdate = allFiles.filter((af) => existingFiles.find((f) => f.metadata?.relPath === af.rel));
|
|
8610
8692
|
const output = {
|
|
@@ -8631,13 +8713,13 @@ class KnowledgeManager {
|
|
|
8631
8713
|
}
|
|
8632
8714
|
}
|
|
8633
8715
|
for (const local of toAdd) {
|
|
8634
|
-
const result = await this.upsertFile(client, local,
|
|
8716
|
+
const result = await this.upsertFile(client, local, dsId, tags, force, cachedHashes[local.rel]);
|
|
8635
8717
|
if (result) {
|
|
8636
8718
|
output.added.push(result);
|
|
8637
8719
|
}
|
|
8638
8720
|
}
|
|
8639
8721
|
for (const local of toUpdate) {
|
|
8640
|
-
const result = await this.upsertFile(client, local,
|
|
8722
|
+
const result = await this.upsertFile(client, local, dsId, tags, force, cachedHashes[local.rel]);
|
|
8641
8723
|
if (result) {
|
|
8642
8724
|
output.updated.push(result);
|
|
8643
8725
|
}
|
|
@@ -8660,8 +8742,8 @@ class KnowledgeManager {
|
|
|
8660
8742
|
} while (nextToken);
|
|
8661
8743
|
return files;
|
|
8662
8744
|
}
|
|
8663
|
-
async upsertFile(client, local,
|
|
8664
|
-
const key = `data_source://
|
|
8745
|
+
async upsertFile(client, local, dsId, tags, force, cachedHash) {
|
|
8746
|
+
const key = `data_source://document/${dsId}/${local.rel}`;
|
|
8665
8747
|
const content = await fs16.readFile(local.abs);
|
|
8666
8748
|
const hash = cachedHash ?? crypto4.createHash("sha256").update(content).digest("hex");
|
|
8667
8749
|
try {
|
|
@@ -8673,8 +8755,8 @@ class KnowledgeManager {
|
|
|
8673
8755
|
const title = path33.basename(local.name, path33.extname(local.name));
|
|
8674
8756
|
const metadata = {
|
|
8675
8757
|
hash,
|
|
8676
|
-
|
|
8677
|
-
|
|
8758
|
+
dsId,
|
|
8759
|
+
dsType: "document",
|
|
8678
8760
|
relPath: local.rel,
|
|
8679
8761
|
[WellKnownMetadata.TITLE]: title
|
|
8680
8762
|
};
|
|
@@ -8745,8 +8827,8 @@ class KBSyncFormatter {
|
|
|
8745
8827
|
sections.push(" Sources:");
|
|
8746
8828
|
for (const source of item.sources) {
|
|
8747
8829
|
if (source.needsSync) {
|
|
8748
|
-
const typeLabel = source.
|
|
8749
|
-
sections.push(` ${typeLabel} ${source.
|
|
8830
|
+
const typeLabel = source.dsType === "document" ? "\uD83D\uDCC1" : "\uD83C\uDF10";
|
|
8831
|
+
sections.push(` ${typeLabel} ${source.dsId} (${source.dsType})`);
|
|
8750
8832
|
sections.push(` ${source.reason}`);
|
|
8751
8833
|
if (source.fileChanges) {
|
|
8752
8834
|
const { added, modified, deleted } = source.fileChanges;
|
|
@@ -9358,5 +9440,5 @@ export {
|
|
|
9358
9440
|
AgentProject
|
|
9359
9441
|
};
|
|
9360
9442
|
|
|
9361
|
-
//# debugId=
|
|
9443
|
+
//# debugId=9750E26C20B1650B64756E2164756E21
|
|
9362
9444
|
//# sourceMappingURL=index.js.map
|