@sweetoburrito/backstage-plugin-ai-assistant-backend 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/config.d.ts
CHANGED
|
@@ -28,6 +28,22 @@ export interface Config {
|
|
|
28
28
|
};
|
|
29
29
|
ingestion?: {
|
|
30
30
|
schedule?: SchedulerServiceTaskScheduleDefinitionConfig;
|
|
31
|
+
chunking?: {
|
|
32
|
+
/**
|
|
33
|
+
* The size of text chunks to split documents into during ingestion
|
|
34
|
+
*/
|
|
35
|
+
chunkSize?: number;
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The amount of overlap between text chunks during ingestion
|
|
39
|
+
*/
|
|
40
|
+
chunkOverlap?: number;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* The maximum number of chunks to process in a single batch when ingesting documents
|
|
44
|
+
*/
|
|
45
|
+
maxChunkProcessingSize?: number;
|
|
46
|
+
};
|
|
31
47
|
};
|
|
32
48
|
mcp: {
|
|
33
49
|
/**
|
|
@@ -11,6 +11,9 @@ const DEFAULT_DATA_INGESTION_SCHEDULE = {
|
|
|
11
11
|
hours: 3
|
|
12
12
|
}
|
|
13
13
|
};
|
|
14
|
+
const DEFAULT_MAX_CHUNK_PROCESSING_SIZE = 100;
|
|
15
|
+
const DEFAULT_CHUNK_SIZE = 1e3;
|
|
16
|
+
const DEFAULT_CHUNK_OVERLAP = 100;
|
|
14
17
|
const createDataIngestionPipeline = ({
|
|
15
18
|
config,
|
|
16
19
|
logger,
|
|
@@ -21,6 +24,11 @@ const createDataIngestionPipeline = ({
|
|
|
21
24
|
const schedule = config.has("aiAssistant.ingestion.schedule") ? backendPluginApi.readSchedulerServiceTaskScheduleDefinitionFromConfig(
|
|
22
25
|
config.getConfig("aiAssistant.ingestion.schedule")
|
|
23
26
|
) : DEFAULT_DATA_INGESTION_SCHEDULE;
|
|
27
|
+
const chunkSize = config.getOptionalNumber("aiAssistant.ingestion.chunking.chunkSize") ?? DEFAULT_CHUNK_SIZE;
|
|
28
|
+
const chunkOverlap = config.getOptionalNumber("aiAssistant.ingestion.chunking.chunkOverlap") ?? DEFAULT_CHUNK_OVERLAP;
|
|
29
|
+
const maxChunkProcessingSize = config.getOptionalNumber(
|
|
30
|
+
"aiAssistant.ingestion.chunking.maxChunkProcessingSize"
|
|
31
|
+
) ?? DEFAULT_MAX_CHUNK_PROCESSING_SIZE;
|
|
24
32
|
const taskRunner = scheduler.createScheduledTaskRunner(schedule);
|
|
25
33
|
const taskId = `ai-assistant.data-ingestion:start`;
|
|
26
34
|
const dataIngestion = async () => {
|
|
@@ -37,12 +45,10 @@ const createDataIngestionPipeline = ({
|
|
|
37
45
|
`Ingested documents for ${ingestor.id}: ${documents2.length}`
|
|
38
46
|
);
|
|
39
47
|
const splitter = new textsplitters.RecursiveCharacterTextSplitter({
|
|
40
|
-
chunkSize
|
|
41
|
-
|
|
42
|
-
chunkOverlap: 50
|
|
43
|
-
// TODO: Make chunk overlap configurable
|
|
48
|
+
chunkSize,
|
|
49
|
+
chunkOverlap
|
|
44
50
|
});
|
|
45
|
-
const
|
|
51
|
+
const documentChunks = await Promise.all(
|
|
46
52
|
documents2.map(async (document) => {
|
|
47
53
|
logger.debug(
|
|
48
54
|
`Deleting existing documents with id: [${document.metadata.id}] and source: [${ingestor.id}]`
|
|
@@ -51,17 +57,27 @@ const createDataIngestionPipeline = ({
|
|
|
51
57
|
filter: { source: ingestor.id, id: document.metadata.id }
|
|
52
58
|
});
|
|
53
59
|
const chunks = await splitter.splitText(document.content);
|
|
54
|
-
const
|
|
60
|
+
const docChunks = chunks.flatMap(
|
|
55
61
|
(chunk, i) => ({
|
|
56
62
|
metadata: { ...document.metadata, chunk: String(i) },
|
|
57
63
|
content: chunk
|
|
58
64
|
})
|
|
59
65
|
);
|
|
60
|
-
return
|
|
66
|
+
return docChunks;
|
|
61
67
|
})
|
|
62
68
|
);
|
|
63
69
|
logger.info(`Adding documents to vector store...`);
|
|
64
|
-
|
|
70
|
+
const allChunks = documentChunks.flat();
|
|
71
|
+
logger.info(
|
|
72
|
+
`Total document chunks for batch to add for ${ingestor.id}: ${allChunks.length}`
|
|
73
|
+
);
|
|
74
|
+
for (let i = 0; i < allChunks.length; i += maxChunkProcessingSize) {
|
|
75
|
+
const chunkBatch = allChunks.slice(i, i + maxChunkProcessingSize);
|
|
76
|
+
logger.info(
|
|
77
|
+
`Adding batch of ${chunkBatch.length} document chunks to vector store for ${ingestor.id}`
|
|
78
|
+
);
|
|
79
|
+
await vectorStore.addDocuments(chunkBatch);
|
|
80
|
+
}
|
|
65
81
|
logger.info(`Added documents to vector store for ${ingestor.id}`);
|
|
66
82
|
};
|
|
67
83
|
const documents = await ingestor.ingest({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingestor.cjs.js","sources":["../../src/services/ingestor.ts"],"sourcesContent":["import {\n DataIngestionPipeline,\n DataIngestionPipelineOptions,\n EmbeddingDocument,\n} from '@sweetoburrito/backstage-plugin-ai-assistant-node';\n\nimport {\n SchedulerServiceTaskScheduleDefinition,\n readSchedulerServiceTaskScheduleDefinitionFromConfig,\n} from '@backstage/backend-plugin-api';\n\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\n\nconst DEFAULT_DATA_INGESTION_SCHEDULE: SchedulerServiceTaskScheduleDefinition =\n {\n frequency: {\n hours: 24,\n },\n timeout: {\n hours: 3,\n },\n };\n\nexport const createDataIngestionPipeline = ({\n config,\n logger,\n scheduler,\n ingestors,\n vectorStore,\n}: DataIngestionPipelineOptions): DataIngestionPipeline => {\n const schedule = config.has('aiAssistant.ingestion.schedule')\n ? readSchedulerServiceTaskScheduleDefinitionFromConfig(\n config.getConfig('aiAssistant.ingestion.schedule'),\n )\n : DEFAULT_DATA_INGESTION_SCHEDULE;\n\n const taskRunner = scheduler.createScheduledTaskRunner(schedule);\n\n const taskId = `ai-assistant.data-ingestion:start`;\n\n const dataIngestion = async () => {\n logger.info('Starting data ingestion...');\n\n if (ingestors.length === 0) {\n logger.warn('No ingestors available for data ingestion.');\n return;\n }\n\n logger.info(`Ingestors available: ${ingestors.map(i => i.id).join(', ')}`);\n\n for await (const ingestor of ingestors) {\n logger.info(`Running ingestor: ${ingestor.id}`);\n\n const saveDocumentsBatch = async (documents: EmbeddingDocument[]) => {\n logger.info(\n `Ingested documents for ${ingestor.id}: ${documents.length}`,\n );\n\n const splitter = new RecursiveCharacterTextSplitter({\n chunkSize
|
|
1
|
+
{"version":3,"file":"ingestor.cjs.js","sources":["../../src/services/ingestor.ts"],"sourcesContent":["import {\n DataIngestionPipeline,\n DataIngestionPipelineOptions,\n EmbeddingDocument,\n} from '@sweetoburrito/backstage-plugin-ai-assistant-node';\n\nimport {\n SchedulerServiceTaskScheduleDefinition,\n readSchedulerServiceTaskScheduleDefinitionFromConfig,\n} from '@backstage/backend-plugin-api';\n\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\n\nconst DEFAULT_DATA_INGESTION_SCHEDULE: SchedulerServiceTaskScheduleDefinition =\n {\n frequency: {\n hours: 24,\n },\n timeout: {\n hours: 3,\n },\n };\n\nconst DEFAULT_MAX_CHUNK_PROCESSING_SIZE = 100;\nconst DEFAULT_CHUNK_SIZE = 1000;\nconst DEFAULT_CHUNK_OVERLAP = 100;\n\nexport const createDataIngestionPipeline = ({\n config,\n logger,\n scheduler,\n ingestors,\n vectorStore,\n}: DataIngestionPipelineOptions): DataIngestionPipeline => {\n const schedule = config.has('aiAssistant.ingestion.schedule')\n ? readSchedulerServiceTaskScheduleDefinitionFromConfig(\n config.getConfig('aiAssistant.ingestion.schedule'),\n )\n : DEFAULT_DATA_INGESTION_SCHEDULE;\n\n const chunkSize =\n config.getOptionalNumber('aiAssistant.ingestion.chunking.chunkSize') ??\n DEFAULT_CHUNK_SIZE;\n\n const chunkOverlap =\n config.getOptionalNumber('aiAssistant.ingestion.chunking.chunkOverlap') ??\n DEFAULT_CHUNK_OVERLAP;\n\n const maxChunkProcessingSize =\n config.getOptionalNumber(\n 'aiAssistant.ingestion.chunking.maxChunkProcessingSize',\n ) ?? DEFAULT_MAX_CHUNK_PROCESSING_SIZE;\n\n const taskRunner = scheduler.createScheduledTaskRunner(schedule);\n\n const taskId = `ai-assistant.data-ingestion:start`;\n\n const dataIngestion = async () => {\n logger.info('Starting data ingestion...');\n\n if (ingestors.length === 0) {\n logger.warn('No ingestors available for data ingestion.');\n return;\n }\n\n logger.info(`Ingestors available: ${ingestors.map(i => i.id).join(', ')}`);\n\n for await (const ingestor of ingestors) {\n logger.info(`Running ingestor: ${ingestor.id}`);\n\n const saveDocumentsBatch = async (documents: EmbeddingDocument[]) => {\n logger.info(\n `Ingested documents for ${ingestor.id}: ${documents.length}`,\n );\n\n const splitter = new RecursiveCharacterTextSplitter({\n chunkSize,\n chunkOverlap,\n });\n\n const documentChunks = await Promise.all(\n documents.map(async document => {\n // Delete existing documents with this document id and ingestor source\n logger.debug(\n `Deleting existing documents with id: [${document.metadata.id}] and source: [${ingestor.id}]`,\n );\n await vectorStore.deleteDocuments({\n filter: { source: ingestor.id, id: document.metadata.id },\n });\n\n const chunks = await splitter.splitText(document.content);\n\n const docChunks: EmbeddingDocument[] = chunks.flatMap(\n (chunk, i) => ({\n metadata: { ...document.metadata, chunk: String(i) },\n content: chunk,\n }),\n );\n\n return docChunks;\n }),\n );\n\n logger.info(`Adding documents to vector store...`);\n const allChunks = documentChunks.flat();\n\n logger.info(\n `Total document chunks for batch to add for ${ingestor.id}: ${allChunks.length}`,\n );\n\n for (let i = 0; i < allChunks.length; i += maxChunkProcessingSize) {\n const chunkBatch = allChunks.slice(i, i + maxChunkProcessingSize);\n logger.info(\n `Adding batch of ${chunkBatch.length} document chunks to vector store for ${ingestor.id}`,\n );\n\n await vectorStore.addDocuments(chunkBatch);\n }\n\n logger.info(`Added documents to vector store for ${ingestor.id}`);\n };\n\n const documents = await ingestor.ingest({\n saveDocumentsBatch,\n });\n\n if (documents) {\n saveDocumentsBatch(documents);\n }\n\n logger.info(`Finished processing ingestor: ${ingestor.id}`);\n }\n\n logger.info('Data ingestion completed.');\n };\n\n const start = async () => {\n taskRunner.run({\n id: taskId,\n fn: dataIngestion,\n });\n };\n\n return {\n start,\n };\n};\n"],"names":["readSchedulerServiceTaskScheduleDefinitionFromConfig","documents","RecursiveCharacterTextSplitter"],"mappings":";;;;;AAaA,MAAM,+BAAA,GACJ;AAAA,EACE,SAAA,EAAW;AAAA,IACT,KAAA,EAAO;AAAA,GACT;AAAA,EACA,OAAA,EAAS;AAAA,IACP,KAAA,EAAO;AAAA;AAEX,CAAA;AAEF,MAAM,iCAAA,GAAoC,GAAA;AAC1C,MAAM,kBAAA,GAAqB,GAAA;AAC3B,MAAM,qBAAA,GAAwB,GAAA;AAEvB,MAAM,8BAA8B,CAAC;AAAA,EAC1C,MAAA;AAAA,EACA,MAAA;AAAA,EACA,SAAA;AAAA,EACA,SAAA;AAAA,EACA;AACF,CAAA,KAA2D;AACzD,EAAA,MAAM,QAAA,GAAW,MAAA,CAAO,GAAA,CAAI,gCAAgC,CAAA,GACxDA,qEAAA;AAAA,IACE,MAAA,CAAO,UAAU,gCAAgC;AAAA,GACnD,GACA,+BAAA;AAEJ,EAAA,MAAM,SAAA,GACJ,MAAA,CAAO,iBAAA,CAAkB,0CAA0C,CAAA,IACnE,kBAAA;AAEF,EAAA,MAAM,YAAA,GACJ,MAAA,CAAO,iBAAA,CAAkB,6CAA6C,CAAA,IACtE,qBAAA;AAEF,EAAA,MAAM,yBACJ,MAAA,CAAO,iBAAA;AAAA,IACL;AAAA,GACF,IAAK,iCAAA;AAEP,EAAA,MAAM,UAAA,GAAa,SAAA,CAAU,yBAAA,CAA0B,QAAQ,CAAA;AAE/D,EAAA,MAAM,MAAA,GAAS,CAAA,iCAAA,CAAA;AAEf,EAAA,MAAM,gBAAgB,YAAY;AAChC,IAAA,MAAA,CAAO,KAAK,4BAA4B,CAAA;AAExC,IAAA,IAAI,SAAA,CAAU,WAAW,CAAA,EAAG;AAC1B,MAAA,MAAA,CAAO,KAAK,4CAA4C,CAAA;AACxD,MAAA;AAAA,IACF;AAEA,IAAA,MAAA,CAAO,IAAA,CAAK,CAAA,qBAAA,EAAwB,SAAA,CAAU,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,EAAE,CAAA,CAAE,IAAA,CAAK,IAAI,CAAC,CAAA,CAAE,CAAA;AAEzE,IAAA,WAAA,MAAiB,YAAY,SAAA,EAAW;AACtC,MAAA,MAAA,CAAO,IAAA,CAAK,CAAA,kBAAA,EAAqB,QAAA,CAAS,EAAE,CAAA,CAAE,CAAA;AAE9C,MAAA,MAAM,kBAAA,GAAqB,OAAOC,UAAAA,KAAmC;AACnE,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,uBAAA,EAA0B,QAAA,CAAS,EAAE,CAAA,EAAA,EAAKA,WAAU,MAAM,CAAA;AAAA,SAC5D;AAEA,QAAA,MAAM,QAAA,GAAW,IAAIC,4CAAA,CAA+B;AAAA,UAClD,SAAA;AAAA,UACA;AAAA,SACD,CAAA;AAED,QAAA,MAAM,cAAA,GAAiB,MAAM,OAAA,CAAQ,GAAA;AAAA,UACnCD,UAAAA,CAAU,GAAA,CAAI,OAAM,QAAA,KAAY;AAE9B,YAAA,MAAA,CAAO,KAAA;AAAA,cACL,yCAAyC,QAAA,CAAS,QAAA,CAAS,EAAE,CAAA,eAAA,EAAkB,SAAS,EAAE,CAAA,CAAA;AAAA,aAC5F;AACA,YAAA,MAAM,YAAY,eAAA,CAAgB;AAAA,cAChC,MAAA,EAAQ,EAAE,MAAA,EAAQ,QAAA,CAAS,IAAI,EAAA,EAAI,QAAA,CAAS,SAAS,EAAA;AAAG,aACzD,CAAA;AAED,YAAA,MAAM,MAAA,GAAS,MAAM,QAAA,CAAS,SAAA,CAAU,SAAS,OAAO,CAAA;AAExD,YAAA,MAAM,YAAiC,MAAA,CAAO,OAAA;AAAA,cAC5C,CAAC,OAAO,CAAA,MAAO;AAAA,gBACb,QAAA,EAAU,EAAE,GAAG,QAAA,CAAS,UAAU,KAAA,EAAO,MAAA,CAAO,CAAC,CAAA,EAAE;AAAA,gBACnD,OAAA,EAAS;AAAA,eACX;AAAA,aACF;AAEA,YAAA,OAAO,SAAA;AAAA,UACT,CAAC;AAAA,SACH;AAEA,QAAA,MAAA,CAAO,KAAK,CAAA,mCAAA,CAAqC,CAAA;AACjD,QAAA,MAAM,SAAA,GAAY,eAAe,IAAA,EAAK;AAEtC,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,2CAAA,EAA8C,QAAA,CAAS,EAAE,CAAA,EAAA,EAAK,UAAU,MAAM,CAAA;AAAA,SAChF;AAEA,QAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,SAAA,CAAU,MAAA,EAAQ,KAAK,sBAAA,EAAwB;AACjE,UAAA,MAAM,UAAA,GAAa,SAAA,CAAU,KAAA,CAAM,CAAA,EAAG,IAAI,sBAAsB,CAAA;AAChE,UAAA,MAAA,CAAO,IAAA;AAAA,YACL,CAAA,gBAAA,EAAmB,UAAA,CAAW,MAAM,CAAA,qCAAA,EAAwC,SAAS,EAAE,CAAA;AAAA,WACzF;AAEA,UAAA,MAAM,WAAA,CAAY,aAAa,UAAU,CAAA;AAAA,QAC3C;AAEA,QAAA,MAAA,CAAO,IAAA,CAAK,CAAA,oCAAA,EAAuC,QAAA,CAAS,EAAE,CAAA,CAAE,CAAA;AAAA,MAClE,CAAA;AAEA,MAAA,MAAM,SAAA,GAAY,MAAM,QAAA,CAAS,MAAA,CAAO;AAAA,QACtC;AAAA,OACD,CAAA;AAED,MAAA,IAAI,SAAA,EAAW;AACb,QAAA,kBAAA,CAAmB,SAAS,CAAA;AAAA,MAC9B;AAEA,MAAA,MAAA,CAAO,IAAA,CAAK,CAAA,8BAAA,EAAiC,QAAA,CAAS,EAAE,CAAA,CAAE,CAAA;AAAA,IAC5D;AAEA,IAAA,MAAA,CAAO,KAAK,2BAA2B,CAAA;AAAA,EACzC,CAAA;AAEA,EAAA,MAAM,QAAQ,YAAY;AACxB,IAAA,UAAA,CAAW,GAAA,CAAI;AAAA,MACb,EAAA,EAAI,MAAA;AAAA,MACJ,EAAA,EAAI;AAAA,KACL,CAAA;AAAA,EACH,CAAA;AAEA,EAAA,OAAO;AAAA,IACL;AAAA,GACF;AACF;;;;"}
|