@sweetoburrito/backstage-plugin-ai-assistant-backend 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -32,7 +32,6 @@ const createDataIngestionPipeline = ({
|
|
|
32
32
|
logger.info(`Ingestors available: ${ingestors.map((i) => i.id).join(", ")}`);
|
|
33
33
|
for await (const ingestor of ingestors) {
|
|
34
34
|
logger.info(`Running ingestor: ${ingestor.id}`);
|
|
35
|
-
await vectorStore.deleteDocuments({ filter: { source: ingestor.id } });
|
|
36
35
|
const saveDocumentsBatch = async (documents2) => {
|
|
37
36
|
logger.info(
|
|
38
37
|
`Ingested documents for ${ingestor.id}: ${documents2.length}`
|
|
@@ -45,6 +44,12 @@ const createDataIngestionPipeline = ({
|
|
|
45
44
|
});
|
|
46
45
|
const docs = await Promise.all(
|
|
47
46
|
documents2.map(async (document) => {
|
|
47
|
+
logger.debug(
|
|
48
|
+
`Deleting existing documents with id: [${document.metadata.id}] and source: [${ingestor.id}]`
|
|
49
|
+
);
|
|
50
|
+
await vectorStore.deleteDocuments({
|
|
51
|
+
filter: { source: ingestor.id, id: document.metadata.id }
|
|
52
|
+
});
|
|
48
53
|
const chunks = await splitter.splitText(document.content);
|
|
49
54
|
const chunkDocs = chunks.flatMap(
|
|
50
55
|
(chunk, i) => ({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingestor.cjs.js","sources":["../../src/services/ingestor.ts"],"sourcesContent":["import {\n DataIngestionPipeline,\n DataIngestionPipelineOptions,\n EmbeddingDocument,\n} from '@sweetoburrito/backstage-plugin-ai-assistant-node';\n\nimport {\n SchedulerServiceTaskScheduleDefinition,\n readSchedulerServiceTaskScheduleDefinitionFromConfig,\n} from '@backstage/backend-plugin-api';\n\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\n\nconst DEFAULT_DATA_INGESTION_SCHEDULE: SchedulerServiceTaskScheduleDefinition =\n {\n frequency: {\n hours: 24,\n },\n timeout: {\n hours: 3,\n },\n };\n\nexport const createDataIngestionPipeline = ({\n config,\n logger,\n scheduler,\n ingestors,\n vectorStore,\n}: DataIngestionPipelineOptions): DataIngestionPipeline => {\n const schedule = config.has('aiAssistant.ingestion.schedule')\n ? readSchedulerServiceTaskScheduleDefinitionFromConfig(\n config.getConfig('aiAssistant.ingestion.schedule'),\n )\n : DEFAULT_DATA_INGESTION_SCHEDULE;\n\n const taskRunner = scheduler.createScheduledTaskRunner(schedule);\n\n const taskId = `ai-assistant.data-ingestion:start`;\n\n const dataIngestion = async () => {\n logger.info('Starting data ingestion...');\n\n if (ingestors.length === 0) {\n logger.warn('No ingestors available for data ingestion.');\n return;\n }\n\n logger.info(`Ingestors available: ${ingestors.map(i => i.id).join(', ')}`);\n\n for await (const ingestor of ingestors) {\n logger.info(`Running ingestor: ${ingestor.id}`);\n\n
|
|
1
|
+
{"version":3,"file":"ingestor.cjs.js","sources":["../../src/services/ingestor.ts"],"sourcesContent":["import {\n DataIngestionPipeline,\n DataIngestionPipelineOptions,\n EmbeddingDocument,\n} from '@sweetoburrito/backstage-plugin-ai-assistant-node';\n\nimport {\n SchedulerServiceTaskScheduleDefinition,\n readSchedulerServiceTaskScheduleDefinitionFromConfig,\n} from '@backstage/backend-plugin-api';\n\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\n\nconst DEFAULT_DATA_INGESTION_SCHEDULE: SchedulerServiceTaskScheduleDefinition =\n {\n frequency: {\n hours: 24,\n },\n timeout: {\n hours: 3,\n },\n };\n\nexport const createDataIngestionPipeline = ({\n config,\n logger,\n scheduler,\n ingestors,\n vectorStore,\n}: DataIngestionPipelineOptions): DataIngestionPipeline => {\n const schedule = config.has('aiAssistant.ingestion.schedule')\n ? readSchedulerServiceTaskScheduleDefinitionFromConfig(\n config.getConfig('aiAssistant.ingestion.schedule'),\n )\n : DEFAULT_DATA_INGESTION_SCHEDULE;\n\n const taskRunner = scheduler.createScheduledTaskRunner(schedule);\n\n const taskId = `ai-assistant.data-ingestion:start`;\n\n const dataIngestion = async () => {\n logger.info('Starting data ingestion...');\n\n if (ingestors.length === 0) {\n logger.warn('No ingestors available for data ingestion.');\n return;\n }\n\n logger.info(`Ingestors available: ${ingestors.map(i => i.id).join(', ')}`);\n\n for await (const ingestor of ingestors) {\n logger.info(`Running ingestor: ${ingestor.id}`);\n\n const saveDocumentsBatch = async (documents: EmbeddingDocument[]) => {\n logger.info(\n `Ingested documents for ${ingestor.id}: ${documents.length}`,\n );\n\n const splitter = new RecursiveCharacterTextSplitter({\n chunkSize: 500, // TODO: Make chunk size configurable\n chunkOverlap: 50, // TODO: Make chunk overlap configurable\n });\n\n const docs = await Promise.all(\n documents.map(async document => {\n // Delete existing documents with this document id and ingestor source\n logger.debug(\n `Deleting existing documents with id: [${document.metadata.id}] and source: [${ingestor.id}]`,\n );\n await vectorStore.deleteDocuments({\n filter: { source: ingestor.id, id: document.metadata.id },\n });\n\n const chunks = await splitter.splitText(document.content);\n\n const chunkDocs: EmbeddingDocument[] = chunks.flatMap(\n (chunk, i) => ({\n metadata: { ...document.metadata, chunk: String(i) },\n content: chunk,\n }),\n );\n\n return chunkDocs;\n }),\n );\n\n logger.info(`Adding documents to vector store...`);\n await vectorStore.addDocuments(docs.flat());\n logger.info(`Added documents to vector store for ${ingestor.id}`);\n };\n\n const documents = await ingestor.ingest({\n saveDocumentsBatch,\n });\n\n if (documents) {\n saveDocumentsBatch(documents);\n }\n\n logger.info(`Finished processing ingestor: ${ingestor.id}`);\n }\n\n logger.info('Data ingestion completed.');\n };\n\n const start = async () => {\n taskRunner.run({\n id: taskId,\n fn: dataIngestion,\n });\n };\n\n return {\n start,\n };\n};\n"],"names":["readSchedulerServiceTaskScheduleDefinitionFromConfig","documents","RecursiveCharacterTextSplitter"],"mappings":";;;;;AAaA,MAAM,+BAAA,GACJ;AAAA,EACE,SAAA,EAAW;AAAA,IACT,KAAA,EAAO;AAAA,GACT;AAAA,EACA,OAAA,EAAS;AAAA,IACP,KAAA,EAAO;AAAA;AAEX,CAAA;AAEK,MAAM,8BAA8B,CAAC;AAAA,EAC1C,MAAA;AAAA,EACA,MAAA;AAAA,EACA,SAAA;AAAA,EACA,SAAA;AAAA,EACA;AACF,CAAA,KAA2D;AACzD,EAAA,MAAM,QAAA,GAAW,MAAA,CAAO,GAAA,CAAI,gCAAgC,CAAA,GACxDA,qEAAA;AAAA,IACE,MAAA,CAAO,UAAU,gCAAgC;AAAA,GACnD,GACA,+BAAA;AAEJ,EAAA,MAAM,UAAA,GAAa,SAAA,CAAU,yBAAA,CAA0B,QAAQ,CAAA;AAE/D,EAAA,MAAM,MAAA,GAAS,CAAA,iCAAA,CAAA;AAEf,EAAA,MAAM,gBAAgB,YAAY;AAChC,IAAA,MAAA,CAAO,KAAK,4BAA4B,CAAA;AAExC,IAAA,IAAI,SAAA,CAAU,WAAW,CAAA,EAAG;AAC1B,MAAA,MAAA,CAAO,KAAK,4CAA4C,CAAA;AACxD,MAAA;AAAA,IACF;AAEA,IAAA,MAAA,CAAO,IAAA,CAAK,CAAA,qBAAA,EAAwB,SAAA,CAAU,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,EAAE,CAAA,CAAE,IAAA,CAAK,IAAI,CAAC,CAAA,CAAE,CAAA;AAEzE,IAAA,WAAA,MAAiB,YAAY,SAAA,EAAW;AACtC,MAAA,MAAA,CAAO,IAAA,CAAK,CAAA,kBAAA,EAAqB,QAAA,CAAS,EAAE,CAAA,CAAE,CAAA;AAE9C,MAAA,MAAM,kBAAA,GAAqB,OAAOC,UAAAA,KAAmC;AACnE,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,uBAAA,EAA0B,QAAA,CAAS,EAAE,CAAA,EAAA,EAAKA,WAAU,MAAM,CAAA;AAAA,SAC5D;AAEA,QAAA,MAAM,QAAA,GAAW,IAAIC,4CAAA,CAA+B;AAAA,UAClD,SAAA,EAAW,GAAA;AAAA;AAAA,UACX,YAAA,EAAc;AAAA;AAAA,SACf,CAAA;AAED,QAAA,MAAM,IAAA,GAAO,MAAM,OAAA,CAAQ,GAAA;AAAA,UACzBD,UAAAA,CAAU,GAAA,CAAI,OAAM,QAAA,KAAY;AAE9B,YAAA,MAAA,CAAO,KAAA;AAAA,cACL,yCAAyC,QAAA,CAAS,QAAA,CAAS,EAAE,CAAA,eAAA,EAAkB,SAAS,EAAE,CAAA,CAAA;AAAA,aAC5F;AACA,YAAA,MAAM,YAAY,eAAA,CAAgB;AAAA,cAChC,MAAA,EAAQ,EAAE,MAAA,EAAQ,QAAA,CAAS,IAAI,EAAA,EAAI,QAAA,CAAS,SAAS,EAAA;AAAG,aACzD,CAAA;AAED,YAAA,MAAM,MAAA,GAAS,MAAM,QAAA,CAAS,SAAA,CAAU,SAAS,OAAO,CAAA;AAExD,YAAA,MAAM,YAAiC,MAAA,CAAO,OAAA;AAAA,cAC5C,CAAC,OAAO,CAAA,MAAO;AAAA,gBACb,QAAA,EAAU,EAAE,GAAG,QAAA,CAAS,UAAU,KAAA,EAAO,MAAA,CAAO,CAAC,CAAA,EAAE;AAAA,gBACnD,OAAA,EAAS;AAAA,eACX;AAAA,aACF;AAEA,YAAA,OAAO,SAAA;AAAA,UACT,CAAC;AAAA,SACH;AAEA,QAAA,MAAA,CAAO,KAAK,CAAA,mCAAA,CAAqC,CAAA;AACjD,QAAA,MAAM,WAAA,CAAY,YAAA,CAAa,IAAA,CAAK,IAAA,EAAM,CAAA;AAC1C,QAAA,MAAA,CAAO,IAAA,CAAK,CAAA,oCAAA,EAAuC,QAAA,CAAS,EAAE,CAAA,CAAE,CAAA;AAAA,MAClE,CAAA;AAEA,MAAA,MAAM,SAAA,GAAY,MAAM,QAAA,CAAS,MAAA,CAAO;AAAA,QACtC;AAAA,OACD,CAAA;AAED,MAAA,IAAI,SAAA,EAAW;AACb,QAAA,kBAAA,CAAmB,SAAS,CAAA;AAAA,MAC9B;AAEA,MAAA,MAAA,CAAO,IAAA,CAAK,CAAA,8BAAA,EAAiC,QAAA,CAAS,EAAE,CAAA,CAAE,CAAA;AAAA,IAC5D;AAEA,IAAA,MAAA,CAAO,KAAK,2BAA2B,CAAA;AAAA,EACzC,CAAA;AAEA,EAAA,MAAM,QAAQ,YAAY;AACxB,IAAA,UAAA,CAAW,GAAA,CAAI;AAAA,MACb,EAAA,EAAI,MAAA;AAAA,MACJ,EAAA,EAAI;AAAA,KACL,CAAA;AAAA,EACH,CAAA;AAEA,EAAA,OAAO;AAAA,IACL;AAAA,GACF;AACF;;;;"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sweetoburrito/backstage-plugin-ai-assistant-backend",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.1",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"main": "dist/index.cjs.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
"@langchain/langgraph": "^0.4.9",
|
|
42
42
|
"@langchain/textsplitters": "^0.1.0",
|
|
43
43
|
"@sweetoburrito/backstage-plugin-ai-assistant-common": "^0.5.0",
|
|
44
|
-
"@sweetoburrito/backstage-plugin-ai-assistant-node": "^0.5.
|
|
44
|
+
"@sweetoburrito/backstage-plugin-ai-assistant-node": "^0.5.1",
|
|
45
45
|
"express": "^4.17.1",
|
|
46
46
|
"express-promise-router": "^4.1.0",
|
|
47
47
|
"knex": "^3.1.0",
|