@sweetoburrito/backstage-plugin-ai-assistant-backend-module-ingestor-catalog 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/module.cjs.js
CHANGED
|
@@ -12,7 +12,8 @@ const aiAssistantModuleIngestorCatalog = backendPluginApi.createBackendModule({
|
|
|
12
12
|
deps: {
|
|
13
13
|
dataIngestor: backstagePluginAiAssistantNode.dataIngestorExtensionPoint,
|
|
14
14
|
auth: backendPluginApi.coreServices.auth,
|
|
15
|
-
discovery: backendPluginApi.coreServices.discovery
|
|
15
|
+
discovery: backendPluginApi.coreServices.discovery,
|
|
16
|
+
logger: backendPluginApi.coreServices.logger
|
|
16
17
|
},
|
|
17
18
|
async init(options) {
|
|
18
19
|
const { dataIngestor } = options;
|
package/dist/module.cjs.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"module.cjs.js","sources":["../src/module.ts"],"sourcesContent":["import {\n coreServices,\n createBackendModule,\n} from '@backstage/backend-plugin-api';\nimport { dataIngestorExtensionPoint } from '@sweetoburrito/backstage-plugin-ai-assistant-node';\nimport { createCatalogIngestor } from './services/ingestor';\n\nexport const aiAssistantModuleIngestorCatalog = createBackendModule({\n pluginId: 'ai-assistant',\n moduleId: 'ingestor-catalog',\n register(reg) {\n reg.registerInit({\n deps: {\n dataIngestor: dataIngestorExtensionPoint,\n auth: coreServices.auth,\n discovery: coreServices.discovery,\n },\n async init(options) {\n const { dataIngestor } = options;\n dataIngestor.registerIngestor(await createCatalogIngestor(options));\n },\n });\n },\n});\n"],"names":["createBackendModule","dataIngestorExtensionPoint","coreServices","createCatalogIngestor"],"mappings":";;;;;;AAOO,MAAM,mCAAmCA,oCAAA,CAAoB;AAAA,EAClE,QAAA,EAAU,cAAA;AAAA,EACV,QAAA,EAAU,kBAAA;AAAA,EACV,SAAS,GAAA,EAAK;AACZ,IAAA,GAAA,CAAI,YAAA,CAAa;AAAA,MACf,IAAA,EAAM;AAAA,QACJ,YAAA,EAAcC,yDAAA;AAAA,QACd,MAAMC,6BAAA,CAAa,IAAA;AAAA,QACnB,WAAWA,6BAAA,CAAa;AAAA,
|
|
1
|
+
{"version":3,"file":"module.cjs.js","sources":["../src/module.ts"],"sourcesContent":["import {\n coreServices,\n createBackendModule,\n} from '@backstage/backend-plugin-api';\nimport { dataIngestorExtensionPoint } from '@sweetoburrito/backstage-plugin-ai-assistant-node';\nimport { createCatalogIngestor } from './services/ingestor';\n\nexport const aiAssistantModuleIngestorCatalog = createBackendModule({\n pluginId: 'ai-assistant',\n moduleId: 'ingestor-catalog',\n register(reg) {\n reg.registerInit({\n deps: {\n dataIngestor: dataIngestorExtensionPoint,\n auth: coreServices.auth,\n discovery: coreServices.discovery,\n logger: coreServices.logger,\n },\n async init(options) {\n const { dataIngestor } = options;\n dataIngestor.registerIngestor(await createCatalogIngestor(options));\n },\n });\n },\n});\n"],"names":["createBackendModule","dataIngestorExtensionPoint","coreServices","createCatalogIngestor"],"mappings":";;;;;;AAOO,MAAM,mCAAmCA,oCAAA,CAAoB;AAAA,EAClE,QAAA,EAAU,cAAA;AAAA,EACV,QAAA,EAAU,kBAAA;AAAA,EACV,SAAS,GAAA,EAAK;AACZ,IAAA,GAAA,CAAI,YAAA,CAAa;AAAA,MACf,IAAA,EAAM;AAAA,QACJ,YAAA,EAAcC,yDAAA;AAAA,QACd,MAAMC,6BAAA,CAAa,IAAA;AAAA,QACnB,WAAWA,6BAAA,CAAa,SAAA;AAAA,QACxB,QAAQA,6BAAA,CAAa;AAAA,OACvB;AAAA,MACA,MAAM,KAAK,OAAA,EAAS;AAClB,QAAA,MAAM,EAAE,cAAa,GAAI,OAAA;AACzB,QAAA,YAAA,CAAa,gBAAA,CAAiB,MAAMC,2BAAA,CAAsB,OAAO,CAAC,CAAA;AAAA,MACpE;AAAA,KACD,CAAA;AAAA,EACH;AACF,CAAC;;;;"}
|
|
@@ -2,11 +2,18 @@
|
|
|
2
2
|
|
|
3
3
|
var entityToEmbeddingDoc = require('./helpers/entity-to-embedding-doc.cjs.js');
|
|
4
4
|
var module$1 = require('../../constants/module.cjs.js');
|
|
5
|
+
var fetchRetry = require('fetch-retry');
|
|
5
6
|
|
|
6
|
-
|
|
7
|
+
function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
|
|
8
|
+
|
|
9
|
+
var fetchRetry__default = /*#__PURE__*/_interopDefaultCompat(fetchRetry);
|
|
10
|
+
|
|
11
|
+
const fetch = fetchRetry__default.default(global.fetch);
|
|
12
|
+
const MAX_RESULTS = 25;
|
|
7
13
|
const createCatalogIngestor = async ({
|
|
8
14
|
auth,
|
|
9
|
-
discovery
|
|
15
|
+
discovery,
|
|
16
|
+
logger
|
|
10
17
|
}) => {
|
|
11
18
|
const ingestCatalogBatch = async (saveDocumentsBatch, cursor) => {
|
|
12
19
|
const baseUrl = await discovery.getBaseUrl("catalog");
|
|
@@ -20,15 +27,28 @@ const createCatalogIngestor = async ({
|
|
|
20
27
|
params.set("cursor", cursor);
|
|
21
28
|
}
|
|
22
29
|
params.set("limit", MAX_RESULTS.toString());
|
|
30
|
+
logger.debug(
|
|
31
|
+
`Fetching catalog entities batch, cursor: ${cursor || "initial"}`
|
|
32
|
+
);
|
|
23
33
|
const response = await fetch(
|
|
24
34
|
`${baseUrl}/entities/by-query?${params.toString()}`,
|
|
25
35
|
{
|
|
26
36
|
headers: {
|
|
27
37
|
Authorization: `Bearer ${token}`
|
|
38
|
+
},
|
|
39
|
+
retries: 3,
|
|
40
|
+
retryDelay: (attempt) => {
|
|
41
|
+
logger.debug(`Retrying catalog fetch, attempt: ${attempt}`);
|
|
42
|
+
const exponentialDelay = Math.pow(2, attempt) * 500;
|
|
43
|
+
const jitter = Math.random() * 500;
|
|
44
|
+
return exponentialDelay + jitter;
|
|
28
45
|
}
|
|
29
46
|
}
|
|
30
47
|
);
|
|
31
48
|
if (!response.ok) {
|
|
49
|
+
logger.error(
|
|
50
|
+
`Failed to fetch catalog entities: ${response.status} ${response.statusText}`
|
|
51
|
+
);
|
|
32
52
|
return;
|
|
33
53
|
}
|
|
34
54
|
const data = await response.json();
|
|
@@ -39,6 +59,7 @@ const createCatalogIngestor = async ({
|
|
|
39
59
|
);
|
|
40
60
|
await saveDocumentsBatch(documents);
|
|
41
61
|
if (!nextCursor) {
|
|
62
|
+
logger.info("Catalog ingestion completed - no more pages");
|
|
42
63
|
return;
|
|
43
64
|
}
|
|
44
65
|
await ingestCatalogBatch(saveDocumentsBatch, nextCursor);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.cjs.js","sources":["../../../src/services/ingestor/index.ts"],"sourcesContent":["import {\n EmbeddingDocument,\n Ingestor,\n IngestorOptions,\n} from '@sweetoburrito/backstage-plugin-ai-assistant-node';\nimport {
|
|
1
|
+
{"version":3,"file":"index.cjs.js","sources":["../../../src/services/ingestor/index.ts"],"sourcesContent":["import {\n EmbeddingDocument,\n Ingestor,\n IngestorOptions,\n} from '@sweetoburrito/backstage-plugin-ai-assistant-node';\nimport {\n AuthService,\n DiscoveryService,\n LoggerService,\n} from '@backstage/backend-plugin-api';\nimport { Entity } from '@backstage/catalog-model';\nimport { mapEntityToEmbeddingDoc } from './helpers/entity-to-embedding-doc';\nimport { MODULE_ID } from '../../constants/module';\nimport fetchRetry from 'fetch-retry';\n\nconst fetch = fetchRetry(global.fetch);\n\ntype CatalogIngestorOptions = {\n auth: AuthService;\n discovery: DiscoveryService;\n logger: LoggerService;\n};\n\nconst MAX_RESULTS = 25;\n\nexport const createCatalogIngestor = async ({\n auth,\n discovery,\n logger,\n}: CatalogIngestorOptions): Promise<Ingestor> => {\n const ingestCatalogBatch = async (\n saveDocumentsBatch: IngestorOptions['saveDocumentsBatch'],\n cursor?: string,\n ) => {\n const baseUrl = await discovery.getBaseUrl('catalog');\n\n const credentials = await auth.getOwnServiceCredentials();\n const { token } = await auth.getPluginRequestToken({\n onBehalfOf: credentials,\n targetPluginId: 'catalog',\n });\n\n const params = new URLSearchParams();\n if (cursor) {\n params.set('cursor', cursor);\n }\n\n params.set('limit', MAX_RESULTS.toString());\n\n logger.debug(\n `Fetching catalog entities batch, cursor: ${cursor || 'initial'}`,\n );\n\n const response = await fetch(\n `${baseUrl}/entities/by-query?${params.toString()}`,\n {\n headers: {\n Authorization: `Bearer ${token}`,\n },\n retries: 3,\n retryDelay: attempt => {\n // Adds exponential backoff with jitter for retries\n\n logger.debug(`Retrying catalog fetch, attempt: ${attempt}`);\n\n const exponentialDelay = Math.pow(2, attempt) * 500;\n const jitter = Math.random() * 500;\n return exponentialDelay + jitter;\n },\n },\n );\n\n if (!response.ok) {\n logger.error(\n `Failed to fetch catalog entities: ${response.status} ${response.statusText}`,\n );\n return;\n }\n\n const data = await response.json();\n\n const entities = data.items as Entity[];\n const nextCursor = data.pageInfo?.nextCursor;\n\n const documents: EmbeddingDocument[] = entities.map(\n mapEntityToEmbeddingDoc,\n );\n\n await saveDocumentsBatch(documents);\n\n if (!nextCursor) {\n logger.info('Catalog ingestion completed - no more pages');\n return;\n }\n\n await ingestCatalogBatch(saveDocumentsBatch, nextCursor);\n };\n\n const ingest: Ingestor['ingest'] = async ({ saveDocumentsBatch }) => {\n await ingestCatalogBatch(saveDocumentsBatch);\n };\n\n return {\n id: MODULE_ID,\n ingest,\n };\n};\n"],"names":["fetchRetry","mapEntityToEmbeddingDoc","MODULE_ID"],"mappings":";;;;;;;;;;AAeA,MAAM,KAAA,GAAQA,2BAAA,CAAW,MAAA,CAAO,KAAK,CAAA;AAQrC,MAAM,WAAA,GAAc,EAAA;AAEb,MAAM,wBAAwB,OAAO;AAAA,EAC1C,IAAA;AAAA,EACA,SAAA;AAAA,EACA;AACF,CAAA,KAAiD;AAC/C,EAAA,MAAM,kBAAA,GAAqB,OACzB,kBAAA,EACA,MAAA,KACG;AACH,IAAA,MAAM,OAAA,GAAU,MAAM,SAAA,CAAU,UAAA,CAAW,SAAS,CAAA;AAEpD,IAAA,MAAM,WAAA,GAAc,MAAM,IAAA,CAAK,wBAAA,EAAyB;AACxD,IAAA,MAAM,EAAE,KAAA,EAAM,GAAI,MAAM,KAAK,qBAAA,CAAsB;AAAA,MACjD,UAAA,EAAY,WAAA;AAAA,MACZ,cAAA,EAAgB;AAAA,KACjB,CAAA;AAED,IAAA,MAAM,MAAA,GAAS,IAAI,eAAA,EAAgB;AACnC,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,MAAA,CAAO,GAAA,CAAI,UAAU,MAAM,CAAA;AAAA,IAC7B;AAEA,IAAA,MAAA,CAAO,GAAA,CAAI,OAAA,EAAS,WAAA,CAAY,QAAA,EAAU,CAAA;AAE1C,IAAA,MAAA,CAAO,KAAA;AAAA,MACL,CAAA,yCAAA,EAA4C,UAAU,SAAS,CAAA;AAAA,KACjE;AAEA,IAAA,MAAM,WAAW,MAAM,KAAA;AAAA,MACrB,CAAA,EAAG,OAAO,CAAA,mBAAA,EAAsB,MAAA,CAAO,UAAU,CAAA,CAAA;AAAA,MACjD;AAAA,QACE,OAAA,EAAS;AAAA,UACP,aAAA,EAAe,UAAU,KAAK,CAAA;AAAA,SAChC;AAAA,QACA,OAAA,EAAS,CAAA;AAAA,QACT,YAAY,CAAA,OAAA,KAAW;AAGrB,UAAA,MAAA,CAAO,KAAA,CAAM,CAAA,iCAAA,EAAoC,OAAO,CAAA,CAAE,CAAA;AAE1D,UAAA,MAAM,gBAAA,GAAmB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,OAAO,CAAA,GAAI,GAAA;AAChD,UAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,EAAO,GAAI,GAAA;AAC/B,UAAA,OAAO,gBAAA,GAAmB,MAAA;AAAA,QAC5B;AAAA;AACF,KACF;AAEA,IAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,CAAA,kCAAA,EAAqC,QAAA,CAAS,MAAM,CAAA,CAAA,EAAI,SAAS,UAAU,CAAA;AAAA,OAC7E;AACA,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,IAAA,GAAO,MAAM,QAAA,CAAS,IAAA,EAAK;AAEjC,IAAA,MAAM,WAAW,IAAA,CAAK,KAAA;AACtB,IAAA,MAAM,UAAA,GAAa,KAAK,QAAA,EAAU,UAAA;AAElC,IAAA,MAAM,YAAiC,QAAA,CAAS,GAAA;AAAA,MAC9CC;AAAA,KACF;AAEA,IAAA,MAAM,mBAAmB,SAAS,CAAA;AAElC,IAAA,IAAI,CAAC,UAAA,EAAY;AACf,MAAA,MAAA,CAAO,KAAK,6CAA6C,CAAA;AACzD,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,kBAAA,CAAmB,oBAAoB,UAAU,CAAA;AAAA,EACzD,CAAA;AAEA,EAAA,MAAM,MAAA,GAA6B,OAAO,EAAE,kBAAA,EAAmB,KAAM;AACnE,IAAA,MAAM,mBAAmB,kBAAkB,CAAA;AAAA,EAC7C,CAAA;AAEA,EAAA,OAAO;AAAA,IACL,EAAA,EAAIC,kBAAA;AAAA,IACJ;AAAA,GACF;AACF;;;;"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sweetoburrito/backstage-plugin-ai-assistant-backend-module-ingestor-catalog",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"description": "The ingestor-catalog backend module for the ai-assistant plugin.",
|
|
6
6
|
"main": "dist/index.cjs.js",
|
|
@@ -31,7 +31,8 @@
|
|
|
31
31
|
"@backstage/backend-plugin-api": "backstage:^",
|
|
32
32
|
"@backstage/catalog-model": "backstage:^",
|
|
33
33
|
"@backstage/plugin-catalog-node": "backstage:^",
|
|
34
|
-
"@sweetoburrito/backstage-plugin-ai-assistant-node": "^0.7.0"
|
|
34
|
+
"@sweetoburrito/backstage-plugin-ai-assistant-node": "^0.7.0",
|
|
35
|
+
"fetch-retry": "^6.0.0"
|
|
35
36
|
},
|
|
36
37
|
"devDependencies": {
|
|
37
38
|
"@backstage/backend-defaults": "backstage:^",
|