@sweetoburrito/backstage-plugin-ai-assistant-backend-module-ingestor-catalog 0.3.6 → 0.4.1

package/dist/module.cjs.js

@@ -12,7 +12,8 @@ const aiAssistantModuleIngestorCatalog = backendPluginApi.createBackendModule({
       deps: {
         dataIngestor: backstagePluginAiAssistantNode.dataIngestorExtensionPoint,
         auth: backendPluginApi.coreServices.auth,
-        discovery: backendPluginApi.coreServices.discovery
+        discovery: backendPluginApi.coreServices.discovery,
+        logger: backendPluginApi.coreServices.logger
       },
       async init(options) {
         const { dataIngestor } = options;

package/dist/module.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"module.cjs.js","sources":["../src/module.ts"],"sourcesContent":["import {\n  coreServices,\n  createBackendModule,\n} from '@backstage/backend-plugin-api';\nimport { dataIngestorExtensionPoint } from '@sweetoburrito/backstage-plugin-ai-assistant-node';\nimport { createCatalogIngestor } from './services/ingestor';\n\nexport const aiAssistantModuleIngestorCatalog = createBackendModule({\n  pluginId: 'ai-assistant',\n  moduleId: 'ingestor-catalog',\n  register(reg) {\n    reg.registerInit({\n      deps: {\n        dataIngestor: dataIngestorExtensionPoint,\n        auth: coreServices.auth,\n        discovery: coreServices.discovery,\n      },\n      async init(options) {\n        const { dataIngestor } = options;\n        dataIngestor.registerIngestor(await createCatalogIngestor(options));\n      },\n    });\n  },\n});\n"],"names":["createBackendModule","dataIngestorExtensionPoint","coreServices","createCatalogIngestor"],"mappings":";;;;;;AAOO,MAAM,mCAAmCA,oCAAA,CAAoB;AAAA,EAClE,QAAA,EAAU,cAAA;AAAA,EACV,QAAA,EAAU,kBAAA;AAAA,EACV,SAAS,GAAA,EAAK;AACZ,IAAA,GAAA,CAAI,YAAA,CAAa;AAAA,MACf,IAAA,EAAM;AAAA,QACJ,YAAA,EAAcC,yDAAA;AAAA,QACd,MAAMC,6BAAA,CAAa,IAAA;AAAA,QACnB,WAAWA,6BAAA,CAAa;AAAA,OAC1B;AAAA,MACA,MAAM,KAAK,OAAA,EAAS;AAClB,QAAA,MAAM,EAAE,cAAa,GAAI,OAAA;AACzB,QAAA,YAAA,CAAa,gBAAA,CAAiB,MAAMC,2BAAA,CAAsB,OAAO,CAAC,CAAA;AAAA,MACpE;AAAA,KACD,CAAA;AAAA,EACH;AACF,CAAC;;;;"}
+{"version":3,"file":"module.cjs.js","sources":["../src/module.ts"],"sourcesContent":["import {\n  coreServices,\n  createBackendModule,\n} from '@backstage/backend-plugin-api';\nimport { dataIngestorExtensionPoint } from '@sweetoburrito/backstage-plugin-ai-assistant-node';\nimport { createCatalogIngestor } from './services/ingestor';\n\nexport const aiAssistantModuleIngestorCatalog = createBackendModule({\n  pluginId: 'ai-assistant',\n  moduleId: 'ingestor-catalog',\n  register(reg) {\n    reg.registerInit({\n      deps: {\n        dataIngestor: dataIngestorExtensionPoint,\n        auth: coreServices.auth,\n        discovery: coreServices.discovery,\n        logger: coreServices.logger,\n      },\n      async init(options) {\n        const { dataIngestor } = options;\n        dataIngestor.registerIngestor(await createCatalogIngestor(options));\n      },\n    });\n  },\n});\n"],"names":["createBackendModule","dataIngestorExtensionPoint","coreServices","createCatalogIngestor"],"mappings":";;;;;;AAOO,MAAM,mCAAmCA,oCAAA,CAAoB;AAAA,EAClE,QAAA,EAAU,cAAA;AAAA,EACV,QAAA,EAAU,kBAAA;AAAA,EACV,SAAS,GAAA,EAAK;AACZ,IAAA,GAAA,CAAI,YAAA,CAAa;AAAA,MACf,IAAA,EAAM;AAAA,QACJ,YAAA,EAAcC,yDAAA;AAAA,QACd,MAAMC,6BAAA,CAAa,IAAA;AAAA,QACnB,WAAWA,6BAAA,CAAa,SAAA;AAAA,QACxB,QAAQA,6BAAA,CAAa;AAAA,OACvB;AAAA,MACA,MAAM,KAAK,OAAA,EAAS;AAClB,QAAA,MAAM,EAAE,cAAa,GAAI,OAAA;AACzB,QAAA,YAAA,CAAa,gBAAA,CAAiB,MAAMC,2BAAA,CAAsB,OAAO,CAAC,CAAA;AAAA,MACpE;AAAA,KACD,CAAA;AAAA,EACH;AACF,CAAC;;;;"}

package/dist/services/ingestor/helpers/entity-to-embedding-doc.cjs.js

@@ -42,7 +42,8 @@ function mapEntityToEmbeddingDoc(entity) {
       kind: entity.kind,
       namespace: ns,
       name: entity.metadata.name,
-      uid: entity.metadata.uid
+      uid: entity.metadata.uid,
+      url: `/catalog/${entity.metadata.namespace || "default"}/${entity.kind.toLowerCase()}/${entity.metadata.name}`
     },
     content: parts.join("\n")
   };

package/dist/services/ingestor/helpers/entity-to-embedding-doc.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"entity-to-embedding-doc.cjs.js","sources":["../../../../src/services/ingestor/helpers/entity-to-embedding-doc.ts"],"sourcesContent":["import { Entity } from '@backstage/catalog-model';\nimport { EmbeddingDocument } from '@sweetoburrito/backstage-plugin-ai-assistant-node';\nimport { MODULE_ID } from '../../../constants/module';\n\nexport function mapEntityToEmbeddingDoc(entity: Entity): EmbeddingDocument {\n  const ns = entity.metadata.namespace || 'default';\n  const id = `${entity.kind.toLowerCase()}:${ns}/${entity.metadata.name}`;\n\n  // Build content string (natural language summary)\n  const parts: string[] = [];\n\n  // Title & description\n  if (entity.metadata.title) {\n    parts.push(\n      `${entity.kind} \"${entity.metadata.title}\" (${entity.metadata.name})`,\n    );\n  } else {\n    parts.push(`${entity.kind} \"${entity.metadata.name}\"`);\n  }\n\n  if (entity.metadata.description) {\n    parts.push(`Description: ${entity.metadata.description}.`);\n  }\n\n  // Spec (flatten key fields if present)\n  if (entity.spec) {\n    for (const [key, value] of Object.entries(entity.spec)) {\n      if (typeof value === 'string') {\n        parts.push(`${key}: ${value}`);\n      } else if (Array.isArray(value)) {\n        parts.push(`${key}: ${value.join(', ')}`);\n      } else if (typeof value === 'object' && value !== null) {\n        for (const [k, v] of Object.entries(value)) {\n          if (typeof v === 'string') {\n            parts.push(`${k}: ${v}`);\n          }\n        }\n      }\n    }\n  }\n\n  // Relations (express in plain text)\n  if (entity.relations?.length) {\n    const relationText = entity.relations\n      .map(r => `${r.type} → ${r.targetRef}`)\n      .join('; ');\n    parts.push(`Relations: ${relationText}`);\n  }\n\n  return {\n    metadata: {\n      source: MODULE_ID,\n      id,\n      kind: entity.kind,\n      namespace: ns,\n      name: entity.metadata.name,\n      uid: entity.metadata.uid!,\n    },\n    content: parts.join('\\n'),\n  };\n}\n"],"names":["MODULE_ID"],"mappings":";;;;AAIO,SAAS,wBAAwB,MAAA,EAAmC;AACzE,EAAA,MAAM,EAAA,GAAK,MAAA,CAAO,QAAA,CAAS,SAAA,IAAa,SAAA;AACxC,EAAA,MAAM,EAAA,GAAK,CAAA,EAAG,MAAA,CAAO,IAAA,CAAK,WAAA,EAAa,CAAA,CAAA,EAAI,EAAE,CAAA,CAAA,EAAI,MAAA,CAAO,QAAA,CAAS,IAAI,CAAA,CAAA;AAGrE,EAAA,MAAM,QAAkB,EAAC;AAGzB,EAAA,IAAI,MAAA,CAAO,SAAS,KAAA,EAAO;AACzB,IAAA,KAAA,CAAM,IAAA;AAAA,MACJ,CAAA,EAAG,MAAA,CAAO,IAAI,CAAA,EAAA,EAAK,MAAA,CAAO,SAAS,KAAK,CAAA,GAAA,EAAM,MAAA,CAAO,QAAA,CAAS,IAAI,CAAA,CAAA;AAAA,KACpE;AAAA,EACF,CAAA,MAAO;AACL,IAAA,KAAA,CAAM,IAAA,CAAK,GAAG,MAAA,CAAO,IAAI,KAAK,MAAA,CAAO,QAAA,CAAS,IAAI,CAAA,CAAA,CAAG,CAAA;AAAA,EACvD;AAEA,EAAA,IAAI,MAAA,CAAO,SAAS,WAAA,EAAa;AAC/B,IAAA,KAAA,CAAM,IAAA,CAAK,CAAA,aAAA,EAAgB,MAAA,CAAO,QAAA,CAAS,WAAW,CAAA,CAAA,CAAG,CAAA;AAAA,EAC3D;AAGA,EAAA,IAAI,OAAO,IAAA,EAAM;AACf,IAAA,KAAA,MAAW,CAAC,KAAK,KAAK,CAAA,IAAK,OAAO,OAAA,CAAQ,MAAA,CAAO,IAAI,CAAA,EAAG;AACtD,MAAA,IAAI,OAAO,UAAU,QAAA,EAAU;AAC7B,QAAA,KAAA,CAAM,IAAA,CAAK,CAAA,EAAG,GAAG,CAAA,EAAA,EAAK,KAAK,CAAA,CAAE,CAAA;AAAA,MAC/B,CAAA,MAAA,IAAW,KAAA,CAAM,OAAA,CAAQ,KAAK,CAAA,EAAG;AAC/B,QAAA,KAAA,CAAM,IAAA,CAAK,GAAG,GAAG,CAAA,EAAA,EAAK,MAAM,IAAA,CAAK,IAAI,CAAC,CAAA,CAAE,CAAA;AAAA,MAC1C,CAAA,MAAA,IAAW,OAAO,KAAA,KAAU,QAAA,IAAY,UAAU,IAAA,EAAM;AACtD,QAAA,KAAA,MAAW,CAAC,CAAA,EAAG,CAAC,KAAK,MAAA,CAAO,OAAA,CAAQ,KAAK,CAAA,EAAG;AAC1C,UAAA,IAAI,OAAO,MAAM,QAAA,EAAU;AACzB,YAAA,KAAA,CAAM,IAAA,CAAK,CAAA,EAAG,CAAC,CAAA,EAAA,EAAK,CAAC,CAAA,CAAE,CAAA;AAAA,UACzB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,EAAA,IAAI,MAAA,CAAO,WAAW,MAAA,EAAQ;AAC5B,IAAA,MAAM,YAAA,GAAe,MAAA,CAAO,SAAA,CACzB,GAAA,CAAI,OAAK,CAAA,EAAG,CAAA,CAAE,IAAI,CAAA,QAAA,EAAM,CAAA,CAAE,SAAS,CAAA,CAAE,CAAA,CACrC,KAAK,IAAI,CAAA;AACZ,IAAA,KAAA,CAAM,IAAA,CAAK,CAAA,WAAA,EAAc,YAAY,CAAA,CAAE,CAAA;AAAA,EACzC;AAEA,EAAA,OAAO;AAAA,IACL,QAAA,EAAU;AAAA,MACR,MAAA,EAAQA,kBAAA;AAAA,MACR,EAAA;AAAA,MACA,MAAM,MAAA,CAAO,IAAA;AAAA,MACb,SAAA,EAAW,EAAA;AAAA,MACX,IAAA,EAAM,OAAO,QAAA,CAAS,IAAA;AAAA,MACtB,GAAA,EAAK,OAAO,QAAA,CAAS;AAAA,KACvB;AAAA,IACA,OAAA,EAAS,KAAA,CAAM,IAAA,CAAK,IAAI;AAAA,GAC1B;AACF;;;;"}
+{"version":3,"file":"entity-to-embedding-doc.cjs.js","sources":["../../../../src/services/ingestor/helpers/entity-to-embedding-doc.ts"],"sourcesContent":["import { Entity } from '@backstage/catalog-model';\nimport { EmbeddingDocument } from '@sweetoburrito/backstage-plugin-ai-assistant-node';\nimport { MODULE_ID } from '../../../constants/module';\n\nexport function mapEntityToEmbeddingDoc(entity: Entity): EmbeddingDocument {\n  const ns = entity.metadata.namespace || 'default';\n  const id = `${entity.kind.toLowerCase()}:${ns}/${entity.metadata.name}`;\n\n  // Build content string (natural language summary)\n  const parts: string[] = [];\n\n  // Title & description\n  if (entity.metadata.title) {\n    parts.push(\n      `${entity.kind} \"${entity.metadata.title}\" (${entity.metadata.name})`,\n    );\n  } else {\n    parts.push(`${entity.kind} \"${entity.metadata.name}\"`);\n  }\n\n  if (entity.metadata.description) {\n    parts.push(`Description: ${entity.metadata.description}.`);\n  }\n\n  // Spec (flatten key fields if present)\n  if (entity.spec) {\n    for (const [key, value] of Object.entries(entity.spec)) {\n      if (typeof value === 'string') {\n        parts.push(`${key}: ${value}`);\n      } else if (Array.isArray(value)) {\n        parts.push(`${key}: ${value.join(', ')}`);\n      } else if (typeof value === 'object' && value !== null) {\n        for (const [k, v] of Object.entries(value)) {\n          if (typeof v === 'string') {\n            parts.push(`${k}: ${v}`);\n          }\n        }\n      }\n    }\n  }\n\n  // Relations (express in plain text)\n  if (entity.relations?.length) {\n    const relationText = entity.relations\n      .map(r => `${r.type} → ${r.targetRef}`)\n      .join('; ');\n    parts.push(`Relations: ${relationText}`);\n  }\n\n  return {\n    metadata: {\n      source: MODULE_ID,\n      id,\n      kind: entity.kind,\n      namespace: ns,\n      name: entity.metadata.name,\n      uid: entity.metadata.uid!,\n      url: `/catalog/${\n        entity.metadata.namespace || 'default'\n      }/${entity.kind.toLowerCase()}/${entity.metadata.name}`,\n    },\n    content: parts.join('\\n'),\n  };\n}\n"],"names":["MODULE_ID"],"mappings":";;;;AAIO,SAAS,wBAAwB,MAAA,EAAmC;AACzE,EAAA,MAAM,EAAA,GAAK,MAAA,CAAO,QAAA,CAAS,SAAA,IAAa,SAAA;AACxC,EAAA,MAAM,EAAA,GAAK,CAAA,EAAG,MAAA,CAAO,IAAA,CAAK,WAAA,EAAa,CAAA,CAAA,EAAI,EAAE,CAAA,CAAA,EAAI,MAAA,CAAO,QAAA,CAAS,IAAI,CAAA,CAAA;AAGrE,EAAA,MAAM,QAAkB,EAAC;AAGzB,EAAA,IAAI,MAAA,CAAO,SAAS,KAAA,EAAO;AACzB,IAAA,KAAA,CAAM,IAAA;AAAA,MACJ,CAAA,EAAG,MAAA,CAAO,IAAI,CAAA,EAAA,EAAK,MAAA,CAAO,SAAS,KAAK,CAAA,GAAA,EAAM,MAAA,CAAO,QAAA,CAAS,IAAI,CAAA,CAAA;AAAA,KACpE;AAAA,EACF,CAAA,MAAO;AACL,IAAA,KAAA,CAAM,IAAA,CAAK,GAAG,MAAA,CAAO,IAAI,KAAK,MAAA,CAAO,QAAA,CAAS,IAAI,CAAA,CAAA,CAAG,CAAA;AAAA,EACvD;AAEA,EAAA,IAAI,MAAA,CAAO,SAAS,WAAA,EAAa;AAC/B,IAAA,KAAA,CAAM,IAAA,CAAK,CAAA,aAAA,EAAgB,MAAA,CAAO,QAAA,CAAS,WAAW,CAAA,CAAA,CAAG,CAAA;AAAA,EAC3D;AAGA,EAAA,IAAI,OAAO,IAAA,EAAM;AACf,IAAA,KAAA,MAAW,CAAC,KAAK,KAAK,CAAA,IAAK,OAAO,OAAA,CAAQ,MAAA,CAAO,IAAI,CAAA,EAAG;AACtD,MAAA,IAAI,OAAO,UAAU,QAAA,EAAU;AAC7B,QAAA,KAAA,CAAM,IAAA,CAAK,CAAA,EAAG,GAAG,CAAA,EAAA,EAAK,KAAK,CAAA,CAAE,CAAA;AAAA,MAC/B,CAAA,MAAA,IAAW,KAAA,CAAM,OAAA,CAAQ,KAAK,CAAA,EAAG;AAC/B,QAAA,KAAA,CAAM,IAAA,CAAK,GAAG,GAAG,CAAA,EAAA,EAAK,MAAM,IAAA,CAAK,IAAI,CAAC,CAAA,CAAE,CAAA;AAAA,MAC1C,CAAA,MAAA,IAAW,OAAO,KAAA,KAAU,QAAA,IAAY,UAAU,IAAA,EAAM;AACtD,QAAA,KAAA,MAAW,CAAC,CAAA,EAAG,CAAC,KAAK,MAAA,CAAO,OAAA,CAAQ,KAAK,CAAA,EAAG;AAC1C,UAAA,IAAI,OAAO,MAAM,QAAA,EAAU;AACzB,YAAA,KAAA,CAAM,IAAA,CAAK,CAAA,EAAG,CAAC,CAAA,EAAA,EAAK,CAAC,CAAA,CAAE,CAAA;AAAA,UACzB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,EAAA,IAAI,MAAA,CAAO,WAAW,MAAA,EAAQ;AAC5B,IAAA,MAAM,YAAA,GAAe,MAAA,CAAO,SAAA,CACzB,GAAA,CAAI,OAAK,CAAA,EAAG,CAAA,CAAE,IAAI,CAAA,QAAA,EAAM,CAAA,CAAE,SAAS,CAAA,CAAE,CAAA,CACrC,KAAK,IAAI,CAAA;AACZ,IAAA,KAAA,CAAM,IAAA,CAAK,CAAA,WAAA,EAAc,YAAY,CAAA,CAAE,CAAA;AAAA,EACzC;AAEA,EAAA,OAAO;AAAA,IACL,QAAA,EAAU;AAAA,MACR,MAAA,EAAQA,kBAAA;AAAA,MACR,EAAA;AAAA,MACA,MAAM,MAAA,CAAO,IAAA;AAAA,MACb,SAAA,EAAW,EAAA;AAAA,MACX,IAAA,EAAM,OAAO,QAAA,CAAS,IAAA;AAAA,MACtB,GAAA,EAAK,OAAO,QAAA,CAAS,GAAA;AAAA,MACrB,GAAA,EAAK,CAAA,SAAA,EACH,MAAA,CAAO,QAAA,CAAS,aAAa,SAC/B,CAAA,CAAA,EAAI,MAAA,CAAO,IAAA,CAAK,WAAA,EAAa,CAAA,CAAA,EAAI,MAAA,CAAO,SAAS,IAAI,CAAA;AAAA,KACvD;AAAA,IACA,OAAA,EAAS,KAAA,CAAM,IAAA,CAAK,IAAI;AAAA,GAC1B;AACF;;;;"}

package/dist/services/ingestor/index.cjs.js

@@ -2,11 +2,18 @@
 
 var entityToEmbeddingDoc = require('./helpers/entity-to-embedding-doc.cjs.js');
 var module$1 = require('../../constants/module.cjs.js');
+var fetchRetry = require('fetch-retry');
 
-const MAX_RESULTS = 50;
+function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
+
+var fetchRetry__default = /*#__PURE__*/_interopDefaultCompat(fetchRetry);
+
+const fetch = fetchRetry__default.default(global.fetch);
+const MAX_RESULTS = 25;
 const createCatalogIngestor = async ({
   auth,
-  discovery
+  discovery,
+  logger
 }) => {
   const ingestCatalogBatch = async (saveDocumentsBatch, cursor) => {
     const baseUrl = await discovery.getBaseUrl("catalog");
@@ -20,15 +27,28 @@ const createCatalogIngestor = async ({
       params.set("cursor", cursor);
     }
     params.set("limit", MAX_RESULTS.toString());
+    logger.debug(
+      `Fetching catalog entities batch, cursor: ${cursor || "initial"}`
+    );
     const response = await fetch(
       `${baseUrl}/entities/by-query?${params.toString()}`,
       {
         headers: {
           Authorization: `Bearer ${token}`
+        },
+        retries: 3,
+        retryDelay: (attempt) => {
+          logger.debug(`Retrying catalog fetch, attempt: ${attempt}`);
+          const exponentialDelay = Math.pow(2, attempt) * 500;
+          const jitter = Math.random() * 500;
+          return exponentialDelay + jitter;
         }
       }
     );
     if (!response.ok) {
+      logger.error(
+        `Failed to fetch catalog entities: ${response.status} ${response.statusText}`
+      );
       return;
     }
     const data = await response.json();
@@ -39,6 +59,7 @@ const createCatalogIngestor = async ({
     );
     await saveDocumentsBatch(documents);
     if (!nextCursor) {
+      logger.info("Catalog ingestion completed - no more pages");
       return;
     }
     await ingestCatalogBatch(saveDocumentsBatch, nextCursor);

package/dist/services/ingestor/index.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.cjs.js","sources":["../../../src/services/ingestor/index.ts"],"sourcesContent":["import {\n  EmbeddingDocument,\n  Ingestor,\n  IngestorOptions,\n} from '@sweetoburrito/backstage-plugin-ai-assistant-node';\nimport { AuthService, DiscoveryService } from '@backstage/backend-plugin-api';\nimport { Entity } from '@backstage/catalog-model';\nimport { mapEntityToEmbeddingDoc } from './helpers/entity-to-embedding-doc';\nimport { MODULE_ID } from '../../constants/module';\n\ntype CatalogIngestorOptions = {\n  auth: AuthService;\n  discovery: DiscoveryService;\n};\n\nconst MAX_RESULTS = 50;\n\nexport const createCatalogIngestor = async ({\n  auth,\n  discovery,\n}: CatalogIngestorOptions): Promise<Ingestor> => {\n  const ingestCatalogBatch = async (\n    saveDocumentsBatch: IngestorOptions['saveDocumentsBatch'],\n    cursor?: string,\n  ) => {\n    const baseUrl = await discovery.getBaseUrl('catalog');\n\n    const credentials = await auth.getOwnServiceCredentials();\n    const { token } = await auth.getPluginRequestToken({\n      onBehalfOf: credentials,\n      targetPluginId: 'catalog',\n    });\n\n    const params = new URLSearchParams();\n    if (cursor) {\n      params.set('cursor', cursor);\n    }\n\n    params.set('limit', MAX_RESULTS.toString());\n\n    const response = await fetch(\n      `${baseUrl}/entities/by-query?${params.toString()}`,\n      {\n        headers: {\n          Authorization: `Bearer ${token}`,\n        },\n      },\n    );\n\n    if (!response.ok) {\n      return;\n    }\n\n    const data = await response.json();\n\n    const entities = data.items as Entity[];\n    const nextCursor = data.pageInfo?.nextCursor;\n\n    const documents: EmbeddingDocument[] = entities.map(\n      mapEntityToEmbeddingDoc,\n    );\n\n    await saveDocumentsBatch(documents);\n\n    if (!nextCursor) {\n      return;\n    }\n\n    await ingestCatalogBatch(saveDocumentsBatch, nextCursor);\n  };\n\n  const ingest: Ingestor['ingest'] = async ({ saveDocumentsBatch }) => {\n    await ingestCatalogBatch(saveDocumentsBatch);\n  };\n\n  return {\n    id: MODULE_ID,\n    ingest,\n  };\n};\n"],"names":["mapEntityToEmbeddingDoc","MODULE_ID"],"mappings":";;;;;AAeA,MAAM,WAAA,GAAc,EAAA;AAEb,MAAM,wBAAwB,OAAO;AAAA,EAC1C,IAAA;AAAA,EACA;AACF,CAAA,KAAiD;AAC/C,EAAA,MAAM,kBAAA,GAAqB,OACzB,kBAAA,EACA,MAAA,KACG;AACH,IAAA,MAAM,OAAA,GAAU,MAAM,SAAA,CAAU,UAAA,CAAW,SAAS,CAAA;AAEpD,IAAA,MAAM,WAAA,GAAc,MAAM,IAAA,CAAK,wBAAA,EAAyB;AACxD,IAAA,MAAM,EAAE,KAAA,EAAM,GAAI,MAAM,KAAK,qBAAA,CAAsB;AAAA,MACjD,UAAA,EAAY,WAAA;AAAA,MACZ,cAAA,EAAgB;AAAA,KACjB,CAAA;AAED,IAAA,MAAM,MAAA,GAAS,IAAI,eAAA,EAAgB;AACnC,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,MAAA,CAAO,GAAA,CAAI,UAAU,MAAM,CAAA;AAAA,IAC7B;AAEA,IAAA,MAAA,CAAO,GAAA,CAAI,OAAA,EAAS,WAAA,CAAY,QAAA,EAAU,CAAA;AAE1C,IAAA,MAAM,WAAW,MAAM,KAAA;AAAA,MACrB,CAAA,EAAG,OAAO,CAAA,mBAAA,EAAsB,MAAA,CAAO,UAAU,CAAA,CAAA;AAAA,MACjD;AAAA,QACE,OAAA,EAAS;AAAA,UACP,aAAA,EAAe,UAAU,KAAK,CAAA;AAAA;AAChC;AACF,KACF;AAEA,IAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,IAAA,GAAO,MAAM,QAAA,CAAS,IAAA,EAAK;AAEjC,IAAA,MAAM,WAAW,IAAA,CAAK,KAAA;AACtB,IAAA,MAAM,UAAA,GAAa,KAAK,QAAA,EAAU,UAAA;AAElC,IAAA,MAAM,YAAiC,QAAA,CAAS,GAAA;AAAA,MAC9CA;AAAA,KACF;AAEA,IAAA,MAAM,mBAAmB,SAAS,CAAA;AAElC,IAAA,IAAI,CAAC,UAAA,EAAY;AACf,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,kBAAA,CAAmB,oBAAoB,UAAU,CAAA;AAAA,EACzD,CAAA;AAEA,EAAA,MAAM,MAAA,GAA6B,OAAO,EAAE,kBAAA,EAAmB,KAAM;AACnE,IAAA,MAAM,mBAAmB,kBAAkB,CAAA;AAAA,EAC7C,CAAA;AAEA,EAAA,OAAO;AAAA,IACL,EAAA,EAAIC,kBAAA;AAAA,IACJ;AAAA,GACF;AACF;;;;"}
+{"version":3,"file":"index.cjs.js","sources":["../../../src/services/ingestor/index.ts"],"sourcesContent":["import {\n  EmbeddingDocument,\n  Ingestor,\n  IngestorOptions,\n} from '@sweetoburrito/backstage-plugin-ai-assistant-node';\nimport {\n  AuthService,\n  DiscoveryService,\n  LoggerService,\n} from '@backstage/backend-plugin-api';\nimport { Entity } from '@backstage/catalog-model';\nimport { mapEntityToEmbeddingDoc } from './helpers/entity-to-embedding-doc';\nimport { MODULE_ID } from '../../constants/module';\nimport fetchRetry from 'fetch-retry';\n\nconst fetch = fetchRetry(global.fetch);\n\ntype CatalogIngestorOptions = {\n  auth: AuthService;\n  discovery: DiscoveryService;\n  logger: LoggerService;\n};\n\nconst MAX_RESULTS = 25;\n\nexport const createCatalogIngestor = async ({\n  auth,\n  discovery,\n  logger,\n}: CatalogIngestorOptions): Promise<Ingestor> => {\n  const ingestCatalogBatch = async (\n    saveDocumentsBatch: IngestorOptions['saveDocumentsBatch'],\n    cursor?: string,\n  ) => {\n    const baseUrl = await discovery.getBaseUrl('catalog');\n\n    const credentials = await auth.getOwnServiceCredentials();\n    const { token } = await auth.getPluginRequestToken({\n      onBehalfOf: credentials,\n      targetPluginId: 'catalog',\n    });\n\n    const params = new URLSearchParams();\n    if (cursor) {\n      params.set('cursor', cursor);\n    }\n\n    params.set('limit', MAX_RESULTS.toString());\n\n    logger.debug(\n      `Fetching catalog entities batch, cursor: ${cursor || 'initial'}`,\n    );\n\n    const response = await fetch(\n      `${baseUrl}/entities/by-query?${params.toString()}`,\n      {\n        headers: {\n          Authorization: `Bearer ${token}`,\n        },\n        retries: 3,\n        retryDelay: attempt => {\n          // Adds exponential backoff with jitter for retries\n\n          logger.debug(`Retrying catalog fetch, attempt: ${attempt}`);\n\n          const exponentialDelay = Math.pow(2, attempt) * 500;\n          const jitter = Math.random() * 500;\n          return exponentialDelay + jitter;\n        },\n      },\n    );\n\n    if (!response.ok) {\n      logger.error(\n        `Failed to fetch catalog entities: ${response.status} ${response.statusText}`,\n      );\n      return;\n    }\n\n    const data = await response.json();\n\n    const entities = data.items as Entity[];\n    const nextCursor = data.pageInfo?.nextCursor;\n\n    const documents: EmbeddingDocument[] = entities.map(\n      mapEntityToEmbeddingDoc,\n    );\n\n    await saveDocumentsBatch(documents);\n\n    if (!nextCursor) {\n      logger.info('Catalog ingestion completed - no more pages');\n      return;\n    }\n\n    await ingestCatalogBatch(saveDocumentsBatch, nextCursor);\n  };\n\n  const ingest: Ingestor['ingest'] = async ({ saveDocumentsBatch }) => {\n    await ingestCatalogBatch(saveDocumentsBatch);\n  };\n\n  return {\n    id: MODULE_ID,\n    ingest,\n  };\n};\n"],"names":["fetchRetry","mapEntityToEmbeddingDoc","MODULE_ID"],"mappings":";;;;;;;;;;AAeA,MAAM,KAAA,GAAQA,2BAAA,CAAW,MAAA,CAAO,KAAK,CAAA;AAQrC,MAAM,WAAA,GAAc,EAAA;AAEb,MAAM,wBAAwB,OAAO;AAAA,EAC1C,IAAA;AAAA,EACA,SAAA;AAAA,EACA;AACF,CAAA,KAAiD;AAC/C,EAAA,MAAM,kBAAA,GAAqB,OACzB,kBAAA,EACA,MAAA,KACG;AACH,IAAA,MAAM,OAAA,GAAU,MAAM,SAAA,CAAU,UAAA,CAAW,SAAS,CAAA;AAEpD,IAAA,MAAM,WAAA,GAAc,MAAM,IAAA,CAAK,wBAAA,EAAyB;AACxD,IAAA,MAAM,EAAE,KAAA,EAAM,GAAI,MAAM,KAAK,qBAAA,CAAsB;AAAA,MACjD,UAAA,EAAY,WAAA;AAAA,MACZ,cAAA,EAAgB;AAAA,KACjB,CAAA;AAED,IAAA,MAAM,MAAA,GAAS,IAAI,eAAA,EAAgB;AACnC,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,MAAA,CAAO,GAAA,CAAI,UAAU,MAAM,CAAA;AAAA,IAC7B;AAEA,IAAA,MAAA,CAAO,GAAA,CAAI,OAAA,EAAS,WAAA,CAAY,QAAA,EAAU,CAAA;AAE1C,IAAA,MAAA,CAAO,KAAA;AAAA,MACL,CAAA,yCAAA,EAA4C,UAAU,SAAS,CAAA;AAAA,KACjE;AAEA,IAAA,MAAM,WAAW,MAAM,KAAA;AAAA,MACrB,CAAA,EAAG,OAAO,CAAA,mBAAA,EAAsB,MAAA,CAAO,UAAU,CAAA,CAAA;AAAA,MACjD;AAAA,QACE,OAAA,EAAS;AAAA,UACP,aAAA,EAAe,UAAU,KAAK,CAAA;AAAA,SAChC;AAAA,QACA,OAAA,EAAS,CAAA;AAAA,QACT,YAAY,CAAA,OAAA,KAAW;AAGrB,UAAA,MAAA,CAAO,KAAA,CAAM,CAAA,iCAAA,EAAoC,OAAO,CAAA,CAAE,CAAA;AAE1D,UAAA,MAAM,gBAAA,GAAmB,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,OAAO,CAAA,GAAI,GAAA;AAChD,UAAA,MAAM,MAAA,GAAS,IAAA,CAAK,MAAA,EAAO,GAAI,GAAA;AAC/B,UAAA,OAAO,gBAAA,GAAmB,MAAA;AAAA,QAC5B;AAAA;AACF,KACF;AAEA,IAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,CAAA,kCAAA,EAAqC,QAAA,CAAS,MAAM,CAAA,CAAA,EAAI,SAAS,UAAU,CAAA;AAAA,OAC7E;AACA,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,IAAA,GAAO,MAAM,QAAA,CAAS,IAAA,EAAK;AAEjC,IAAA,MAAM,WAAW,IAAA,CAAK,KAAA;AACtB,IAAA,MAAM,UAAA,GAAa,KAAK,QAAA,EAAU,UAAA;AAElC,IAAA,MAAM,YAAiC,QAAA,CAAS,GAAA;AAAA,MAC9CC;AAAA,KACF;AAEA,IAAA,MAAM,mBAAmB,SAAS,CAAA;AAElC,IAAA,IAAI,CAAC,UAAA,EAAY;AACf,MAAA,MAAA,CAAO,KAAK,6CAA6C,CAAA;AACzD,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,kBAAA,CAAmB,oBAAoB,UAAU,CAAA;AAAA,EACzD,CAAA;AAEA,EAAA,MAAM,MAAA,GAA6B,OAAO,EAAE,kBAAA,EAAmB,KAAM;AACnE,IAAA,MAAM,mBAAmB,kBAAkB,CAAA;AAAA,EAC7C,CAAA;AAEA,EAAA,OAAO;AAAA,IACL,EAAA,EAAIC,kBAAA;AAAA,IACJ;AAAA,GACF;AACF;;;;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sweetoburrito/backstage-plugin-ai-assistant-backend-module-ingestor-catalog",
-  "version": "0.3.6",
+  "version": "0.4.1",
   "license": "Apache-2.0",
   "description": "The ingestor-catalog backend module for the ai-assistant plugin.",
   "main": "dist/index.cjs.js",
@@ -31,7 +31,8 @@
     "@backstage/backend-plugin-api": "backstage:^",
     "@backstage/catalog-model": "backstage:^",
     "@backstage/plugin-catalog-node": "backstage:^",
-    "@sweetoburrito/backstage-plugin-ai-assistant-node": "^0.6.0"
+    "@sweetoburrito/backstage-plugin-ai-assistant-node": "^0.7.0",
+    "fetch-retry": "^6.0.0"
   },
   "devDependencies": {
     "@backstage/backend-defaults": "backstage:^",