npm - @backstage/plugin-catalog-backend-module-incremental-ingestion - Versions diffs - 0.7.10-next.1 → 0.7.10 - Mend

@backstage/plugin-catalog-backend-module-incremental-ingestion 0.7.10-next.1 → 0.7.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (8) hide show

package/CHANGELOG.md CHANGED Viewed

@@ -1,5 +1,30 @@
 # @backstage/plugin-catalog-backend-module-incremental-ingestion
+## 0.7.10
+### Patch Changes
+- 5f1e7b8: Migrated metrics from direct `@opentelemetry/api` usage to the alpha `MetricsService`, providing plugin-scoped metric attribution. The `@opentelemetry/api` dependency has been removed.
+- Updated dependencies
+  - @backstage/backend-plugin-api@1.8.0
+  - @backstage/backend-defaults@0.16.0
+  - @backstage/plugin-catalog-backend@3.5.0
+  - @backstage/plugin-catalog-node@2.1.0
+  - @backstage/plugin-permission-common@0.9.7
+  - @backstage/catalog-model@1.7.7
+  - @backstage/plugin-events-node@0.4.20
+## 0.7.10-next.2
+### Patch Changes
+- Updated dependencies
+  - @backstage/backend-plugin-api@1.8.0-next.1
+  - @backstage/backend-defaults@0.16.0-next.2
+  - @backstage/plugin-catalog-backend@3.5.0-next.2
+  - @backstage/plugin-catalog-node@2.1.0-next.2
+  - @backstage/plugin-events-node@0.4.20-next.1
 ## 0.7.10-next.1
 ### Patch Changes

package/dist/engine/IncrementalIngestionEngine.cjs.js CHANGED Viewed

@@ -1,6 +1,5 @@
 'use strict';
-var api = require('@opentelemetry/api');
 var node_perf_hooks = require('node:perf_hooks');
 var luxon = require('luxon');
 var uuid = require('uuid');
@@ -9,7 +8,6 @@ var errors = require('@backstage/errors');
 class IncrementalIngestionEngine {
   constructor(options) {
     this.options = options;
-    const meter = api.metrics.getMeter("default");
     this.manager = options.manager;
     this.restLength = luxon.Duration.fromObject(options.restLength);
     this.burstLength = luxon.Duration.fromObject(options.burstLength);
@@ -19,18 +17,18 @@ class IncrementalIngestionEngine {
       { minutes: 30 },
       { hours: 3 }
     ];
-    this.lastStarted = meter.createGauge(
+    this.lastStarted = options.metrics.createGauge(
       "catalog_incremental.ingestions.started",
       {
-        description: "Epoch timestamp seconds when the ingestion was last started",
-        unit: "seconds"
+        description: "Epoch timestamp when the ingestion was last started",
+        unit: "s"
       }
     );
-    this.lastCompleted = meter.createGauge(
+    this.lastCompleted = options.metrics.createGauge(
       "catalog_incremental.ingestions.completed",
       {
-        description: "Epoch timestamp seconds when the ingestion was last completed",
-        unit: "seconds"
+        description: "Epoch timestamp when the ingestion was last completed",
+        unit: "s"
       }
     );
   }

package/dist/engine/IncrementalIngestionEngine.cjs.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"IncrementalIngestionEngine.cjs.js","sources":["../../src/engine/IncrementalIngestionEngine.ts"],"sourcesContent":["/\n Copyright 2022 The Backstage Authors\n \n Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n \n http://www.apache.org/licenses/LICENSE-2.0\n \n Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n /\n\nimport type { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Gauge, metrics } from '@opentelemetry/api';\nimport { IterationEngine, IterationEngineOptions } from '../types';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { performance } from 'node:perf_hooks';\nimport { Duration } from 'luxon';\nimport { v4 } from 'uuid';\nimport { stringifyError } from '@backstage/errors';\nimport { EventParams } from '@backstage/plugin-events-node';\nimport { HumanDuration } from '@backstage/types';\n\nexport class IncrementalIngestionEngine implements IterationEngine {\n private readonly restLength: Duration;\n private readonly burstLength: Duration;\n private readonly backoff: HumanDuration[];\n private readonly lastStarted: Gauge;\n private readonly lastCompleted: Gauge;\n\n private manager: IncrementalIngestionDatabaseManager;\n\n constructor(private options: IterationEngineOptions) {\n const meter = metrics.getMeter('default');\n\n this.manager = options.manager;\n this.restLength = Duration.fromObject(options.restLength);\n this.burstLength = Duration.fromObject(options.burstLength);\n this.backoff = options.backoff ?? [\n { minutes: 1 },\n { minutes: 5 },\n { minutes: 30 },\n { hours: 3 },\n ];\n\n this.lastStarted = meter.createGauge(\n 'catalog_incremental.ingestions.started',\n {\n description:\n 'Epoch timestamp seconds when the ingestion was last started',\n unit: 'seconds',\n },\n );\n this.lastCompleted = meter.createGauge(\n 'catalog_incremental.ingestions.completed',\n {\n description:\n 'Epoch timestamp seconds when the ingestion was last completed',\n unit: 'seconds',\n },\n );\n }\n\n async taskFn(signal: AbortSignal) {\n try {\n this.options.logger.debug('Begin tick');\n await this.handleNextAction(signal);\n } catch (error) {\n this.options.logger.error(`${error}`);\n throw error;\n } finally {\n this.options.logger.debug('End tick');\n }\n }\n\n async handleNextAction(signal: AbortSignal) {\n await this.options.ready;\n\n const result = await this.getCurrentAction();\n if (result) {\n const { ingestionId, nextActionAt, nextAction, attempts } = result;\n\n switch (nextAction) {\n case 'rest':\n if (Date.now() > nextActionAt) {\n await this.manager.clearFinishedIngestions(\n this.options.provider.getProviderName(),\n );\n this.options.logger.debug(\n `incremental-engine: Ingestion ${ingestionId} rest period complete. Ingestion will start again`,\n );\n\n this.lastStarted.record(Date.now() / 1000, {\n providerName: this.options.provider.getProviderName(),\n });\n await this.manager.setProviderComplete(ingestionId);\n } else {\n this.options.logger.debug(\n `incremental-engine: Ingestion '${ingestionId}' rest period continuing`,\n );\n }\n break;\n case 'ingest':\n try {\n await this.manager.setProviderBursting(ingestionId);\n const done = await this.ingestOneBurst(ingestionId, signal);\n if (done) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${ingestionId}' complete, transitioning to rest period of ${this.restLength.toHuman()}`,\n );\n this.lastCompleted.record(Date.now() / 1000, {\n providerName: this.options.provider.getProviderName(),\n status: 'completed',\n });\n await this.manager.setProviderResting(\n ingestionId,\n this.restLength,\n );\n } else {\n await this.manager.setProviderInterstitial(ingestionId);\n this.options.logger.debug(\n `incremental-engine: Ingestion '${ingestionId}' continuing`,\n );\n }\n } catch (error) {\n if (\n (error as Error).message &&\n (error as Error).message === 'CANCEL'\n ) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${ingestionId}' canceled`,\n );\n await this.manager.setProviderCanceling(\n ingestionId,\n (error as Error).message,\n );\n } else {\n const currentBackoff = Duration.fromObject(\n this.backoff[Math.min(this.backoff.length - 1, attempts)],\n );\n\n const backoffLength = currentBackoff.as('milliseconds');\n this.options.logger.error(\n `incremental-engine: Ingestion '${ingestionId}' failed`,\n error,\n );\n\n const truncatedError = stringifyError(error).substring(0, 700);\n this.options.logger.error(\n `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`,\n );\n this.lastCompleted.record(Date.now() / 1000, {\n providerName: this.options.provider.getProviderName(),\n status: 'failed',\n });\n\n await this.manager.setProviderBackoff(\n ingestionId,\n attempts,\n error as Error,\n backoffLength,\n );\n }\n }\n break;\n case 'backoff':\n if (Date.now() > nextActionAt) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${ingestionId}' backoff complete, will attempt to resume`,\n );\n await this.manager.setProviderIngesting(ingestionId);\n } else {\n this.options.logger.debug(\n `incremental-engine: Ingestion '${ingestionId}' backoff continuing`,\n );\n }\n break;\n case 'cancel':\n this.options.logger.info(\n `incremental-engine: Ingestion '${ingestionId}' canceling, will restart`,\n );\n await this.manager.setProviderCanceled(ingestionId);\n break;\n default:\n this.options.logger.error(\n `incremental-engine: Ingestion '${ingestionId}' received unknown action '${nextAction}'`,\n );\n }\n } else {\n this.options.logger.error(\n `incremental-engine: Engine tried to create duplicate ingestion record for provider '${this.options.provider.getProviderName()}'.`,\n );\n }\n }\n\n async getCurrentAction() {\n const providerName = this.options.provider.getProviderName();\n const record = await this.manager.getCurrentIngestionRecord(providerName);\n if (record) {\n this.options.logger.debug(\n `incremental-engine: Ingestion record found: '${record.id}'`,\n );\n return {\n ingestionId: record.id,\n nextAction: record.next_action as 'rest' \| 'ingest' \| 'backoff',\n attempts: record.attempts as number,\n nextActionAt: record.next_action_at.valueOf() as number,\n };\n }\n const result = await this.manager.createProviderIngestionRecord(\n providerName,\n );\n if (result) {\n this.options.logger.info(\n `incremental-engine: Ingestion record created: '${result.ingestionId}'`,\n );\n }\n return result;\n }\n\n async ingestOneBurst(id: string, signal: AbortSignal) {\n const lastMark = await this.manager.getLastMark(id);\n\n const cursor = lastMark ? lastMark.cursor : undefined;\n let sequence = lastMark ? lastMark.sequence + 1 : 0;\n\n const start = performance.now();\n let count = 0;\n let done = false;\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}' burst initiated`,\n );\n\n await this.options.provider.around(async (context: unknown) => {\n let next = await this.options.provider.next(context, cursor);\n count++;\n for (;;) {\n done = next.done;\n await this.mark({\n id,\n sequence,\n entities: next?.entities,\n done: next.done,\n cursor: next?.cursor,\n });\n if (signal.aborted \|\| next.done) {\n break;\n } else if (\n performance.now() - start >\n this.burstLength.as('milliseconds')\n ) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}' burst ending after ${this.burstLength.toHuman()}.`,\n );\n break;\n } else {\n next = await this.options.provider.next(context, next.cursor);\n count++;\n sequence++;\n }\n }\n });\n\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}' burst complete. (${count} batches in ${Math.round(\n performance.now() - start,\n )}ms).`,\n );\n return done;\n }\n\n async mark(options: {\n id: string;\n sequence: number;\n entities?: DeferredEntity[];\n done: boolean;\n cursor?: unknown;\n }) {\n const { id, sequence, entities, done, cursor } = options;\n this.options.logger.debug(\n `incremental-engine: Ingestion '${id}': MARK ${\n entities ? entities.length : 0\n } entities, cursor: ${\n cursor ? JSON.stringify(cursor) : 'none'\n }, done: ${done}`,\n );\n const markId = v4();\n\n await this.manager.createMark({\n record: {\n id: markId,\n ingestion_id: id,\n cursor,\n sequence,\n },\n });\n\n if (entities && entities.length > 0) {\n await this.manager.createMarkEntities(markId, entities);\n }\n\n const added =\n entities?.map(deferred => ({\n ...deferred,\n entity: {\n ...deferred.entity,\n metadata: {\n ...deferred.entity.metadata,\n annotations: {\n ...deferred.entity.metadata.annotations,\n },\n },\n },\n })) ?? [];\n\n const removed: { entityRef: string }[] = [];\n\n if (done) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}': Final page reached, calculating removed entities`,\n );\n const result = await this.manager.computeRemoved(\n this.options.provider.getProviderName(),\n id,\n );\n\n const { total } = result;\n\n let doRemoval = true;\n if (this.options.rejectEmptySourceCollections) {\n if (total === 0) {\n this.options.logger.error(\n `incremental-engine: Ingestion '${id}': Rejecting empty entity collection!`,\n );\n doRemoval = false;\n }\n }\n\n if (this.options.rejectRemovalsAbovePercentage) {\n // If the total entities upserted in this ingestion is 0, then\n // 100% of entities are stale and marked for removal.\n const percentRemoved =\n total > 0 ? (result.removed.length / total) 100 : 100;\n if (percentRemoved <= this.options.rejectRemovalsAbovePercentage) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}': Removing ${result.removed.length} entities that have no matching assets`,\n );\n } else {\n const notice = `Attempted to remove ${percentRemoved}% of matching entities!`;\n this.options.logger.error(\n `incremental-engine: Ingestion '${id}': ${notice}`,\n );\n await this.manager.updateIngestionRecordById({\n ingestionId: id,\n update: {\n last_error: `REMOVAL_THRESHOLD exceeded on ingestion mark ${markId}: ${notice}`,\n },\n });\n doRemoval = false;\n }\n }\n if (doRemoval) {\n for (const entityRef of result.removed) {\n removed.push(entityRef);\n }\n }\n }\n\n await this.options.connection.applyMutation({\n type: 'delta',\n added,\n removed,\n });\n }\n\n async onEvent(params: EventParams): Promise<void> {\n const { topic } = params;\n if (!this.supportsEventTopics().includes(topic)) {\n return;\n }\n\n const { logger, provider, connection } = this.options;\n const providerName = provider.getProviderName();\n logger.debug(`incremental-engine: ${providerName} received ${topic} event`);\n\n if (!provider.eventHandler) {\n return;\n }\n\n const result = await provider.eventHandler.onEvent(params);\n\n if (result.type === 'delta') {\n if (result.added.length > 0) {\n const ingestionRecord = await this.manager.getCurrentIngestionRecord(\n providerName,\n );\n\n if (!ingestionRecord) {\n logger.debug(\n `incremental-engine: ${providerName} skipping delta addition because incremental ingestion is restarting.`,\n );\n } else {\n const mark =\n ingestionRecord.status === 'resting'\n ? await this.manager.getLastMark(ingestionRecord.id)\n : await this.manager.getFirstMark(ingestionRecord.id);\n\n if (!mark) {\n throw new Error(\n `Cannot apply delta, page records are missing! Please re-run incremental ingestion for ${providerName}.`,\n );\n }\n await this.manager.createMarkEntities(mark.id, result.added);\n }\n }\n\n if (result.removed.length > 0) {\n await this.manager.deleteEntityRecordsByRef(result.removed);\n }\n\n await connection.applyMutation(result);\n logger.debug(\n `incremental-engine: ${providerName} processed delta from '${topic}' event`,\n );\n } else {\n logger.debug(\n `incremental-engine: ${providerName} ignored event from topic '${topic}'`,\n );\n }\n }\n\n supportsEventTopics(): string[] {\n const { provider } = this.options;\n const topics = provider.eventHandler\n ? provider.eventHandler.supportsEventTopics()\n : [];\n return topics;\n }\n}\n"],"names":["metrics","Duration","stringifyError","performance","v4"],"mappings":";;;;;;;;AA2BO,MAAM,0BAAA,CAAsD;AAAA,EASjE,YAAoB,OAAA,EAAiC;AAAjC,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAClB,IAAA,MAAM,KAAA,GAAQA,WAAA,CAAQ,QAAA,CAAS,SAAS,CAAA;AAExC,IAAA,IAAA,CAAK,UAAU,OAAA,CAAQ,OAAA;AACvB,IAAA,IAAA,CAAK,UAAA,GAAaC,cAAA,CAAS,UAAA,CAAW,OAAA,CAAQ,UAAU,CAAA;AACxD,IAAA,IAAA,CAAK,WAAA,GAAcA,cAAA,CAAS,UAAA,CAAW,OAAA,CAAQ,WAAW,CAAA;AAC1D,IAAA,IAAA,CAAK,OAAA,GAAU,QAAQ,OAAA,IAAW;AAAA,MAChC,EAAE,SAAS,CAAA,EAAE;AAAA,MACb,EAAE,SAAS,CAAA,EAAE;AAAA,MACb,EAAE,SAAS,EAAA,EAAG;AAAA,MACd,EAAE,OAAO,CAAA;AAAE,KACb;AAEA,IAAA,IAAA,CAAK,cAAc,KAAA,CAAM,WAAA;AAAA,MACvB,wCAAA;AAAA,MACA;AAAA,QACE,WAAA,EACE,6DAAA;AAAA,QACF,IAAA,EAAM;AAAA;AACR,KACF;AACA,IAAA,IAAA,CAAK,gBAAgB,KAAA,CAAM,WAAA;AAAA,MACzB,0CAAA;AAAA,MACA;AAAA,QACE,WAAA,EACE,+DAAA;AAAA,QACF,IAAA,EAAM;AAAA;AACR,KACF;AAAA,EACF;AAAA,EArCiB,UAAA;AAAA,EACA,WAAA;AAAA,EACA,OAAA;AAAA,EACA,WAAA;AAAA,EACA,aAAA;AAAA,EAET,OAAA;AAAA,EAiCR,MAAM,OAAO,MAAA,EAAqB;AAChC,IAAA,IAAI;AACF,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,YAAY,CAAA;AACtC,MAAA,MAAM,IAAA,CAAK,iBAAiB,MAAM,CAAA;AAAA,IACpC,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,KAAK,CAAA,CAAE,CAAA;AACpC,MAAA,MAAM,KAAA;AAAA,IACR,CAAA,SAAE;AACA,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,UAAU,CAAA;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,MAAM,iBAAiB,MAAA,EAAqB;AAC1C,IAAA,MAAM,KAAK,OAAA,CAAQ,KAAA;AAEnB,IAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,gBAAA,EAAiB;AAC3C,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,MAAM,EAAE,WAAA,EAAa,YAAA,EAAc,UAAA,EAAY,UAAS,GAAI,MAAA;AAE5D,MAAA,QAAQ,UAAA;AAAY,QAClB,KAAK,MAAA;AACH,UAAA,IAAI,IAAA,CAAK,GAAA,EAAI,GAAI,YAAA,EAAc;AAC7B,YAAA,MAAM,KAAK,OAAA,CAAQ,uBAAA;AAAA,cACjB,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA;AAAgB,aACxC;AACA,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,cAClB,iCAAiC,WAAW,CAAA,iDAAA;AAAA,aAC9C;AAEA,YAAA,IAAA,CAAK,WAAA,CAAY,MAAA,CAAO,IAAA,CAAK,GAAA,KAAQ,GAAA,EAAM;AAAA,cACzC,YAAA,EAAc,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA;AAAgB,aACrD,CAAA;AACD,YAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,mBAAA,CAAoB,WAAW,CAAA;AAAA,UACpD,CAAA,MAAO;AACL,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,wBAAA;AAAA,aAC/C;AAAA,UACF;AACA,UAAA;AAAA,QACF,KAAK,QAAA;AACH,UAAA,IAAI;AACF,YAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,mBAAA,CAAoB,WAAW,CAAA;AAClD,YAAA,MAAM,IAAA,GAAO,MAAM,IAAA,CAAK,cAAA,CAAe,aAAa,MAAM,CAAA;AAC1D,YAAA,IAAI,IAAA,EAAM;AACR,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,4CAAA,EAA+C,IAAA,CAAK,UAAA,CAAW,SAAS,CAAA;AAAA,eACvH;AACA,cAAA,IAAA,CAAK,aAAA,CAAc,MAAA,CAAO,IAAA,CAAK,GAAA,KAAQ,GAAA,EAAM;AAAA,gBAC3C,YAAA,EAAc,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAAA,gBACpD,MAAA,EAAQ;AAAA,eACT,CAAA;AACD,cAAA,MAAM,KAAK,OAAA,CAAQ,kBAAA;AAAA,gBACjB,WAAA;AAAA,gBACA,IAAA,CAAK;AAAA,eACP;AAAA,YACF,CAAA,MAAO;AACL,cAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,uBAAA,CAAwB,WAAW,CAAA;AACtD,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,YAAA;AAAA,eAC/C;AAAA,YACF;AAAA,UACF,SAAS,KAAA,EAAO;AACd,YAAA,IACG,KAAA,CAAgB,OAAA,IAChB,KAAA,CAAgB,OAAA,KAAY,QAAA,EAC7B;AACA,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,UAAA;AAAA,eAC/C;AACA,cAAA,MAAM,KAAK,OAAA,CAAQ,oBAAA;AAAA,gBACjB,WAAA;AAAA,gBACC,KAAA,CAAgB;AAAA,eACnB;AAAA,YACF,CAAA,MAAO;AACL,cAAA,MAAM,iBAAiBA,cAAA,CAAS,UAAA;AAAA,gBAC9B,IAAA,CAAK,QAAQ,IAAA,CAAK,GAAA,CAAI,KAAK,OAAA,CAAQ,MAAA,GAAS,CAAA,EAAG,QAAQ,CAAC;AAAA,eAC1D;AAEA,cAAA,MAAM,aAAA,GAAgB,cAAA,CAAe,EAAA,CAAG,cAAc,CAAA;AACtD,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,QAAA,CAAA;AAAA,gBAC7C;AAAA,eACF;AAEA,cAAA,MAAM,iBAAiBC,qBAAA,CAAe,KAAK,CAAA,CAAE,SAAA,CAAU,GAAG,GAAG,CAAA;AAC7D,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,oEAAA,EAAuE,eAAe,OAAA,EAAS,KAAK,cAAc,CAAA,CAAA;AAAA,eACjK;AACA,cAAA,IAAA,CAAK,aAAA,CAAc,MAAA,CAAO,IAAA,CAAK,GAAA,KAAQ,GAAA,EAAM;AAAA,gBAC3C,YAAA,EAAc,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAAA,gBACpD,MAAA,EAAQ;AAAA,eACT,CAAA;AAED,cAAA,MAAM,KAAK,OAAA,CAAQ,kBAAA;AAAA,gBACjB,WAAA;AAAA,gBACA,QAAA;AAAA,gBACA,KAAA;AAAA,gBACA;AAAA,eACF;AAAA,YACF;AAAA,UACF;AACA,UAAA;AAAA,QACF,KAAK,SAAA;AACH,UAAA,IAAI,IAAA,CAAK,GAAA,EAAI,GAAI,YAAA,EAAc;AAC7B,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,0CAAA;AAAA,aAC/C;AACA,YAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,oBAAA,CAAqB,WAAW,CAAA;AAAA,UACrD,CAAA,MAAO;AACL,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,oBAAA;AAAA,aAC/C;AAAA,UACF;AACA,UAAA;AAAA,QACF,KAAK,QAAA;AACH,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,YAClB,kCAAkC,WAAW,CAAA,yBAAA;AAAA,WAC/C;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,mBAAA,CAAoB,WAAW,CAAA;AAClD,UAAA;AAAA,QACF;AACE,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,WAAW,CAAA,2BAAA,EAA8B,UAAU,CAAA,CAAA;AAAA,WACvF;AAAA;AACJ,IACF,CAAA,MAAO;AACL,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,QAClB,CAAA,oFAAA,EAAuF,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,iBAAiB,CAAA,EAAA;AAAA,OAChI;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,gBAAA,GAAmB;AACvB,IAAA,MAAM,YAAA,GAAe,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAC3D,IAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,YAAY,CAAA;AACxE,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,QAClB,CAAA,6CAAA,EAAgD,OAAO,EAAE,CAAA,CAAA;AAAA,OAC3D;AACA,MAAA,OAAO;AAAA,QACL,aAAa,MAAA,CAAO,EAAA;AAAA,QACpB,YAAY,MAAA,CAAO,WAAA;AAAA,QACnB,UAAU,MAAA,CAAO,QAAA;AAAA,QACjB,YAAA,EAAc,MAAA,CAAO,cAAA,CAAe,OAAA;AAAQ,OAC9C;AAAA,IACF;AACA,IAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,6BAAA;AAAA,MAChC;AAAA,KACF;AACA,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,QAClB,CAAA,+CAAA,EAAkD,OAAO,WAAW,CAAA,CAAA;AAAA,OACtE;AAAA,IACF;AACA,IAAA,OAAO,MAAA;AAAA,EACT;AAAA,EAEA,MAAM,cAAA,CAAe,EAAA,EAAY,MAAA,EAAqB;AACpD,IAAA,MAAM,QAAA,GAAW,MAAM,IAAA,CAAK,OAAA,CAAQ,YAAY,EAAE,CAAA;AAElD,IAAA,MAAM,MAAA,GAAS,QAAA,GAAW,QAAA,CAAS,MAAA,GAAS,MAAA;AAC5C,IAAA,IAAI,QAAA,GAAW,QAAA,GAAW,QAAA,CAAS,QAAA,GAAW,CAAA,GAAI,CAAA;AAElD,IAAA,MAAM,KAAA,GAAQC,4BAAY,GAAA,EAAI;AAC9B,IAAA,IAAI,KAAA,GAAQ,CAAA;AACZ,IAAA,IAAI,IAAA,GAAO,KAAA;AACX,IAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,MAClB,kCAAkC,EAAE,CAAA,iBAAA;AAAA,KACtC;AAEA,IAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,MAAA,CAAO,OAAO,OAAA,KAAqB;AAC7D,MAAA,IAAI,OAAO,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,IAAA,CAAK,SAAS,MAAM,CAAA;AAC3D,MAAA,KAAA,EAAA;AACA,MAAA,WAAS;AACP,QAAA,IAAA,GAAO,IAAA,CAAK,IAAA;AACZ,QAAA,MAAM,KAAK,IAAA,CAAK;AAAA,UACd,EAAA;AAAA,UACA,QAAA;AAAA,UACA,UAAU,IAAA,EAAM,QAAA;AAAA,UAChB,MAAM,IAAA,CAAK,IAAA;AAAA,UACX,QAAQ,IAAA,EAAM;AAAA,SACf,CAAA;AACD,QAAA,IAAI,MAAA,CAAO,OAAA,IAAW,IAAA,CAAK,IAAA,EAAM;AAC/B,UAAA;AAAA,QACF,CAAA,MAAA,IACEA,4BAAY,GAAA,EAAI,GAAI,QACpB,IAAA,CAAK,WAAA,CAAY,EAAA,CAAG,cAAc,CAAA,EAClC;AACA,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,YAClB,kCAAkC,EAAE,CAAA,qBAAA,EAAwB,IAAA,CAAK,WAAA,CAAY,SAAS,CAAA,CAAA;AAAA,WACxF;AACA,UAAA;AAAA,QACF,CAAA,MAAO;AACL,UAAA,IAAA,GAAO,MAAM,IAAA,CAAK,OAAA,CAAQ,SAAS,IAAA,CAAK,OAAA,EAAS,KAAK,MAAM,CAAA;AAC5D,UAAA,KAAA,EAAA;AACA,UAAA,QAAA,EAAA;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAED,IAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,MAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,mBAAA,EAAsB,KAAK,eAAe,IAAA,CAAK,KAAA;AAAA,QACjFA,2BAAA,CAAY,KAAI,GAAI;AAAA,OACrB,CAAA,IAAA;AAAA,KACH;AACA,IAAA,OAAO,IAAA;AAAA,EACT;AAAA,EAEA,MAAM,KAAK,OAAA,EAMR;AACD,IAAA,MAAM,EAAE,EAAA,EAAI,QAAA,EAAU,QAAA,EAAU,IAAA,EAAM,QAAO,GAAI,OAAA;AACjD,IAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,MAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,QAAA,EAClC,QAAA,GAAW,SAAS,MAAA,GAAS,CAC/B,CAAA,mBAAA,EACE,MAAA,GAAS,KAAK,SAAA,CAAU,MAAM,CAAA,GAAI,MACpC,WAAW,IAAI,CAAA;AAAA,KACjB;AACA,IAAA,MAAM,SAASC,OAAA,EAAG;AAElB,IAAA,MAAM,IAAA,CAAK,QAAQ,UAAA,CAAW;AAAA,MAC5B,MAAA,EAAQ;AAAA,QACN,EAAA,EAAI,MAAA;AAAA,QACJ,YAAA,EAAc,EAAA;AAAA,QACd,MAAA;AAAA,QACA;AAAA;AACF,KACD,CAAA;AAED,IAAA,IAAI,QAAA,IAAY,QAAA,CAAS,MAAA,GAAS,CAAA,EAAG;AACnC,MAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,kBAAA,CAAmB,MAAA,EAAQ,QAAQ,CAAA;AAAA,IACxD;AAEA,IAAA,MAAM,KAAA,GACJ,QAAA,EAAU,GAAA,CAAI,CAAA,QAAA,MAAa;AAAA,MACzB,GAAG,QAAA;AAAA,MACH,MAAA,EAAQ;AAAA,QACN,GAAG,QAAA,CAAS,MAAA;AAAA,QACZ,QAAA,EAAU;AAAA,UACR,GAAG,SAAS,MAAA,CAAO,QAAA;AAAA,UACnB,WAAA,EAAa;AAAA,YACX,GAAG,QAAA,CAAS,MAAA,CAAO,QAAA,CAAS;AAAA;AAC9B;AACF;AACF,KACF,CAAE,KAAK,EAAC;AAEV,IAAA,MAAM,UAAmC,EAAC;AAE1C,IAAA,IAAI,IAAA,EAAM;AACR,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,QAClB,kCAAkC,EAAE,CAAA,mDAAA;AAAA,OACtC;AACA,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,cAAA;AAAA,QAChC,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAAA,QACtC;AAAA,OACF;AAEA,MAAA,MAAM,EAAE,OAAM,GAAI,MAAA;AAElB,MAAA,IAAI,SAAA,GAAY,IAAA;AAChB,MAAA,IAAI,IAAA,CAAK,QAAQ,4BAAA,EAA8B;AAC7C,QAAA,IAAI,UAAU,CAAA,EAAG;AACf,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,YAClB,kCAAkC,EAAE,CAAA,qCAAA;AAAA,WACtC;AACA,UAAA,SAAA,GAAY,KAAA;AAAA,QACd;AAAA,MACF;AAEA,MAAA,IAAI,IAAA,CAAK,QAAQ,6BAAA,EAA+B;AAG9C,QAAA,MAAM,iBACJ,KAAA,GAAQ,CAAA,GAAK,OAAO,OAAA,CAAQ,MAAA,GAAS,QAAS,GAAA,GAAM,GAAA;AACtD,QAAA,IAAI,cAAA,IAAkB,IAAA,CAAK,OAAA,CAAQ,6BAAA,EAA+B;AAChE,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,YAAA,EAAe,MAAA,CAAO,QAAQ,MAAM,CAAA,sCAAA;AAAA,WAC1E;AAAA,QACF,CAAA,MAAO;AACL,UAAA,MAAM,MAAA,GAAS,uBAAuB,cAAc,CAAA,uBAAA,CAAA;AACpD,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,GAAA,EAAM,MAAM,CAAA;AAAA,WAClD;AACA,UAAA,MAAM,IAAA,CAAK,QAAQ,yBAAA,CAA0B;AAAA,YAC3C,WAAA,EAAa,EAAA;AAAA,YACb,MAAA,EAAQ;AAAA,cACN,UAAA,EAAY,CAAA,6CAAA,EAAgD,MAAM,CAAA,EAAA,EAAK,MAAM,CAAA;AAAA;AAC/E,WACD,CAAA;AACD,UAAA,SAAA,GAAY,KAAA;AAAA,QACd;AAAA,MACF;AACA,MAAA,IAAI,SAAA,EAAW;AACb,QAAA,KAAA,MAAW,SAAA,IAAa,OAAO,OAAA,EAAS;AACtC,UAAA,OAAA,CAAQ,KAAK,SAAS,CAAA;AAAA,QACxB;AAAA,MACF;AAAA,IACF;AAEA,IAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,UAAA,CAAW,aAAA,CAAc;AAAA,MAC1C,IAAA,EAAM,OAAA;AAAA,MACN,KAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ,MAAA,EAAoC;AAChD,IAAA,MAAM,EAAE,OAAM,GAAI,MAAA;AAClB,IAAA,IAAI,CAAC,IAAA,CAAK,mBAAA,EAAoB,CAAE,QAAA,CAAS,KAAK,CAAA,EAAG;AAC/C,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,EAAE,MAAA,EAAQ,QAAA,EAAU,UAAA,KAAe,IAAA,CAAK,OAAA;AAC9C,IAAA,MAAM,YAAA,GAAe,SAAS,eAAA,EAAgB;AAC9C,IAAA,MAAA,CAAO,KAAA,CAAM,CAAA,oBAAA,EAAuB,YAAY,CAAA,UAAA,EAAa,KAAK,CAAA,MAAA,CAAQ,CAAA;AAE1E,IAAA,IAAI,CAAC,SAAS,YAAA,EAAc;AAC1B,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,MAAA,GAAS,MAAM,QAAA,CAAS,YAAA,CAAa,QAAQ,MAAM,CAAA;AAEzD,IAAA,IAAI,MAAA,CAAO,SAAS,OAAA,EAAS;AAC3B,MAAA,IAAI,MAAA,CAAO,KAAA,CAAM,MAAA,GAAS,CAAA,EAAG;AAC3B,QAAA,MAAM,eAAA,GAAkB,MAAM,IAAA,CAAK,OAAA,CAAQ,yBAAA;AAAA,UACzC;AAAA,SACF;AAEA,QAAA,IAAI,CAAC,eAAA,EAAiB;AACpB,UAAA,MAAA,CAAO,KAAA;AAAA,YACL,uBAAuB,YAAY,CAAA,qEAAA;AAAA,WACrC;AAAA,QACF,CAAA,MAAO;AACL,UAAA,MAAM,OACJ,eAAA,CAAgB,MAAA,KAAW,SAAA,GACvB,MAAM,KAAK,OAAA,CAAQ,WAAA,CAAY,eAAA,CAAgB,EAAE,IACjD,MAAM,IAAA,CAAK,OAAA,CAAQ,YAAA,CAAa,gBAAgB,EAAE,CAAA;AAExD,UAAA,IAAI,CAAC,IAAA,EAAM;AACT,YAAA,MAAM,IAAI,KAAA;AAAA,cACR,yFAAyF,YAAY,CAAA,CAAA;AAAA,aACvG;AAAA,UACF;AACA,UAAA,MAAM,KAAK,OAAA,CAAQ,kBAAA,CAAmB,IAAA,CAAK,EAAA,EAAI,OAAO,KAAK,CAAA;AAAA,QAC7D;AAAA,MACF;AAEA,MAAA,IAAI,MAAA,CAAO,OAAA,CAAQ,MAAA,GAAS,CAAA,EAAG;AAC7B,QAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,wBAAA,CAAyB,MAAA,CAAO,OAAO,CAAA;AAAA,MAC5D;AAEA,MAAA,MAAM,UAAA,CAAW,cAAc,MAAM,CAAA;AACrC,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,CAAA,oBAAA,EAAuB,YAAY,CAAA,uBAAA,EAA0B,KAAK,CAAA,OAAA;AAAA,OACpE;AAAA,IACF,CAAA,MAAO;AACL,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,CAAA,oBAAA,EAAuB,YAAY,CAAA,2BAAA,EAA8B,KAAK,CAAA,CAAA;AAAA,OACxE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,mBAAA,GAAgC;AAC9B,IAAA,MAAM,EAAE,QAAA,EAAS,GAAI,IAAA,CAAK,OAAA;AAC1B,IAAA,MAAM,SAAS,QAAA,CAAS,YAAA,GACpB,SAAS,YAAA,CAAa,mBAAA,KACtB,EAAC;AACL,IAAA,OAAO,MAAA;AAAA,EACT;AACF;;;;"}
1	+ {"version":3,"file":"IncrementalIngestionEngine.cjs.js","sources":["../../src/engine/IncrementalIngestionEngine.ts"],"sourcesContent":["/\n Copyright 2022 The Backstage Authors\n \n Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n \n http://www.apache.org/licenses/LICENSE-2.0\n \n Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n /\n\nimport type { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { MetricsServiceGauge } from '@backstage/backend-plugin-api/alpha';\nimport { IterationEngine, IterationEngineOptions } from '../types';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { performance } from 'node:perf_hooks';\nimport { Duration } from 'luxon';\nimport { v4 } from 'uuid';\nimport { stringifyError } from '@backstage/errors';\nimport { EventParams } from '@backstage/plugin-events-node';\nimport { HumanDuration } from '@backstage/types';\n\nexport class IncrementalIngestionEngine implements IterationEngine {\n private readonly restLength: Duration;\n private readonly burstLength: Duration;\n private readonly backoff: HumanDuration[];\n private readonly lastStarted: MetricsServiceGauge;\n private readonly lastCompleted: MetricsServiceGauge;\n\n private manager: IncrementalIngestionDatabaseManager;\n\n constructor(private options: IterationEngineOptions) {\n this.manager = options.manager;\n this.restLength = Duration.fromObject(options.restLength);\n this.burstLength = Duration.fromObject(options.burstLength);\n this.backoff = options.backoff ?? [\n { minutes: 1 },\n { minutes: 5 },\n { minutes: 30 },\n { hours: 3 },\n ];\n\n this.lastStarted = options.metrics.createGauge(\n 'catalog_incremental.ingestions.started',\n {\n description: 'Epoch timestamp when the ingestion was last started',\n unit: 's',\n },\n );\n this.lastCompleted = options.metrics.createGauge(\n 'catalog_incremental.ingestions.completed',\n {\n description: 'Epoch timestamp when the ingestion was last completed',\n unit: 's',\n },\n );\n }\n\n async taskFn(signal: AbortSignal) {\n try {\n this.options.logger.debug('Begin tick');\n await this.handleNextAction(signal);\n } catch (error) {\n this.options.logger.error(`${error}`);\n throw error;\n } finally {\n this.options.logger.debug('End tick');\n }\n }\n\n async handleNextAction(signal: AbortSignal) {\n await this.options.ready;\n\n const result = await this.getCurrentAction();\n if (result) {\n const { ingestionId, nextActionAt, nextAction, attempts } = result;\n\n switch (nextAction) {\n case 'rest':\n if (Date.now() > nextActionAt) {\n await this.manager.clearFinishedIngestions(\n this.options.provider.getProviderName(),\n );\n this.options.logger.debug(\n `incremental-engine: Ingestion ${ingestionId} rest period complete. Ingestion will start again`,\n );\n\n this.lastStarted.record(Date.now() / 1000, {\n providerName: this.options.provider.getProviderName(),\n });\n await this.manager.setProviderComplete(ingestionId);\n } else {\n this.options.logger.debug(\n `incremental-engine: Ingestion '${ingestionId}' rest period continuing`,\n );\n }\n break;\n case 'ingest':\n try {\n await this.manager.setProviderBursting(ingestionId);\n const done = await this.ingestOneBurst(ingestionId, signal);\n if (done) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${ingestionId}' complete, transitioning to rest period of ${this.restLength.toHuman()}`,\n );\n this.lastCompleted.record(Date.now() / 1000, {\n providerName: this.options.provider.getProviderName(),\n status: 'completed',\n });\n await this.manager.setProviderResting(\n ingestionId,\n this.restLength,\n );\n } else {\n await this.manager.setProviderInterstitial(ingestionId);\n this.options.logger.debug(\n `incremental-engine: Ingestion '${ingestionId}' continuing`,\n );\n }\n } catch (error) {\n if (\n (error as Error).message &&\n (error as Error).message === 'CANCEL'\n ) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${ingestionId}' canceled`,\n );\n await this.manager.setProviderCanceling(\n ingestionId,\n (error as Error).message,\n );\n } else {\n const currentBackoff = Duration.fromObject(\n this.backoff[Math.min(this.backoff.length - 1, attempts)],\n );\n\n const backoffLength = currentBackoff.as('milliseconds');\n this.options.logger.error(\n `incremental-engine: Ingestion '${ingestionId}' failed`,\n error,\n );\n\n const truncatedError = stringifyError(error).substring(0, 700);\n this.options.logger.error(\n `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`,\n );\n this.lastCompleted.record(Date.now() / 1000, {\n providerName: this.options.provider.getProviderName(),\n status: 'failed',\n });\n\n await this.manager.setProviderBackoff(\n ingestionId,\n attempts,\n error as Error,\n backoffLength,\n );\n }\n }\n break;\n case 'backoff':\n if (Date.now() > nextActionAt) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${ingestionId}' backoff complete, will attempt to resume`,\n );\n await this.manager.setProviderIngesting(ingestionId);\n } else {\n this.options.logger.debug(\n `incremental-engine: Ingestion '${ingestionId}' backoff continuing`,\n );\n }\n break;\n case 'cancel':\n this.options.logger.info(\n `incremental-engine: Ingestion '${ingestionId}' canceling, will restart`,\n );\n await this.manager.setProviderCanceled(ingestionId);\n break;\n default:\n this.options.logger.error(\n `incremental-engine: Ingestion '${ingestionId}' received unknown action '${nextAction}'`,\n );\n }\n } else {\n this.options.logger.error(\n `incremental-engine: Engine tried to create duplicate ingestion record for provider '${this.options.provider.getProviderName()}'.`,\n );\n }\n }\n\n async getCurrentAction() {\n const providerName = this.options.provider.getProviderName();\n const record = await this.manager.getCurrentIngestionRecord(providerName);\n if (record) {\n this.options.logger.debug(\n `incremental-engine: Ingestion record found: '${record.id}'`,\n );\n return {\n ingestionId: record.id,\n nextAction: record.next_action as 'rest' \| 'ingest' \| 'backoff',\n attempts: record.attempts as number,\n nextActionAt: record.next_action_at.valueOf() as number,\n };\n }\n const result = await this.manager.createProviderIngestionRecord(\n providerName,\n );\n if (result) {\n this.options.logger.info(\n `incremental-engine: Ingestion record created: '${result.ingestionId}'`,\n );\n }\n return result;\n }\n\n async ingestOneBurst(id: string, signal: AbortSignal) {\n const lastMark = await this.manager.getLastMark(id);\n\n const cursor = lastMark ? lastMark.cursor : undefined;\n let sequence = lastMark ? lastMark.sequence + 1 : 0;\n\n const start = performance.now();\n let count = 0;\n let done = false;\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}' burst initiated`,\n );\n\n await this.options.provider.around(async (context: unknown) => {\n let next = await this.options.provider.next(context, cursor);\n count++;\n for (;;) {\n done = next.done;\n await this.mark({\n id,\n sequence,\n entities: next?.entities,\n done: next.done,\n cursor: next?.cursor,\n });\n if (signal.aborted \|\| next.done) {\n break;\n } else if (\n performance.now() - start >\n this.burstLength.as('milliseconds')\n ) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}' burst ending after ${this.burstLength.toHuman()}.`,\n );\n break;\n } else {\n next = await this.options.provider.next(context, next.cursor);\n count++;\n sequence++;\n }\n }\n });\n\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}' burst complete. (${count} batches in ${Math.round(\n performance.now() - start,\n )}ms).`,\n );\n return done;\n }\n\n async mark(options: {\n id: string;\n sequence: number;\n entities?: DeferredEntity[];\n done: boolean;\n cursor?: unknown;\n }) {\n const { id, sequence, entities, done, cursor } = options;\n this.options.logger.debug(\n `incremental-engine: Ingestion '${id}': MARK ${\n entities ? entities.length : 0\n } entities, cursor: ${\n cursor ? JSON.stringify(cursor) : 'none'\n }, done: ${done}`,\n );\n const markId = v4();\n\n await this.manager.createMark({\n record: {\n id: markId,\n ingestion_id: id,\n cursor,\n sequence,\n },\n });\n\n if (entities && entities.length > 0) {\n await this.manager.createMarkEntities(markId, entities);\n }\n\n const added =\n entities?.map(deferred => ({\n ...deferred,\n entity: {\n ...deferred.entity,\n metadata: {\n ...deferred.entity.metadata,\n annotations: {\n ...deferred.entity.metadata.annotations,\n },\n },\n },\n })) ?? [];\n\n const removed: { entityRef: string }[] = [];\n\n if (done) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}': Final page reached, calculating removed entities`,\n );\n const result = await this.manager.computeRemoved(\n this.options.provider.getProviderName(),\n id,\n );\n\n const { total } = result;\n\n let doRemoval = true;\n if (this.options.rejectEmptySourceCollections) {\n if (total === 0) {\n this.options.logger.error(\n `incremental-engine: Ingestion '${id}': Rejecting empty entity collection!`,\n );\n doRemoval = false;\n }\n }\n\n if (this.options.rejectRemovalsAbovePercentage) {\n // If the total entities upserted in this ingestion is 0, then\n // 100% of entities are stale and marked for removal.\n const percentRemoved =\n total > 0 ? (result.removed.length / total) 100 : 100;\n if (percentRemoved <= this.options.rejectRemovalsAbovePercentage) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}': Removing ${result.removed.length} entities that have no matching assets`,\n );\n } else {\n const notice = `Attempted to remove ${percentRemoved}% of matching entities!`;\n this.options.logger.error(\n `incremental-engine: Ingestion '${id}': ${notice}`,\n );\n await this.manager.updateIngestionRecordById({\n ingestionId: id,\n update: {\n last_error: `REMOVAL_THRESHOLD exceeded on ingestion mark ${markId}: ${notice}`,\n },\n });\n doRemoval = false;\n }\n }\n if (doRemoval) {\n for (const entityRef of result.removed) {\n removed.push(entityRef);\n }\n }\n }\n\n await this.options.connection.applyMutation({\n type: 'delta',\n added,\n removed,\n });\n }\n\n async onEvent(params: EventParams): Promise<void> {\n const { topic } = params;\n if (!this.supportsEventTopics().includes(topic)) {\n return;\n }\n\n const { logger, provider, connection } = this.options;\n const providerName = provider.getProviderName();\n logger.debug(`incremental-engine: ${providerName} received ${topic} event`);\n\n if (!provider.eventHandler) {\n return;\n }\n\n const result = await provider.eventHandler.onEvent(params);\n\n if (result.type === 'delta') {\n if (result.added.length > 0) {\n const ingestionRecord = await this.manager.getCurrentIngestionRecord(\n providerName,\n );\n\n if (!ingestionRecord) {\n logger.debug(\n `incremental-engine: ${providerName} skipping delta addition because incremental ingestion is restarting.`,\n );\n } else {\n const mark =\n ingestionRecord.status === 'resting'\n ? await this.manager.getLastMark(ingestionRecord.id)\n : await this.manager.getFirstMark(ingestionRecord.id);\n\n if (!mark) {\n throw new Error(\n `Cannot apply delta, page records are missing! Please re-run incremental ingestion for ${providerName}.`,\n );\n }\n await this.manager.createMarkEntities(mark.id, result.added);\n }\n }\n\n if (result.removed.length > 0) {\n await this.manager.deleteEntityRecordsByRef(result.removed);\n }\n\n await connection.applyMutation(result);\n logger.debug(\n `incremental-engine: ${providerName} processed delta from '${topic}' event`,\n );\n } else {\n logger.debug(\n `incremental-engine: ${providerName} ignored event from topic '${topic}'`,\n );\n }\n }\n\n supportsEventTopics(): string[] {\n const { provider } = this.options;\n const topics = provider.eventHandler\n ? provider.eventHandler.supportsEventTopics()\n : [];\n return topics;\n }\n}\n"],"names":["Duration","stringifyError","performance","v4"],"mappings":";;;;;;;AA2BO,MAAM,0BAAA,CAAsD;AAAA,EASjE,YAAoB,OAAA,EAAiC;AAAjC,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAClB,IAAA,IAAA,CAAK,UAAU,OAAA,CAAQ,OAAA;AACvB,IAAA,IAAA,CAAK,UAAA,GAAaA,cAAA,CAAS,UAAA,CAAW,OAAA,CAAQ,UAAU,CAAA;AACxD,IAAA,IAAA,CAAK,WAAA,GAAcA,cAAA,CAAS,UAAA,CAAW,OAAA,CAAQ,WAAW,CAAA;AAC1D,IAAA,IAAA,CAAK,OAAA,GAAU,QAAQ,OAAA,IAAW;AAAA,MAChC,EAAE,SAAS,CAAA,EAAE;AAAA,MACb,EAAE,SAAS,CAAA,EAAE;AAAA,MACb,EAAE,SAAS,EAAA,EAAG;AAAA,MACd,EAAE,OAAO,CAAA;AAAE,KACb;AAEA,IAAA,IAAA,CAAK,WAAA,GAAc,QAAQ,OAAA,CAAQ,WAAA;AAAA,MACjC,wCAAA;AAAA,MACA;AAAA,QACE,WAAA,EAAa,qDAAA;AAAA,QACb,IAAA,EAAM;AAAA;AACR,KACF;AACA,IAAA,IAAA,CAAK,aAAA,GAAgB,QAAQ,OAAA,CAAQ,WAAA;AAAA,MACnC,0CAAA;AAAA,MACA;AAAA,QACE,WAAA,EAAa,uDAAA;AAAA,QACb,IAAA,EAAM;AAAA;AACR,KACF;AAAA,EACF;AAAA,EAjCiB,UAAA;AAAA,EACA,WAAA;AAAA,EACA,OAAA;AAAA,EACA,WAAA;AAAA,EACA,aAAA;AAAA,EAET,OAAA;AAAA,EA6BR,MAAM,OAAO,MAAA,EAAqB;AAChC,IAAA,IAAI;AACF,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,YAAY,CAAA;AACtC,MAAA,MAAM,IAAA,CAAK,iBAAiB,MAAM,CAAA;AAAA,IACpC,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,KAAK,CAAA,CAAE,CAAA;AACpC,MAAA,MAAM,KAAA;AAAA,IACR,CAAA,SAAE;AACA,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,UAAU,CAAA;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,MAAM,iBAAiB,MAAA,EAAqB;AAC1C,IAAA,MAAM,KAAK,OAAA,CAAQ,KAAA;AAEnB,IAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,gBAAA,EAAiB;AAC3C,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,MAAM,EAAE,WAAA,EAAa,YAAA,EAAc,UAAA,EAAY,UAAS,GAAI,MAAA;AAE5D,MAAA,QAAQ,UAAA;AAAY,QAClB,KAAK,MAAA;AACH,UAAA,IAAI,IAAA,CAAK,GAAA,EAAI,GAAI,YAAA,EAAc;AAC7B,YAAA,MAAM,KAAK,OAAA,CAAQ,uBAAA;AAAA,cACjB,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA;AAAgB,aACxC;AACA,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,cAClB,iCAAiC,WAAW,CAAA,iDAAA;AAAA,aAC9C;AAEA,YAAA,IAAA,CAAK,WAAA,CAAY,MAAA,CAAO,IAAA,CAAK,GAAA,KAAQ,GAAA,EAAM;AAAA,cACzC,YAAA,EAAc,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA;AAAgB,aACrD,CAAA;AACD,YAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,mBAAA,CAAoB,WAAW,CAAA;AAAA,UACpD,CAAA,MAAO;AACL,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,wBAAA;AAAA,aAC/C;AAAA,UACF;AACA,UAAA;AAAA,QACF,KAAK,QAAA;AACH,UAAA,IAAI;AACF,YAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,mBAAA,CAAoB,WAAW,CAAA;AAClD,YAAA,MAAM,IAAA,GAAO,MAAM,IAAA,CAAK,cAAA,CAAe,aAAa,MAAM,CAAA;AAC1D,YAAA,IAAI,IAAA,EAAM;AACR,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,4CAAA,EAA+C,IAAA,CAAK,UAAA,CAAW,SAAS,CAAA;AAAA,eACvH;AACA,cAAA,IAAA,CAAK,aAAA,CAAc,MAAA,CAAO,IAAA,CAAK,GAAA,KAAQ,GAAA,EAAM;AAAA,gBAC3C,YAAA,EAAc,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAAA,gBACpD,MAAA,EAAQ;AAAA,eACT,CAAA;AACD,cAAA,MAAM,KAAK,OAAA,CAAQ,kBAAA;AAAA,gBACjB,WAAA;AAAA,gBACA,IAAA,CAAK;AAAA,eACP;AAAA,YACF,CAAA,MAAO;AACL,cAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,uBAAA,CAAwB,WAAW,CAAA;AACtD,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,YAAA;AAAA,eAC/C;AAAA,YACF;AAAA,UACF,SAAS,KAAA,EAAO;AACd,YAAA,IACG,KAAA,CAAgB,OAAA,IAChB,KAAA,CAAgB,OAAA,KAAY,QAAA,EAC7B;AACA,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,UAAA;AAAA,eAC/C;AACA,cAAA,MAAM,KAAK,OAAA,CAAQ,oBAAA;AAAA,gBACjB,WAAA;AAAA,gBACC,KAAA,CAAgB;AAAA,eACnB;AAAA,YACF,CAAA,MAAO;AACL,cAAA,MAAM,iBAAiBA,cAAA,CAAS,UAAA;AAAA,gBAC9B,IAAA,CAAK,QAAQ,IAAA,CAAK,GAAA,CAAI,KAAK,OAAA,CAAQ,MAAA,GAAS,CAAA,EAAG,QAAQ,CAAC;AAAA,eAC1D;AAEA,cAAA,MAAM,aAAA,GAAgB,cAAA,CAAe,EAAA,CAAG,cAAc,CAAA;AACtD,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,QAAA,CAAA;AAAA,gBAC7C;AAAA,eACF;AAEA,cAAA,MAAM,iBAAiBC,qBAAA,CAAe,KAAK,CAAA,CAAE,SAAA,CAAU,GAAG,GAAG,CAAA;AAC7D,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,oEAAA,EAAuE,eAAe,OAAA,EAAS,KAAK,cAAc,CAAA,CAAA;AAAA,eACjK;AACA,cAAA,IAAA,CAAK,aAAA,CAAc,MAAA,CAAO,IAAA,CAAK,GAAA,KAAQ,GAAA,EAAM;AAAA,gBAC3C,YAAA,EAAc,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAAA,gBACpD,MAAA,EAAQ;AAAA,eACT,CAAA;AAED,cAAA,MAAM,KAAK,OAAA,CAAQ,kBAAA;AAAA,gBACjB,WAAA;AAAA,gBACA,QAAA;AAAA,gBACA,KAAA;AAAA,gBACA;AAAA,eACF;AAAA,YACF;AAAA,UACF;AACA,UAAA;AAAA,QACF,KAAK,SAAA;AACH,UAAA,IAAI,IAAA,CAAK,GAAA,EAAI,GAAI,YAAA,EAAc;AAC7B,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,0CAAA;AAAA,aAC/C;AACA,YAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,oBAAA,CAAqB,WAAW,CAAA;AAAA,UACrD,CAAA,MAAO;AACL,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,oBAAA;AAAA,aAC/C;AAAA,UACF;AACA,UAAA;AAAA,QACF,KAAK,QAAA;AACH,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,YAClB,kCAAkC,WAAW,CAAA,yBAAA;AAAA,WAC/C;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,mBAAA,CAAoB,WAAW,CAAA;AAClD,UAAA;AAAA,QACF;AACE,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,WAAW,CAAA,2BAAA,EAA8B,UAAU,CAAA,CAAA;AAAA,WACvF;AAAA;AACJ,IACF,CAAA,MAAO;AACL,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,QAClB,CAAA,oFAAA,EAAuF,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,iBAAiB,CAAA,EAAA;AAAA,OAChI;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,gBAAA,GAAmB;AACvB,IAAA,MAAM,YAAA,GAAe,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAC3D,IAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,YAAY,CAAA;AACxE,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,QAClB,CAAA,6CAAA,EAAgD,OAAO,EAAE,CAAA,CAAA;AAAA,OAC3D;AACA,MAAA,OAAO;AAAA,QACL,aAAa,MAAA,CAAO,EAAA;AAAA,QACpB,YAAY,MAAA,CAAO,WAAA;AAAA,QACnB,UAAU,MAAA,CAAO,QAAA;AAAA,QACjB,YAAA,EAAc,MAAA,CAAO,cAAA,CAAe,OAAA;AAAQ,OAC9C;AAAA,IACF;AACA,IAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,6BAAA;AAAA,MAChC;AAAA,KACF;AACA,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,QAClB,CAAA,+CAAA,EAAkD,OAAO,WAAW,CAAA,CAAA;AAAA,OACtE;AAAA,IACF;AACA,IAAA,OAAO,MAAA;AAAA,EACT;AAAA,EAEA,MAAM,cAAA,CAAe,EAAA,EAAY,MAAA,EAAqB;AACpD,IAAA,MAAM,QAAA,GAAW,MAAM,IAAA,CAAK,OAAA,CAAQ,YAAY,EAAE,CAAA;AAElD,IAAA,MAAM,MAAA,GAAS,QAAA,GAAW,QAAA,CAAS,MAAA,GAAS,MAAA;AAC5C,IAAA,IAAI,QAAA,GAAW,QAAA,GAAW,QAAA,CAAS,QAAA,GAAW,CAAA,GAAI,CAAA;AAElD,IAAA,MAAM,KAAA,GAAQC,4BAAY,GAAA,EAAI;AAC9B,IAAA,IAAI,KAAA,GAAQ,CAAA;AACZ,IAAA,IAAI,IAAA,GAAO,KAAA;AACX,IAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,MAClB,kCAAkC,EAAE,CAAA,iBAAA;AAAA,KACtC;AAEA,IAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,MAAA,CAAO,OAAO,OAAA,KAAqB;AAC7D,MAAA,IAAI,OAAO,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,IAAA,CAAK,SAAS,MAAM,CAAA;AAC3D,MAAA,KAAA,EAAA;AACA,MAAA,WAAS;AACP,QAAA,IAAA,GAAO,IAAA,CAAK,IAAA;AACZ,QAAA,MAAM,KAAK,IAAA,CAAK;AAAA,UACd,EAAA;AAAA,UACA,QAAA;AAAA,UACA,UAAU,IAAA,EAAM,QAAA;AAAA,UAChB,MAAM,IAAA,CAAK,IAAA;AAAA,UACX,QAAQ,IAAA,EAAM;AAAA,SACf,CAAA;AACD,QAAA,IAAI,MAAA,CAAO,OAAA,IAAW,IAAA,CAAK,IAAA,EAAM;AAC/B,UAAA;AAAA,QACF,CAAA,MAAA,IACEA,4BAAY,GAAA,EAAI,GAAI,QACpB,IAAA,CAAK,WAAA,CAAY,EAAA,CAAG,cAAc,CAAA,EAClC;AACA,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,YAClB,kCAAkC,EAAE,CAAA,qBAAA,EAAwB,IAAA,CAAK,WAAA,CAAY,SAAS,CAAA,CAAA;AAAA,WACxF;AACA,UAAA;AAAA,QACF,CAAA,MAAO;AACL,UAAA,IAAA,GAAO,MAAM,IAAA,CAAK,OAAA,CAAQ,SAAS,IAAA,CAAK,OAAA,EAAS,KAAK,MAAM,CAAA;AAC5D,UAAA,KAAA,EAAA;AACA,UAAA,QAAA,EAAA;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAED,IAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,MAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,mBAAA,EAAsB,KAAK,eAAe,IAAA,CAAK,KAAA;AAAA,QACjFA,2BAAA,CAAY,KAAI,GAAI;AAAA,OACrB,CAAA,IAAA;AAAA,KACH;AACA,IAAA,OAAO,IAAA;AAAA,EACT;AAAA,EAEA,MAAM,KAAK,OAAA,EAMR;AACD,IAAA,MAAM,EAAE,EAAA,EAAI,QAAA,EAAU,QAAA,EAAU,IAAA,EAAM,QAAO,GAAI,OAAA;AACjD,IAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,MAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,QAAA,EAClC,QAAA,GAAW,SAAS,MAAA,GAAS,CAC/B,CAAA,mBAAA,EACE,MAAA,GAAS,KAAK,SAAA,CAAU,MAAM,CAAA,GAAI,MACpC,WAAW,IAAI,CAAA;AAAA,KACjB;AACA,IAAA,MAAM,SAASC,OAAA,EAAG;AAElB,IAAA,MAAM,IAAA,CAAK,QAAQ,UAAA,CAAW;AAAA,MAC5B,MAAA,EAAQ;AAAA,QACN,EAAA,EAAI,MAAA;AAAA,QACJ,YAAA,EAAc,EAAA;AAAA,QACd,MAAA;AAAA,QACA;AAAA;AACF,KACD,CAAA;AAED,IAAA,IAAI,QAAA,IAAY,QAAA,CAAS,MAAA,GAAS,CAAA,EAAG;AACnC,MAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,kBAAA,CAAmB,MAAA,EAAQ,QAAQ,CAAA;AAAA,IACxD;AAEA,IAAA,MAAM,KAAA,GACJ,QAAA,EAAU,GAAA,CAAI,CAAA,QAAA,MAAa;AAAA,MACzB,GAAG,QAAA;AAAA,MACH,MAAA,EAAQ;AAAA,QACN,GAAG,QAAA,CAAS,MAAA;AAAA,QACZ,QAAA,EAAU;AAAA,UACR,GAAG,SAAS,MAAA,CAAO,QAAA;AAAA,UACnB,WAAA,EAAa;AAAA,YACX,GAAG,QAAA,CAAS,MAAA,CAAO,QAAA,CAAS;AAAA;AAC9B;AACF;AACF,KACF,CAAE,KAAK,EAAC;AAEV,IAAA,MAAM,UAAmC,EAAC;AAE1C,IAAA,IAAI,IAAA,EAAM;AACR,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,QAClB,kCAAkC,EAAE,CAAA,mDAAA;AAAA,OACtC;AACA,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,cAAA;AAAA,QAChC,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAAA,QACtC;AAAA,OACF;AAEA,MAAA,MAAM,EAAE,OAAM,GAAI,MAAA;AAElB,MAAA,IAAI,SAAA,GAAY,IAAA;AAChB,MAAA,IAAI,IAAA,CAAK,QAAQ,4BAAA,EAA8B;AAC7C,QAAA,IAAI,UAAU,CAAA,EAAG;AACf,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,YAClB,kCAAkC,EAAE,CAAA,qCAAA;AAAA,WACtC;AACA,UAAA,SAAA,GAAY,KAAA;AAAA,QACd;AAAA,MACF;AAEA,MAAA,IAAI,IAAA,CAAK,QAAQ,6BAAA,EAA+B;AAG9C,QAAA,MAAM,iBACJ,KAAA,GAAQ,CAAA,GAAK,OAAO,OAAA,CAAQ,MAAA,GAAS,QAAS,GAAA,GAAM,GAAA;AACtD,QAAA,IAAI,cAAA,IAAkB,IAAA,CAAK,OAAA,CAAQ,6BAAA,EAA+B;AAChE,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,YAAA,EAAe,MAAA,CAAO,QAAQ,MAAM,CAAA,sCAAA;AAAA,WAC1E;AAAA,QACF,CAAA,MAAO;AACL,UAAA,MAAM,MAAA,GAAS,uBAAuB,cAAc,CAAA,uBAAA,CAAA;AACpD,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,GAAA,EAAM,MAAM,CAAA;AAAA,WAClD;AACA,UAAA,MAAM,IAAA,CAAK,QAAQ,yBAAA,CAA0B;AAAA,YAC3C,WAAA,EAAa,EAAA;AAAA,YACb,MAAA,EAAQ;AAAA,cACN,UAAA,EAAY,CAAA,6CAAA,EAAgD,MAAM,CAAA,EAAA,EAAK,MAAM,CAAA;AAAA;AAC/E,WACD,CAAA;AACD,UAAA,SAAA,GAAY,KAAA;AAAA,QACd;AAAA,MACF;AACA,MAAA,IAAI,SAAA,EAAW;AACb,QAAA,KAAA,MAAW,SAAA,IAAa,OAAO,OAAA,EAAS;AACtC,UAAA,OAAA,CAAQ,KAAK,SAAS,CAAA;AAAA,QACxB;AAAA,MACF;AAAA,IACF;AAEA,IAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,UAAA,CAAW,aAAA,CAAc;AAAA,MAC1C,IAAA,EAAM,OAAA;AAAA,MACN,KAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ,MAAA,EAAoC;AAChD,IAAA,MAAM,EAAE,OAAM,GAAI,MAAA;AAClB,IAAA,IAAI,CAAC,IAAA,CAAK,mBAAA,EAAoB,CAAE,QAAA,CAAS,KAAK,CAAA,EAAG;AAC/C,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,EAAE,MAAA,EAAQ,QAAA,EAAU,UAAA,KAAe,IAAA,CAAK,OAAA;AAC9C,IAAA,MAAM,YAAA,GAAe,SAAS,eAAA,EAAgB;AAC9C,IAAA,MAAA,CAAO,KAAA,CAAM,CAAA,oBAAA,EAAuB,YAAY,CAAA,UAAA,EAAa,KAAK,CAAA,MAAA,CAAQ,CAAA;AAE1E,IAAA,IAAI,CAAC,SAAS,YAAA,EAAc;AAC1B,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,MAAA,GAAS,MAAM,QAAA,CAAS,YAAA,CAAa,QAAQ,MAAM,CAAA;AAEzD,IAAA,IAAI,MAAA,CAAO,SAAS,OAAA,EAAS;AAC3B,MAAA,IAAI,MAAA,CAAO,KAAA,CAAM,MAAA,GAAS,CAAA,EAAG;AAC3B,QAAA,MAAM,eAAA,GAAkB,MAAM,IAAA,CAAK,OAAA,CAAQ,yBAAA;AAAA,UACzC;AAAA,SACF;AAEA,QAAA,IAAI,CAAC,eAAA,EAAiB;AACpB,UAAA,MAAA,CAAO,KAAA;AAAA,YACL,uBAAuB,YAAY,CAAA,qEAAA;AAAA,WACrC;AAAA,QACF,CAAA,MAAO;AACL,UAAA,MAAM,OACJ,eAAA,CAAgB,MAAA,KAAW,SAAA,GACvB,MAAM,KAAK,OAAA,CAAQ,WAAA,CAAY,eAAA,CAAgB,EAAE,IACjD,MAAM,IAAA,CAAK,OAAA,CAAQ,YAAA,CAAa,gBAAgB,EAAE,CAAA;AAExD,UAAA,IAAI,CAAC,IAAA,EAAM;AACT,YAAA,MAAM,IAAI,KAAA;AAAA,cACR,yFAAyF,YAAY,CAAA,CAAA;AAAA,aACvG;AAAA,UACF;AACA,UAAA,MAAM,KAAK,OAAA,CAAQ,kBAAA,CAAmB,IAAA,CAAK,EAAA,EAAI,OAAO,KAAK,CAAA;AAAA,QAC7D;AAAA,MACF;AAEA,MAAA,IAAI,MAAA,CAAO,OAAA,CAAQ,MAAA,GAAS,CAAA,EAAG;AAC7B,QAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,wBAAA,CAAyB,MAAA,CAAO,OAAO,CAAA;AAAA,MAC5D;AAEA,MAAA,MAAM,UAAA,CAAW,cAAc,MAAM,CAAA;AACrC,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,CAAA,oBAAA,EAAuB,YAAY,CAAA,uBAAA,EAA0B,KAAK,CAAA,OAAA;AAAA,OACpE;AAAA,IACF,CAAA,MAAO;AACL,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,CAAA,oBAAA,EAAuB,YAAY,CAAA,2BAAA,EAA8B,KAAK,CAAA,CAAA;AAAA,OACxE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,mBAAA,GAAgC;AAC9B,IAAA,MAAM,EAAE,QAAA,EAAS,GAAI,IAAA,CAAK,OAAA;AAC1B,IAAA,MAAM,SAAS,QAAA,CAAS,YAAA,GACpB,SAAS,YAAA,CAAa,mBAAA,KACtB,EAAC;AACL,IAAA,OAAO,MAAA;AAAA,EACT;AACF;;;;"}

package/dist/module/WrapperProviders.cjs.js CHANGED Viewed

@@ -58,7 +58,8 @@ class WrapperProviders {
         logger,
         provider,
         restLength,
-        connection
+        connection,
+        metrics: this.options.metrics
       });
       let frequency = luxon.Duration.isDuration(burstInterval) ? burstInterval : luxon.Duration.fromObject(burstInterval);
       if (frequency.as("milliseconds") < 5e3) {

package/dist/module/WrapperProviders.cjs.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"WrapperProviders.cjs.js","sources":["../../src/module/WrapperProviders.ts"],"sourcesContent":["/\n Copyright 2022 The Backstage Authors\n \n Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n \n http://www.apache.org/licenses/LICENSE-2.0\n \n Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n /\n\nimport {\n LoggerService,\n RootConfigService,\n SchedulerService,\n} from '@backstage/backend-plugin-api';\nimport { stringifyError } from '@backstage/errors';\nimport {\n EntityProvider,\n EntityProviderConnection,\n} from '@backstage/plugin-catalog-node';\nimport { createDeferred } from '@backstage/types';\nimport express from 'express';\nimport { Knex } from 'knex';\nimport { Duration } from 'luxon';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { applyDatabaseMigrations } from '../database/migrations';\nimport { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine';\nimport { IncrementalProviderRouter } from '../router/routes';\nimport {\n IncrementalEntityProvider,\n IncrementalEntityProviderOptions,\n} from '../types';\nimport { EventsService } from '@backstage/plugin-events-node';\n\n/\n Helps in the creation of the catalog entity providers that wrap the\n * incremental ones.\n */\nexport class WrapperProviders {\n private migrate: Promise<void> \| undefined;\n private numberOfProvidersToConnect = 0;\n private readonly readySignal = createDeferred();\n\n constructor(\n private readonly options: {\n config: RootConfigService;\n logger: LoggerService;\n client: Knex;\n scheduler: SchedulerService;\n applyDatabaseMigrations?: typeof applyDatabaseMigrations;\n events: EventsService;\n },\n ) {}\n\n wrap(\n provider: IncrementalEntityProvider<unknown, unknown>,\n options: IncrementalEntityProviderOptions,\n ): EntityProvider {\n this.numberOfProvidersToConnect += 1;\n return {\n getProviderName: () => provider.getProviderName(),\n connect: async connection => {\n await this.startProvider(provider, options, connection);\n this.numberOfProvidersToConnect -= 1;\n if (this.numberOfProvidersToConnect === 0) {\n this.readySignal.resolve();\n }\n },\n };\n }\n\n adminRouter(): express.Router {\n return new IncrementalProviderRouter(\n new IncrementalIngestionDatabaseManager({ client: this.options.client }),\n this.options.logger,\n ).createRouter();\n }\n\n private async startProvider(\n provider: IncrementalEntityProvider<unknown, unknown>,\n providerOptions: IncrementalEntityProviderOptions,\n connection: EntityProviderConnection,\n ) {\n const logger = this.options.logger.child({\n entityProvider: provider.getProviderName(),\n });\n\n try {\n if (!this.migrate) {\n this.migrate = Promise.resolve().then(async () => {\n const apply =\n this.options.applyDatabaseMigrations ?? applyDatabaseMigrations;\n await apply(this.options.client);\n });\n }\n\n await this.migrate;\n\n const { burstInterval, burstLength, restLength } = providerOptions;\n\n logger.info(`Connecting`);\n\n const manager = new IncrementalIngestionDatabaseManager({\n client: this.options.client,\n });\n const engine = new IncrementalIngestionEngine({\n ...providerOptions,\n ready: this.readySignal,\n manager,\n logger,\n provider,\n restLength,\n connection,\n });\n\n let frequency = Duration.isDuration(burstInterval)\n ? burstInterval\n : Duration.fromObject(burstInterval);\n if (frequency.as('milliseconds') < 5000) {\n frequency = Duration.fromObject({ seconds: 5 }); // don't let it be silly low, to not overload the scheduler\n }\n\n let length = Duration.isDuration(burstLength)\n ? burstLength\n : Duration.fromObject(burstLength);\n length = length.plus(Duration.fromObject({ minutes: 1 })); // some margin from the actual completion\n\n await this.options.scheduler.scheduleTask({\n id: provider.getProviderName(),\n fn: engine.taskFn.bind(engine),\n frequency,\n timeout: length,\n });\n\n const topics = engine.supportsEventTopics();\n if (topics.length > 0) {\n logger.info(\n `Provider ${provider.getProviderName()} subscribing to events for topics: ${topics.join(\n ',',\n )}`,\n );\n await this.options.events.subscribe({\n topics,\n id: `catalog-backend-module-incremental-ingestion:${provider.getProviderName()}`,\n onEvent: evt => engine.onEvent(evt),\n });\n }\n } catch (error) {\n logger.warn(\n `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${stringifyError(\n error,\n )}`,\n );\n throw error;\n }\n }\n}\n"],"names":["createDeferred","IncrementalProviderRouter","IncrementalIngestionDatabaseManager","applyDatabaseMigrations","IncrementalIngestionEngine","Duration","stringifyError"],"mappings":";;;;;;;;;;~~AA4CO~~,MAAM,gBAAA,CAAiB;AAAA,EAK5B,YACmB,OAAA,~~EAQjB~~;~~AARiB~~,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA,~~EAQhB~~;AAAA,~~EAbK~~,OAAA;AAAA,EACA,0BAAA,GAA6B,CAAA;AAAA,EACpB,cAAcA,oBAAA,EAAe;AAAA,~~EAa9C~~,IAAA,CACE,UACA,OAAA,EACgB;AAChB,IAAA,IAAA,CAAK,0BAAA,IAA8B,CAAA;AACnC,IAAA,OAAO;AAAA,MACL,eAAA,EAAiB,MAAM,QAAA,CAAS,eAAA,EAAgB;AAAA,MAChD,OAAA,EAAS,OAAM,UAAA,KAAc;AAC3B,QAAA,MAAM,IAAA,CAAK,aAAA,CAAc,QAAA,EAAU,OAAA,EAAS,UAAU,CAAA;AACtD,QAAA,IAAA,CAAK,0BAAA,IAA8B,CAAA;AACnC,QAAA,IAAI,IAAA,CAAK,+BAA+B,CAAA,EAAG;AACzC,UAAA,IAAA,CAAK,YAAY,OAAA,EAAQ;AAAA,QAC3B;AAAA,MACF;AAAA,KACF;AAAA,EACF;AAAA,EAEA,WAAA,GAA8B;AAC5B,IAAA,OAAO,IAAIC,gCAAA;AAAA,MACT,IAAIC,uEAAA,CAAoC,EAAE,QAAQ,IAAA,CAAK,OAAA,CAAQ,QAAQ,CAAA;AAAA,MACvE,KAAK,OAAA,CAAQ;AAAA,MACb,YAAA,EAAa;AAAA,EACjB;AAAA,EAEA,MAAc,aAAA,CACZ,QAAA,EACA,eAAA,EACA,UAAA,EACA;AACA,IAAA,MAAM,MAAA,GAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM;AAAA,MACvC,cAAA,EAAgB,SAAS,eAAA;AAAgB,KAC1C,CAAA;AAED,IAAA,IAAI;AACF,MAAA,IAAI,CAAC,KAAK,OAAA,EAAS;AACjB,QAAA,IAAA,CAAK,OAAA,GAAU,OAAA,CAAQ,OAAA,EAAQ,CAAE,KAAK,YAAY;AAChD,UAAA,MAAM,KAAA,GACJ,IAAA,CAAK,OAAA,CAAQ,uBAAA,IAA2BC,kCAAA;AAC1C,UAAA,MAAM,KAAA,CAAM,IAAA,CAAK,OAAA,CAAQ,MAAM,CAAA;AAAA,QACjC,CAAC,CAAA;AAAA,MACH;AAEA,MAAA,MAAM,IAAA,CAAK,OAAA;AAEX,MAAA,MAAM,EAAE,aAAA,EAAe,WAAA,EAAa,UAAA,EAAW,GAAI,eAAA;AAEnD,MAAA,MAAA,CAAO,KAAK,CAAA,UAAA,CAAY,CAAA;AAExB,MAAA,MAAM,OAAA,GAAU,IAAID,uEAAA,CAAoC;AAAA,QACtD,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,MAAM,MAAA,GAAS,IAAIE,qDAAA,CAA2B;AAAA,QAC5C,GAAG,eAAA;AAAA,QACH,OAAO,IAAA,CAAK,WAAA;AAAA,QACZ,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA;AAAA,~~OACD~~,CAAA;AAED,MAAA,IAAI,SAAA,GAAYC,eAAS,UAAA,CAAW,aAAa,IAC7C,aAAA,GACAA,cAAA,CAAS,WAAW,aAAa,CAAA;AACrC,MAAA,IAAI,SAAA,CAAU,EAAA,CAAG,cAAc,CAAA,GAAI,GAAA,EAAM;AACvC,QAAA,SAAA,GAAYA,cAAA,CAAS,UAAA,CAAW,EAAE,OAAA,EAAS,GAAG,CAAA;AAAA,MAChD;AAEA,MAAA,IAAI,MAAA,GAASA,eAAS,UAAA,CAAW,WAAW,IACxC,WAAA,GACAA,cAAA,CAAS,WAAW,WAAW,CAAA;AACnC,MAAA,MAAA,GAAS,MAAA,CAAO,KAAKA,cAAA,CAAS,UAAA,CAAW,EAAE,OAAA,EAAS,CAAA,EAAG,CAAC,CAAA;AAExD,MAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa;AAAA,QACxC,EAAA,EAAI,SAAS,eAAA,EAAgB;AAAA,QAC7B,EAAA,EAAI,MAAA,CAAO,MAAA,CAAO,IAAA,CAAK,MAAM,CAAA;AAAA,QAC7B,SAAA;AAAA,QACA,OAAA,EAAS;AAAA,OACV,CAAA;AAED,MAAA,MAAM,MAAA,GAAS,OAAO,mBAAA,EAAoB;AAC1C,MAAA,IAAI,MAAA,CAAO,SAAS,CAAA,EAAG;AACrB,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,SAAA,EAAY,QAAA,CAAS,eAAA,EAAiB,sCAAsC,MAAA,CAAO,IAAA;AAAA,YACjF;AAAA,WACD,CAAA;AAAA,SACH;AACA,QAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,SAAA,CAAU;AAAA,UAClC,MAAA;AAAA,UACA,EAAA,EAAI,CAAA,6CAAA,EAAgD,QAAA,CAAS,eAAA,EAAiB,CAAA,CAAA;AAAA,UAC9E,OAAA,EAAS,CAAA,GAAA,KAAO,MAAA,CAAO,OAAA,CAAQ,GAAG;AAAA,SACnC,CAAA;AAAA,MACH;AAAA,IACF,SAAS,KAAA,EAAO;AACd,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,oDAAA,EAAuD,QAAA,CAAS,eAAA,EAAiB,CAAA,EAAA,EAAKC,qBAAA;AAAA,UACpF;AAAA,SACD,CAAA;AAAA,OACH;AACA,MAAA,MAAM,KAAA;AAAA,IACR;AAAA,EACF;AACF;;;;"}
1	+ {"version":3,"file":"WrapperProviders.cjs.js","sources":["../../src/module/WrapperProviders.ts"],"sourcesContent":["/\n Copyright 2022 The Backstage Authors\n \n Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n \n http://www.apache.org/licenses/LICENSE-2.0\n \n Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n /\n\nimport {\n LoggerService,\n RootConfigService,\n SchedulerService,\n} from '@backstage/backend-plugin-api';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\nimport { stringifyError } from '@backstage/errors';\nimport {\n EntityProvider,\n EntityProviderConnection,\n} from '@backstage/plugin-catalog-node';\nimport { createDeferred } from '@backstage/types';\nimport express from 'express';\nimport { Knex } from 'knex';\nimport { Duration } from 'luxon';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { applyDatabaseMigrations } from '../database/migrations';\nimport { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine';\nimport { IncrementalProviderRouter } from '../router/routes';\nimport {\n IncrementalEntityProvider,\n IncrementalEntityProviderOptions,\n} from '../types';\nimport { EventsService } from '@backstage/plugin-events-node';\n\n/\n Helps in the creation of the catalog entity providers that wrap the\n * incremental ones.\n */\nexport class WrapperProviders {\n private migrate: Promise<void> \| undefined;\n private numberOfProvidersToConnect = 0;\n private readonly readySignal = createDeferred();\n\n constructor(\n private readonly options: {\n config: RootConfigService;\n logger: LoggerService;\n client: Knex;\n scheduler: SchedulerService;\n applyDatabaseMigrations?: typeof applyDatabaseMigrations;\n events: EventsService;\n metrics: MetricsService;\n },\n ) {}\n\n wrap(\n provider: IncrementalEntityProvider<unknown, unknown>,\n options: IncrementalEntityProviderOptions,\n ): EntityProvider {\n this.numberOfProvidersToConnect += 1;\n return {\n getProviderName: () => provider.getProviderName(),\n connect: async connection => {\n await this.startProvider(provider, options, connection);\n this.numberOfProvidersToConnect -= 1;\n if (this.numberOfProvidersToConnect === 0) {\n this.readySignal.resolve();\n }\n },\n };\n }\n\n adminRouter(): express.Router {\n return new IncrementalProviderRouter(\n new IncrementalIngestionDatabaseManager({ client: this.options.client }),\n this.options.logger,\n ).createRouter();\n }\n\n private async startProvider(\n provider: IncrementalEntityProvider<unknown, unknown>,\n providerOptions: IncrementalEntityProviderOptions,\n connection: EntityProviderConnection,\n ) {\n const logger = this.options.logger.child({\n entityProvider: provider.getProviderName(),\n });\n\n try {\n if (!this.migrate) {\n this.migrate = Promise.resolve().then(async () => {\n const apply =\n this.options.applyDatabaseMigrations ?? applyDatabaseMigrations;\n await apply(this.options.client);\n });\n }\n\n await this.migrate;\n\n const { burstInterval, burstLength, restLength } = providerOptions;\n\n logger.info(`Connecting`);\n\n const manager = new IncrementalIngestionDatabaseManager({\n client: this.options.client,\n });\n const engine = new IncrementalIngestionEngine({\n ...providerOptions,\n ready: this.readySignal,\n manager,\n logger,\n provider,\n restLength,\n connection,\n metrics: this.options.metrics,\n });\n\n let frequency = Duration.isDuration(burstInterval)\n ? burstInterval\n : Duration.fromObject(burstInterval);\n if (frequency.as('milliseconds') < 5000) {\n frequency = Duration.fromObject({ seconds: 5 }); // don't let it be silly low, to not overload the scheduler\n }\n\n let length = Duration.isDuration(burstLength)\n ? burstLength\n : Duration.fromObject(burstLength);\n length = length.plus(Duration.fromObject({ minutes: 1 })); // some margin from the actual completion\n\n await this.options.scheduler.scheduleTask({\n id: provider.getProviderName(),\n fn: engine.taskFn.bind(engine),\n frequency,\n timeout: length,\n });\n\n const topics = engine.supportsEventTopics();\n if (topics.length > 0) {\n logger.info(\n `Provider ${provider.getProviderName()} subscribing to events for topics: ${topics.join(\n ',',\n )}`,\n );\n await this.options.events.subscribe({\n topics,\n id: `catalog-backend-module-incremental-ingestion:${provider.getProviderName()}`,\n onEvent: evt => engine.onEvent(evt),\n });\n }\n } catch (error) {\n logger.warn(\n `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${stringifyError(\n error,\n )}`,\n );\n throw error;\n }\n }\n}\n"],"names":["createDeferred","IncrementalProviderRouter","IncrementalIngestionDatabaseManager","applyDatabaseMigrations","IncrementalIngestionEngine","Duration","stringifyError"],"mappings":";;;;;;;;;;AA6CO,MAAM,gBAAA,CAAiB;AAAA,EAK5B,YACmB,OAAA,EASjB;AATiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA,EAShB;AAAA,EAdK,OAAA;AAAA,EACA,0BAAA,GAA6B,CAAA;AAAA,EACpB,cAAcA,oBAAA,EAAe;AAAA,EAc9C,IAAA,CACE,UACA,OAAA,EACgB;AAChB,IAAA,IAAA,CAAK,0BAAA,IAA8B,CAAA;AACnC,IAAA,OAAO;AAAA,MACL,eAAA,EAAiB,MAAM,QAAA,CAAS,eAAA,EAAgB;AAAA,MAChD,OAAA,EAAS,OAAM,UAAA,KAAc;AAC3B,QAAA,MAAM,IAAA,CAAK,aAAA,CAAc,QAAA,EAAU,OAAA,EAAS,UAAU,CAAA;AACtD,QAAA,IAAA,CAAK,0BAAA,IAA8B,CAAA;AACnC,QAAA,IAAI,IAAA,CAAK,+BAA+B,CAAA,EAAG;AACzC,UAAA,IAAA,CAAK,YAAY,OAAA,EAAQ;AAAA,QAC3B;AAAA,MACF;AAAA,KACF;AAAA,EACF;AAAA,EAEA,WAAA,GAA8B;AAC5B,IAAA,OAAO,IAAIC,gCAAA;AAAA,MACT,IAAIC,uEAAA,CAAoC,EAAE,QAAQ,IAAA,CAAK,OAAA,CAAQ,QAAQ,CAAA;AAAA,MACvE,KAAK,OAAA,CAAQ;AAAA,MACb,YAAA,EAAa;AAAA,EACjB;AAAA,EAEA,MAAc,aAAA,CACZ,QAAA,EACA,eAAA,EACA,UAAA,EACA;AACA,IAAA,MAAM,MAAA,GAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM;AAAA,MACvC,cAAA,EAAgB,SAAS,eAAA;AAAgB,KAC1C,CAAA;AAED,IAAA,IAAI;AACF,MAAA,IAAI,CAAC,KAAK,OAAA,EAAS;AACjB,QAAA,IAAA,CAAK,OAAA,GAAU,OAAA,CAAQ,OAAA,EAAQ,CAAE,KAAK,YAAY;AAChD,UAAA,MAAM,KAAA,GACJ,IAAA,CAAK,OAAA,CAAQ,uBAAA,IAA2BC,kCAAA;AAC1C,UAAA,MAAM,KAAA,CAAM,IAAA,CAAK,OAAA,CAAQ,MAAM,CAAA;AAAA,QACjC,CAAC,CAAA;AAAA,MACH;AAEA,MAAA,MAAM,IAAA,CAAK,OAAA;AAEX,MAAA,MAAM,EAAE,aAAA,EAAe,WAAA,EAAa,UAAA,EAAW,GAAI,eAAA;AAEnD,MAAA,MAAA,CAAO,KAAK,CAAA,UAAA,CAAY,CAAA;AAExB,MAAA,MAAM,OAAA,GAAU,IAAID,uEAAA,CAAoC;AAAA,QACtD,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,MAAM,MAAA,GAAS,IAAIE,qDAAA,CAA2B;AAAA,QAC5C,GAAG,eAAA;AAAA,QACH,OAAO,IAAA,CAAK,WAAA;AAAA,QACZ,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,UAAA;AAAA,QACA,OAAA,EAAS,KAAK,OAAA,CAAQ;AAAA,OACvB,CAAA;AAED,MAAA,IAAI,SAAA,GAAYC,eAAS,UAAA,CAAW,aAAa,IAC7C,aAAA,GACAA,cAAA,CAAS,WAAW,aAAa,CAAA;AACrC,MAAA,IAAI,SAAA,CAAU,EAAA,CAAG,cAAc,CAAA,GAAI,GAAA,EAAM;AACvC,QAAA,SAAA,GAAYA,cAAA,CAAS,UAAA,CAAW,EAAE,OAAA,EAAS,GAAG,CAAA;AAAA,MAChD;AAEA,MAAA,IAAI,MAAA,GAASA,eAAS,UAAA,CAAW,WAAW,IACxC,WAAA,GACAA,cAAA,CAAS,WAAW,WAAW,CAAA;AACnC,MAAA,MAAA,GAAS,MAAA,CAAO,KAAKA,cAAA,CAAS,UAAA,CAAW,EAAE,OAAA,EAAS,CAAA,EAAG,CAAC,CAAA;AAExD,MAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa;AAAA,QACxC,EAAA,EAAI,SAAS,eAAA,EAAgB;AAAA,QAC7B,EAAA,EAAI,MAAA,CAAO,MAAA,CAAO,IAAA,CAAK,MAAM,CAAA;AAAA,QAC7B,SAAA;AAAA,QACA,OAAA,EAAS;AAAA,OACV,CAAA;AAED,MAAA,MAAM,MAAA,GAAS,OAAO,mBAAA,EAAoB;AAC1C,MAAA,IAAI,MAAA,CAAO,SAAS,CAAA,EAAG;AACrB,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,SAAA,EAAY,QAAA,CAAS,eAAA,EAAiB,sCAAsC,MAAA,CAAO,IAAA;AAAA,YACjF;AAAA,WACD,CAAA;AAAA,SACH;AACA,QAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,SAAA,CAAU;AAAA,UAClC,MAAA;AAAA,UACA,EAAA,EAAI,CAAA,6CAAA,EAAgD,QAAA,CAAS,eAAA,EAAiB,CAAA,CAAA;AAAA,UAC9E,OAAA,EAAS,CAAA,GAAA,KAAO,MAAA,CAAO,OAAA,CAAQ,GAAG;AAAA,SACnC,CAAA;AAAA,MACH;AAAA,IACF,SAAS,KAAA,EAAO;AACd,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,oDAAA,EAAuD,QAAA,CAAS,eAAA,EAAiB,CAAA,EAAA,EAAKC,qBAAA;AAAA,UACpF;AAAA,SACD,CAAA;AAAA,OACH;AACA,MAAA,MAAM,KAAA;AAAA,IACR;AAAA,EACF;AACF;;;;"}

package/dist/module/catalogModuleIncrementalIngestionEntityProvider.cjs.js CHANGED Viewed

@@ -1,6 +1,7 @@
 'use strict';
 var backendPluginApi = require('@backstage/backend-plugin-api');
+var alpha = require('@backstage/backend-plugin-api/alpha');
 var pluginCatalogNode = require('@backstage/plugin-catalog-node');
 var WrapperProviders = require('./WrapperProviders.cjs.js');
 var pluginEventsNode = require('@backstage/plugin-events-node');
@@ -26,7 +27,8 @@ const catalogModuleIncrementalIngestionEntityProvider = backendPluginApi.createB
         httpRouter: backendPluginApi.coreServices.httpRouter,
         logger: backendPluginApi.coreServices.logger,
         scheduler: backendPluginApi.coreServices.scheduler,
-        events: pluginEventsNode.eventsServiceRef
+        events: pluginEventsNode.eventsServiceRef,
+        metrics: alpha.metricsServiceRef
       },
       async init({
         catalog,
@@ -35,7 +37,8 @@ const catalogModuleIncrementalIngestionEntityProvider = backendPluginApi.createB
         httpRouter,
         logger,
         scheduler,
-        events
+        events,
+        metrics
       }) {
         const client = await database.getClient();
         const providers = new WrapperProviders.WrapperProviders({
@@ -43,7 +46,8 @@ const catalogModuleIncrementalIngestionEntityProvider = backendPluginApi.createB
           logger,
           client,
           scheduler,
-          events
+          events,
+          metrics
         });
         for (const entry of addedProviders) {
           const wrapped = providers.wrap(entry.provider, entry.options);

package/dist/module/catalogModuleIncrementalIngestionEntityProvider.cjs.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"catalogModuleIncrementalIngestionEntityProvider.cjs.js","sources":["../../src/module/catalogModuleIncrementalIngestionEntityProvider.ts"],"sourcesContent":["/\n Copyright 2022 The Backstage Authors\n \n Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n \n http://www.apache.org/licenses/LICENSE-2.0\n \n Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n /\n\nimport {\n coreServices,\n createBackendModule,\n createExtensionPoint,\n} from '@backstage/backend-plugin-api';\nimport { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node';\nimport { WrapperProviders } from './WrapperProviders';\nimport { eventsServiceRef } from '@backstage/plugin-events-node';\nimport {\n IncrementalEntityProvider,\n IncrementalEntityProviderOptions,\n} from '../types';\n\n/\n @public\n * Interface for {@link incrementalIngestionProvidersExtensionPoint}.\n /\nexport interface IncrementalIngestionProviderExtensionPoint {\n /* Adds a new incremental entity provider /\n addProvider<TCursor, TContext>(config: {\n options: IncrementalEntityProviderOptions;\n provider: IncrementalEntityProvider<TCursor, TContext>;\n }): void;\n}\n\n/\n @public\n \n Extension point for registering incremental ingestion providers.\n * The `catalogModuleIncrementalIngestionEntityProvider` must be installed for these providers to work.\n \n @example\n \n ```ts\n * backend.add(createBackendModule({\n * pluginId: 'catalog',\n * moduleId: 'my-incremental-provider',\n * register(env) {\n * env.registerInit({\n * deps: {\n * extension: incrementalIngestionProvidersExtensionPoint,\n * },\n * async init({ extension }) {\n * extension.addProvider({\n * burstInterval: ...,\n * burstLength: ...,\n * restLength: ...,\n * }, {\n * next(context, cursor) {\n * ...\n * },\n * ...\n * })\n * })\n * })\n * }\n * }))\n * ```\n /\nexport const incrementalIngestionProvidersExtensionPoint =\n createExtensionPoint<IncrementalIngestionProviderExtensionPoint>({\n id: 'catalog.incrementalIngestionProvider.providers',\n });\n\n/\n Registers the incremental entity provider with the catalog processing extension point.\n \n @public\n */\nexport const catalogModuleIncrementalIngestionEntityProvider =\n createBackendModule({\n pluginId: 'catalog',\n moduleId: 'incremental-ingestion-entity-provider',\n register(env) {\n const addedProviders = new Array<{\n provider: IncrementalEntityProvider<unknown, unknown>;\n options: IncrementalEntityProviderOptions;\n }>();\n\n env.registerExtensionPoint(incrementalIngestionProvidersExtensionPoint, {\n addProvider({ options, provider }) {\n addedProviders.push({ options, provider });\n },\n });\n\n env.registerInit({\n deps: {\n catalog: catalogProcessingExtensionPoint,\n config: coreServices.rootConfig,\n database: coreServices.database,\n httpRouter: coreServices.httpRouter,\n logger: coreServices.logger,\n scheduler: coreServices.scheduler,\n events: eventsServiceRef,\n },\n async init({\n catalog,\n config,\n database,\n httpRouter,\n logger,\n scheduler,\n events,\n }) {\n const client = await database.getClient();\n\n const providers = new WrapperProviders({\n config,\n logger,\n client,\n scheduler,\n events,\n });\n\n for (const entry of addedProviders) {\n const wrapped = providers.wrap(entry.provider, entry.options);\n catalog.addEntityProvider(wrapped);\n }\n\n httpRouter.use(providers.adminRouter());\n },\n });\n },\n });\n"],"names":["createExtensionPoint","createBackendModule","catalogProcessingExtensionPoint","coreServices","eventsServiceRef","WrapperProviders"],"mappings":"~~;;;;;;;AA2EO~~,MAAM,8CACXA,qCAAA,CAAiE;AAAA,EAC/D,EAAA,EAAI;AACN,CAAC;AAOI,MAAM,kDACXC,oCAAA,CAAoB;AAAA,EAClB,QAAA,EAAU,SAAA;AAAA,EACV,QAAA,EAAU,uCAAA;AAAA,EACV,SAAS,GAAA,EAAK;AACZ,IAAA,MAAM,cAAA,GAAiB,IAAI,KAAA,EAGxB;AAEH,IAAA,GAAA,CAAI,uBAAuB,2CAAA,EAA6C;AAAA,MACtE,WAAA,CAAY,EAAE,OAAA,EAAS,QAAA,EAAS,EAAG;AACjC,QAAA,cAAA,CAAe,IAAA,CAAK,EAAE,OAAA,EAAS,QAAA,EAAU,CAAA;AAAA,MAC3C;AAAA,KACD,CAAA;AAED,IAAA,GAAA,CAAI,YAAA,CAAa;AAAA,MACf,IAAA,EAAM;AAAA,QACJ,OAAA,EAASC,iDAAA;AAAA,QACT,QAAQC,6BAAA,CAAa,UAAA;AAAA,QACrB,UAAUA,6BAAA,CAAa,QAAA;AAAA,QACvB,YAAYA,6BAAA,CAAa,UAAA;AAAA,QACzB,QAAQA,6BAAA,CAAa,MAAA;AAAA,QACrB,WAAWA,6BAAA,CAAa,SAAA;AAAA,QACxB,MAAA,EAAQC;AAAA,~~OACV~~;AAAA,MACA,MAAM,IAAA,CAAK;AAAA,QACT,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,MAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACF,EAAG;AACD,QAAA,MAAM,MAAA,GAAS,MAAM,QAAA,CAAS,SAAA,EAAU;AAExC,QAAA,MAAM,SAAA,GAAY,IAAIC,iCAAA,CAAiB;AAAA,UACrC,MAAA;AAAA,UACA,MAAA;AAAA,UACA,MAAA;AAAA,UACA,SAAA;AAAA,UACA;AAAA,SACD,CAAA;AAED,QAAA,KAAA,MAAW,SAAS,cAAA,EAAgB;AAClC,UAAA,MAAM,UAAU,SAAA,CAAU,IAAA,CAAK,KAAA,CAAM,QAAA,EAAU,MAAM,OAAO,CAAA;AAC5D,UAAA,OAAA,CAAQ,kBAAkB,OAAO,CAAA;AAAA,QACnC;AAEA,QAAA,UAAA,CAAW,GAAA,CAAI,SAAA,CAAU,WAAA,EAAa,CAAA;AAAA,MACxC;AAAA,KACD,CAAA;AAAA,EACH;AACF,CAAC;;;;;"}
1	+ {"version":3,"file":"catalogModuleIncrementalIngestionEntityProvider.cjs.js","sources":["../../src/module/catalogModuleIncrementalIngestionEntityProvider.ts"],"sourcesContent":["/\n Copyright 2022 The Backstage Authors\n \n Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n \n http://www.apache.org/licenses/LICENSE-2.0\n \n Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n /\n\nimport {\n coreServices,\n createBackendModule,\n createExtensionPoint,\n} from '@backstage/backend-plugin-api';\nimport { metricsServiceRef } from '@backstage/backend-plugin-api/alpha';\nimport { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node';\nimport { WrapperProviders } from './WrapperProviders';\nimport { eventsServiceRef } from '@backstage/plugin-events-node';\nimport {\n IncrementalEntityProvider,\n IncrementalEntityProviderOptions,\n} from '../types';\n\n/\n @public\n * Interface for {@link incrementalIngestionProvidersExtensionPoint}.\n /\nexport interface IncrementalIngestionProviderExtensionPoint {\n /* Adds a new incremental entity provider /\n addProvider<TCursor, TContext>(config: {\n options: IncrementalEntityProviderOptions;\n provider: IncrementalEntityProvider<TCursor, TContext>;\n }): void;\n}\n\n/\n @public\n \n Extension point for registering incremental ingestion providers.\n * The `catalogModuleIncrementalIngestionEntityProvider` must be installed for these providers to work.\n \n @example\n \n ```ts\n * backend.add(createBackendModule({\n * pluginId: 'catalog',\n * moduleId: 'my-incremental-provider',\n * register(env) {\n * env.registerInit({\n * deps: {\n * extension: incrementalIngestionProvidersExtensionPoint,\n * },\n * async init({ extension }) {\n * extension.addProvider({\n * burstInterval: ...,\n * burstLength: ...,\n * restLength: ...,\n * }, {\n * next(context, cursor) {\n * ...\n * },\n * ...\n * })\n * })\n * })\n * }\n * }))\n * ```\n /\nexport const incrementalIngestionProvidersExtensionPoint =\n createExtensionPoint<IncrementalIngestionProviderExtensionPoint>({\n id: 'catalog.incrementalIngestionProvider.providers',\n });\n\n/\n Registers the incremental entity provider with the catalog processing extension point.\n \n @public\n */\nexport const catalogModuleIncrementalIngestionEntityProvider =\n createBackendModule({\n pluginId: 'catalog',\n moduleId: 'incremental-ingestion-entity-provider',\n register(env) {\n const addedProviders = new Array<{\n provider: IncrementalEntityProvider<unknown, unknown>;\n options: IncrementalEntityProviderOptions;\n }>();\n\n env.registerExtensionPoint(incrementalIngestionProvidersExtensionPoint, {\n addProvider({ options, provider }) {\n addedProviders.push({ options, provider });\n },\n });\n\n env.registerInit({\n deps: {\n catalog: catalogProcessingExtensionPoint,\n config: coreServices.rootConfig,\n database: coreServices.database,\n httpRouter: coreServices.httpRouter,\n logger: coreServices.logger,\n scheduler: coreServices.scheduler,\n events: eventsServiceRef,\n metrics: metricsServiceRef,\n },\n async init({\n catalog,\n config,\n database,\n httpRouter,\n logger,\n scheduler,\n events,\n metrics,\n }) {\n const client = await database.getClient();\n\n const providers = new WrapperProviders({\n config,\n logger,\n client,\n scheduler,\n events,\n metrics,\n });\n\n for (const entry of addedProviders) {\n const wrapped = providers.wrap(entry.provider, entry.options);\n catalog.addEntityProvider(wrapped);\n }\n\n httpRouter.use(providers.adminRouter());\n },\n });\n },\n });\n"],"names":["createExtensionPoint","createBackendModule","catalogProcessingExtensionPoint","coreServices","eventsServiceRef","metricsServiceRef","WrapperProviders"],"mappings":";;;;;;;;AA4EO,MAAM,8CACXA,qCAAA,CAAiE;AAAA,EAC/D,EAAA,EAAI;AACN,CAAC;AAOI,MAAM,kDACXC,oCAAA,CAAoB;AAAA,EAClB,QAAA,EAAU,SAAA;AAAA,EACV,QAAA,EAAU,uCAAA;AAAA,EACV,SAAS,GAAA,EAAK;AACZ,IAAA,MAAM,cAAA,GAAiB,IAAI,KAAA,EAGxB;AAEH,IAAA,GAAA,CAAI,uBAAuB,2CAAA,EAA6C;AAAA,MACtE,WAAA,CAAY,EAAE,OAAA,EAAS,QAAA,EAAS,EAAG;AACjC,QAAA,cAAA,CAAe,IAAA,CAAK,EAAE,OAAA,EAAS,QAAA,EAAU,CAAA;AAAA,MAC3C;AAAA,KACD,CAAA;AAED,IAAA,GAAA,CAAI,YAAA,CAAa;AAAA,MACf,IAAA,EAAM;AAAA,QACJ,OAAA,EAASC,iDAAA;AAAA,QACT,QAAQC,6BAAA,CAAa,UAAA;AAAA,QACrB,UAAUA,6BAAA,CAAa,QAAA;AAAA,QACvB,YAAYA,6BAAA,CAAa,UAAA;AAAA,QACzB,QAAQA,6BAAA,CAAa,MAAA;AAAA,QACrB,WAAWA,6BAAA,CAAa,SAAA;AAAA,QACxB,MAAA,EAAQC,iCAAA;AAAA,QACR,OAAA,EAASC;AAAA,OACX;AAAA,MACA,MAAM,IAAA,CAAK;AAAA,QACT,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,MAAA;AAAA,QACA,SAAA;AAAA,QACA,MAAA;AAAA,QACA;AAAA,OACF,EAAG;AACD,QAAA,MAAM,MAAA,GAAS,MAAM,QAAA,CAAS,SAAA,EAAU;AAExC,QAAA,MAAM,SAAA,GAAY,IAAIC,iCAAA,CAAiB;AAAA,UACrC,MAAA;AAAA,UACA,MAAA;AAAA,UACA,MAAA;AAAA,UACA,SAAA;AAAA,UACA,MAAA;AAAA,UACA;AAAA,SACD,CAAA;AAED,QAAA,KAAA,MAAW,SAAS,cAAA,EAAgB;AAClC,UAAA,MAAM,UAAU,SAAA,CAAU,IAAA,CAAK,KAAA,CAAM,QAAA,EAAU,MAAM,OAAO,CAAA;AAC5D,UAAA,OAAA,CAAQ,kBAAkB,OAAO,CAAA;AAAA,QACnC;AAEA,QAAA,UAAA,CAAW,GAAA,CAAI,SAAA,CAAU,WAAA,EAAa,CAAA;AAAA,MACxC;AAAA,KACD,CAAA;AAAA,EACH;AACF,CAAC;;;;;"}

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend-module-incremental-ingestion",
-  "version": "0.7.10-next.1",
+  "version": "0.7.10",
   "description": "An entity provider for streaming large asset sources into the catalog",
   "backstage": {
     "role": "backend-plugin-module",
@@ -55,17 +55,16 @@
     "test": "backstage-cli package test"
   },
   "dependencies": {
-    "@backstage/backend-defaults": "0.16.0-next.1",
-    "@backstage/backend-plugin-api": "1.7.1-next.0",
-    "@backstage/catalog-model": "1.7.6",
-    "@backstage/config": "1.3.6",
-    "@backstage/errors": "1.2.7",
-    "@backstage/plugin-catalog-backend": "3.5.0-next.1",
-    "@backstage/plugin-catalog-node": "2.1.0-next.1",
-    "@backstage/plugin-events-node": "0.4.20-next.0",
-    "@backstage/plugin-permission-common": "0.9.6",
-    "@backstage/types": "1.2.2",
-    "@opentelemetry/api": "^1.9.0",
+    "@backstage/backend-defaults": "^0.16.0",
+    "@backstage/backend-plugin-api": "^1.8.0",
+    "@backstage/catalog-model": "^1.7.7",
+    "@backstage/config": "^1.3.6",
+    "@backstage/errors": "^1.2.7",
+    "@backstage/plugin-catalog-backend": "^3.5.0",
+    "@backstage/plugin-catalog-node": "^2.1.0",
+    "@backstage/plugin-events-node": "^0.4.20",
+    "@backstage/plugin-permission-common": "^0.9.7",
+    "@backstage/types": "^1.2.2",
     "express": "^4.22.0",
     "express-promise-router": "^4.1.0",
     "knex": "^3.0.0",
@@ -73,8 +72,8 @@
     "uuid": "^11.0.0"
   },
   "devDependencies": {
-    "@backstage/backend-test-utils": "1.11.1-next.1",
-    "@backstage/cli": "0.36.0-next.1",
+    "@backstage/backend-test-utils": "^1.11.1",
+    "@backstage/cli": "^0.36.0",
     "@types/express": "^4.17.6"
   }
 }