npm - @backstage/plugin-catalog-backend-module-incremental-ingestion - Versions diffs - 0.6.1-next.0 → 0.6.1-next.1 - Mend

@backstage/plugin-catalog-backend-module-incremental-ingestion 0.6.1-next.0 → 0.6.1-next.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (4) hide show

package/CHANGELOG.md +16 -0
package/dist/engine/IncrementalIngestionEngine.cjs.js +29 -0
package/dist/engine/IncrementalIngestionEngine.cjs.js.map +1 -1
package/package.json +9 -8

package/CHANGELOG.md CHANGED Viewed

@@ -1,5 +1,21 @@
 # @backstage/plugin-catalog-backend-module-incremental-ingestion
+## 0.6.1-next.1
+### Patch Changes
+- fe87fbf: Add task metrics as two gauges that track the last start and end timestamps as epoch seconds.
+- Updated dependencies
+  - @backstage/plugin-catalog-backend@1.29.0-next.1
+  - @backstage/plugin-catalog-node@1.15.0-next.1
+  - @backstage/backend-plugin-api@1.1.0-next.1
+  - @backstage/catalog-model@1.7.1
+  - @backstage/config@1.3.0
+  - @backstage/errors@1.2.5
+  - @backstage/types@1.2.0
+  - @backstage/plugin-events-node@0.4.6-next.1
+  - @backstage/plugin-permission-common@0.8.2
 ## 0.6.1-next.0
 ### Patch Changes

package/dist/engine/IncrementalIngestionEngine.cjs.js CHANGED Viewed

@@ -1,5 +1,6 @@
 'use strict';
+var api = require('@opentelemetry/api');
 var perf_hooks = require('perf_hooks');
 var luxon = require('luxon');
 var uuid = require('uuid');
@@ -8,6 +9,7 @@ var errors = require('@backstage/errors');
 class IncrementalIngestionEngine {
   constructor(options) {
     this.options = options;
+    const meter = api.metrics.getMeter("default");
     this.manager = options.manager;
     this.restLength = luxon.Duration.fromObject(options.restLength);
     this.backoff = options.backoff ?? [
@@ -16,9 +18,25 @@ class IncrementalIngestionEngine {
       { minutes: 30 },
       { hours: 3 }
     ];
+    this.lastStarted = meter.createGauge(
+      "catalog_incremental.ingestions.started",
+      {
+        description: "Epoch timestamp seconds when the ingestion was last started",
+        unit: "seconds"
+      }
+    );
+    this.lastCompleted = meter.createGauge(
+      "catalog_incremental.ingestions.completed",
+      {
+        description: "Epoch timestamp seconds when the ingestion was last completed",
+        unit: "seconds"
+      }
+    );
   }
   restLength;
   backoff;
+  lastStarted;
+  lastCompleted;
   manager;
   async taskFn(signal) {
     try {
@@ -45,6 +63,9 @@ class IncrementalIngestionEngine {
             this.options.logger.debug(
               `incremental-engine: Ingestion ${ingestionId} rest period complete. Ingestion will start again`
             );
+            this.lastStarted.record(Date.now() / 1e3, {
+              providerName: this.options.provider.getProviderName()
+            });
             await this.manager.setProviderComplete(ingestionId);
           } else {
             this.options.logger.debug(
@@ -60,6 +81,10 @@ class IncrementalIngestionEngine {
               this.options.logger.info(
                 `incremental-engine: Ingestion '${ingestionId}' complete, transitioning to rest period of ${this.restLength.toHuman()}`
               );
+              this.lastCompleted.record(Date.now() / 1e3, {
+                providerName: this.options.provider.getProviderName(),
+                status: "completed"
+              });
               await this.manager.setProviderResting(
                 ingestionId,
                 this.restLength
@@ -92,6 +117,10 @@ class IncrementalIngestionEngine {
               this.options.logger.error(
                 `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`
               );
+              this.lastCompleted.record(Date.now() / 1e3, {
+                providerName: this.options.provider.getProviderName(),
+                status: "failed"
+              });
               await this.manager.setProviderBackoff(
                 ingestionId,
                 attempts,

package/dist/engine/IncrementalIngestionEngine.cjs.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"IncrementalIngestionEngine.cjs.js","sources":["../../src/engine/IncrementalIngestionEngine.ts"],"sourcesContent":["/\n Copyright 2022 The Backstage Authors\n \n Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n \n http://www.apache.org/licenses/LICENSE-2.0\n \n Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n /\n\nimport type { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { IterationEngine, IterationEngineOptions } from '../types';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { performance } from 'perf_hooks';\nimport { Duration } from 'luxon';\nimport { v4 } from 'uuid';\nimport { stringifyError } from '@backstage/errors';\nimport { EventParams } from '@backstage/plugin-events-node';\nimport { HumanDuration } from '@backstage/types';\n\nexport class IncrementalIngestionEngine implements IterationEngine {\n private readonly restLength: Duration;\n private readonly backoff: HumanDuration[];\n\n private manager: IncrementalIngestionDatabaseManager;\n\n constructor(private options: IterationEngineOptions) {\n this.manager = options.manager;\n this.restLength = Duration.fromObject(options.restLength);\n this.backoff = options.backoff ?? [\n { minutes: 1 },\n { minutes: 5 },\n { minutes: 30 },\n { hours: 3 },\n ];\n }\n\n async taskFn(signal: AbortSignal) {\n try {\n this.options.logger.debug('Begin tick');\n await this.handleNextAction(signal);\n } catch (error) {\n this.options.logger.error(`${error}`);\n throw error;\n } finally {\n this.options.logger.debug('End tick');\n }\n }\n\n async handleNextAction(signal: AbortSignal) {\n await this.options.ready;\n\n const result = await this.getCurrentAction();\n if (result) {\n const { ingestionId, nextActionAt, nextAction, attempts } = result;\n\n switch (nextAction) {\n case 'rest':\n if (Date.now() > nextActionAt) {\n await this.manager.clearFinishedIngestions(\n this.options.provider.getProviderName(),\n );\n this.options.logger.debug(\n `incremental-engine: Ingestion ${ingestionId} rest period complete. Ingestion will start again`,\n );\n\n await this.manager.setProviderComplete(ingestionId);\n } else {\n this.options.logger.debug(\n `incremental-engine: Ingestion '${ingestionId}' rest period continuing`,\n );\n }\n break;\n case 'ingest':\n try {\n await this.manager.setProviderBursting(ingestionId);\n const done = await this.ingestOneBurst(ingestionId, signal);\n if (done) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${ingestionId}' complete, transitioning to rest period of ${this.restLength.toHuman()}`,\n );\n await this.manager.setProviderResting(\n ingestionId,\n this.restLength,\n );\n } else {\n await this.manager.setProviderInterstitial(ingestionId);\n this.options.logger.debug(\n `incremental-engine: Ingestion '${ingestionId}' continuing`,\n );\n }\n } catch (error) {\n if (\n (error as Error).message &&\n (error as Error).message === 'CANCEL'\n ) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${ingestionId}' canceled`,\n );\n await this.manager.setProviderCanceling(\n ingestionId,\n (error as Error).message,\n );\n } else {\n const currentBackoff = Duration.fromObject(\n this.backoff[Math.min(this.backoff.length - 1, attempts)],\n );\n\n const backoffLength = currentBackoff.as('milliseconds');\n this.options.logger.error(\n `incremental-engine: Ingestion '${ingestionId}' failed`,\n error,\n );\n\n const truncatedError = stringifyError(error).substring(0, 700);\n this.options.logger.error(\n `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`,\n );\n\n await this.manager.setProviderBackoff(\n ingestionId,\n attempts,\n error as Error,\n backoffLength,\n );\n }\n }\n break;\n case 'backoff':\n if (Date.now() > nextActionAt) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${ingestionId}' backoff complete, will attempt to resume`,\n );\n await this.manager.setProviderIngesting(ingestionId);\n } else {\n this.options.logger.debug(\n `incremental-engine: Ingestion '${ingestionId}' backoff continuing`,\n );\n }\n break;\n case 'cancel':\n this.options.logger.info(\n `incremental-engine: Ingestion '${ingestionId}' canceling, will restart`,\n );\n await this.manager.setProviderCanceled(ingestionId);\n break;\n default:\n this.options.logger.error(\n `incremental-engine: Ingestion '${ingestionId}' received unknown action '${nextAction}'`,\n );\n }\n } else {\n this.options.logger.error(\n `incremental-engine: Engine tried to create duplicate ingestion record for provider '${this.options.provider.getProviderName()}'.`,\n );\n }\n }\n\n async getCurrentAction() {\n const providerName = this.options.provider.getProviderName();\n const record = await this.manager.getCurrentIngestionRecord(providerName);\n if (record) {\n this.options.logger.debug(\n `incremental-engine: Ingestion record found: '${record.id}'`,\n );\n return {\n ingestionId: record.id,\n nextAction: record.next_action as 'rest' \| 'ingest' \| 'backoff',\n attempts: record.attempts as number,\n nextActionAt: record.next_action_at.valueOf() as number,\n };\n }\n const result = await this.manager.createProviderIngestionRecord(\n providerName,\n );\n if (result) {\n this.options.logger.info(\n `incremental-engine: Ingestion record created: '${result.ingestionId}'`,\n );\n }\n return result;\n }\n\n async ingestOneBurst(id: string, signal: AbortSignal) {\n const lastMark = await this.manager.getLastMark(id);\n\n const cursor = lastMark ? lastMark.cursor : undefined;\n let sequence = lastMark ? lastMark.sequence + 1 : 0;\n\n const start = performance.now();\n let count = 0;\n let done = false;\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}' burst initiated`,\n );\n\n await this.options.provider.around(async (context: unknown) => {\n let next = await this.options.provider.next(context, cursor);\n count++;\n for (;;) {\n done = next.done;\n await this.mark({\n id,\n sequence,\n entities: next?.entities,\n done: next.done,\n cursor: next?.cursor,\n });\n if (signal.aborted \|\| next.done) {\n break;\n } else {\n next = await this.options.provider.next(context, next.cursor);\n count++;\n sequence++;\n }\n }\n });\n\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}' burst complete. (${count} batches in ${Math.round(\n performance.now() - start,\n )}ms).`,\n );\n return done;\n }\n\n async mark(options: {\n id: string;\n sequence: number;\n entities?: DeferredEntity[];\n done: boolean;\n cursor?: unknown;\n }) {\n const { id, sequence, entities, done, cursor } = options;\n this.options.logger.debug(\n `incremental-engine: Ingestion '${id}': MARK ${\n entities ? entities.length : 0\n } entities, cursor: ${\n cursor ? JSON.stringify(cursor) : 'none'\n }, done: ${done}`,\n );\n const markId = v4();\n\n await this.manager.createMark({\n record: {\n id: markId,\n ingestion_id: id,\n cursor,\n sequence,\n },\n });\n\n if (entities && entities.length > 0) {\n await this.manager.createMarkEntities(markId, entities);\n }\n\n const added =\n entities?.map(deferred => ({\n ...deferred,\n entity: {\n ...deferred.entity,\n metadata: {\n ...deferred.entity.metadata,\n annotations: {\n ...deferred.entity.metadata.annotations,\n },\n },\n },\n })) ?? [];\n\n const removed: { entityRef: string }[] = [];\n\n if (done) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}': Final page reached, calculating removed entities`,\n );\n const result = await this.manager.computeRemoved(\n this.options.provider.getProviderName(),\n id,\n );\n\n const { total } = result;\n\n let doRemoval = true;\n if (this.options.rejectEmptySourceCollections) {\n if (total === 0) {\n this.options.logger.error(\n `incremental-engine: Ingestion '${id}': Rejecting empty entity collection!`,\n );\n doRemoval = false;\n }\n }\n\n if (this.options.rejectRemovalsAbovePercentage) {\n // If the total entities upserted in this ingestion is 0, then\n // 100% of entities are stale and marked for removal.\n const percentRemoved =\n total > 0 ? (result.removed.length / total) 100 : 100;\n if (percentRemoved <= this.options.rejectRemovalsAbovePercentage) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}': Removing ${result.removed.length} entities that have no matching assets`,\n );\n } else {\n const notice = `Attempted to remove ${percentRemoved}% of matching entities!`;\n this.options.logger.error(\n `incremental-engine: Ingestion '${id}': ${notice}`,\n );\n await this.manager.updateIngestionRecordById({\n ingestionId: id,\n update: {\n last_error: `REMOVAL_THRESHOLD exceeded on ingestion mark ${markId}: ${notice}`,\n },\n });\n doRemoval = false;\n }\n }\n if (doRemoval) {\n for (const entityRef of result.removed) {\n removed.push(entityRef);\n }\n }\n }\n\n await this.options.connection.applyMutation({\n type: 'delta',\n added,\n removed,\n });\n }\n\n async onEvent(params: EventParams): Promise<void> {\n const { topic } = params;\n if (!this.supportsEventTopics().includes(topic)) {\n return;\n }\n\n const { logger, provider, connection } = this.options;\n const providerName = provider.getProviderName();\n logger.debug(`incremental-engine: ${providerName} received ${topic} event`);\n\n if (!provider.eventHandler) {\n return;\n }\n\n const result = await provider.eventHandler.onEvent(params);\n\n if (result.type === 'delta') {\n if (result.added.length > 0) {\n const ingestionRecord = await this.manager.getCurrentIngestionRecord(\n providerName,\n );\n\n if (!ingestionRecord) {\n logger.debug(\n `incremental-engine: ${providerName} skipping delta addition because incremental ingestion is restarting.`,\n );\n } else {\n const mark =\n ingestionRecord.status === 'resting'\n ? await this.manager.getLastMark(ingestionRecord.id)\n : await this.manager.getFirstMark(ingestionRecord.id);\n\n if (!mark) {\n throw new Error(\n `Cannot apply delta, page records are missing! Please re-run incremental ingestion for ${providerName}.`,\n );\n }\n await this.manager.createMarkEntities(mark.id, result.added);\n }\n }\n\n if (result.removed.length > 0) {\n await this.manager.deleteEntityRecordsByRef(result.removed);\n }\n\n await connection.applyMutation(result);\n logger.debug(\n `incremental-engine: ${providerName} processed delta from '${topic}' event`,\n );\n } else {\n logger.debug(\n `incremental-engine: ${providerName} ignored event from topic '${topic}'`,\n );\n }\n }\n\n supportsEventTopics(): string[] {\n const { provider } = this.options;\n const topics = provider.eventHandler\n ? provider.eventHandler.supportsEventTopics()\n : [];\n return topics;\n }\n}\n"],"names":["Duration","stringifyError","performance","v4"],"mappings":";;;;;;;AA0BO,MAAM,0BAAsD,CAAA;AAAA,EAMjE,YAAoB,OAAiC,EAAA;AAAjC,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAClB,IAAA,IAAA,CAAK,UAAU,OAAQ,CAAA,OAAA;AACvB,IAAA,IAAA,CAAK,UAAa,GAAAA,cAAA,CAAS,UAAW,CAAA,OAAA,CAAQ,UAAU,CAAA;AACxD,IAAK,IAAA,CAAA,OAAA,GAAU,QAAQ,OAAW,IAAA;AAAA,MAChC,EAAE,SAAS,CAAE,EAAA;AAAA,MACb,EAAE,SAAS,CAAE,EAAA;AAAA,MACb,EAAE,SAAS,EAAG,EAAA;AAAA,MACd,EAAE,OAAO,CAAE;AAAA,KACb;AAAA;AACF,EAdiB,UAAA;AAAA,EACA,OAAA;AAAA,EAET,OAAA;AAAA,EAaR,MAAM,OAAO,MAAqB,EAAA;AAChC,IAAI,IAAA;AACF,MAAK,IAAA,CAAA,OAAA,CAAQ,MAAO,CAAA,KAAA,CAAM,YAAY,CAAA;AACtC,MAAM,MAAA,IAAA,CAAK,iBAAiB,MAAM,CAAA;AAAA,aAC3B,KAAO,EAAA;AACd,MAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,CAAO,KAAM,CAAA,CAAA,EAAG,KAAK,CAAE,CAAA,CAAA;AACpC,MAAM,MAAA,KAAA;AAAA,KACN,SAAA;AACA,MAAK,IAAA,CAAA,OAAA,CAAQ,MAAO,CAAA,KAAA,CAAM,UAAU,CAAA;AAAA;AACtC;AACF,EAEA,MAAM,iBAAiB,MAAqB,EAAA;AAC1C,IAAA,MAAM,KAAK,OAAQ,CAAA,KAAA;AAEnB,IAAM,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,gBAAiB,EAAA;AAC3C,IAAA,IAAI,MAAQ,EAAA;AACV,MAAA,MAAM,EAAE,WAAA,EAAa,YAAc,EAAA,UAAA,EAAY,UAAa,GAAA,MAAA;AAE5D,MAAA,QAAQ,UAAY;AAAA,QAClB,KAAK,MAAA;AACH,UAAI,IAAA,IAAA,CAAK,GAAI,EAAA,GAAI,YAAc,EAAA;AAC7B,YAAA,MAAM,KAAK,OAAQ,CAAA,uBAAA;AAAA,cACjB,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,eAAgB;AAAA,aACxC;AACA,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,cAClB,iCAAiC,WAAW,CAAA,iDAAA;AAAA,aAC9C;AAEA,YAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,mBAAA,CAAoB,WAAW,CAAA;AAAA,WAC7C,MAAA;AACL,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,wBAAA;AAAA,aAC/C;AAAA;AAEF,UAAA;AAAA,QACF,KAAK,QAAA;AACH,UAAI,IAAA;AACF,YAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,mBAAA,CAAoB,WAAW,CAAA;AAClD,YAAA,MAAM,IAAO,GAAA,MAAM,IAAK,CAAA,cAAA,CAAe,aAAa,MAAM,CAAA;AAC1D,YAAA,IAAI,IAAM,EAAA;AACR,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,4CAAA,EAA+C,IAAK,CAAA,UAAA,CAAW,SAAS,CAAA;AAAA,eACvH;AACA,cAAA,MAAM,KAAK,OAAQ,CAAA,kBAAA;AAAA,gBACjB,WAAA;AAAA,gBACA,IAAK,CAAA;AAAA,eACP;AAAA,aACK,MAAA;AACL,cAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,uBAAA,CAAwB,WAAW,CAAA;AACtD,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,YAAA;AAAA,eAC/C;AAAA;AACF,mBACO,KAAO,EAAA;AACd,YAAA,IACG,KAAgB,CAAA,OAAA,IAChB,KAAgB,CAAA,OAAA,KAAY,QAC7B,EAAA;AACA,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,UAAA;AAAA,eAC/C;AACA,cAAA,MAAM,KAAK,OAAQ,CAAA,oBAAA;AAAA,gBACjB,WAAA;AAAA,gBACC,KAAgB,CAAA;AAAA,eACnB;AAAA,aACK,MAAA;AACL,cAAA,MAAM,iBAAiBA,cAAS,CAAA,UAAA;AAAA,gBAC9B,IAAA,CAAK,QAAQ,IAAK,CAAA,GAAA,CAAI,KAAK,OAAQ,CAAA,MAAA,GAAS,CAAG,EAAA,QAAQ,CAAC;AAAA,eAC1D;AAEA,cAAM,MAAA,aAAA,GAAgB,cAAe,CAAA,EAAA,CAAG,cAAc,CAAA;AACtD,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,QAAA,CAAA;AAAA,gBAC7C;AAAA,eACF;AAEA,cAAA,MAAM,iBAAiBC,qBAAe,CAAA,KAAK,CAAE,CAAA,SAAA,CAAU,GAAG,GAAG,CAAA;AAC7D,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,oEAAA,EAAuE,eAAe,OAAQ,EAAC,KAAK,cAAc,CAAA,CAAA;AAAA,eACjK;AAEA,cAAA,MAAM,KAAK,OAAQ,CAAA,kBAAA;AAAA,gBACjB,WAAA;AAAA,gBACA,QAAA;AAAA,gBACA,KAAA;AAAA,gBACA;AAAA,eACF;AAAA;AACF;AAEF,UAAA;AAAA,QACF,KAAK,SAAA;AACH,UAAI,IAAA,IAAA,CAAK,GAAI,EAAA,GAAI,YAAc,EAAA;AAC7B,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,0CAAA;AAAA,aAC/C;AACA,YAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,oBAAA,CAAqB,WAAW,CAAA;AAAA,WAC9C,MAAA;AACL,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,oBAAA;AAAA,aAC/C;AAAA;AAEF,UAAA;AAAA,QACF,KAAK,QAAA;AACH,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,YAClB,kCAAkC,WAAW,CAAA,yBAAA;AAAA,WAC/C;AACA,UAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,mBAAA,CAAoB,WAAW,CAAA;AAClD,UAAA;AAAA,QACF;AACE,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,WAAW,CAAA,2BAAA,EAA8B,UAAU,CAAA,CAAA;AAAA,WACvF;AAAA;AACJ,KACK,MAAA;AACL,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,QAClB,CAAuF,oFAAA,EAAA,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,iBAAiB,CAAA,EAAA;AAAA,OAChI;AAAA;AACF;AACF,EAEA,MAAM,gBAAmB,GAAA;AACvB,IAAA,MAAM,YAAe,GAAA,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,eAAgB,EAAA;AAC3D,IAAA,MAAM,MAAS,GAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,0BAA0B,YAAY,CAAA;AACxE,IAAA,IAAI,MAAQ,EAAA;AACV,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,QAClB,CAAA,6CAAA,EAAgD,OAAO,EAAE,CAAA,CAAA;AAAA,OAC3D;AACA,MAAO,OAAA;AAAA,QACL,aAAa,MAAO,CAAA,EAAA;AAAA,QACpB,YAAY,MAAO,CAAA,WAAA;AAAA,QACnB,UAAU,MAAO,CAAA,QAAA;AAAA,QACjB,YAAA,EAAc,MAAO,CAAA,cAAA,CAAe,OAAQ;AAAA,OAC9C;AAAA;AAEF,IAAM,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,OAAQ,CAAA,6BAAA;AAAA,MAChC;AAAA,KACF;AACA,IAAA,IAAI,MAAQ,EAAA;AACV,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,QAClB,CAAA,+CAAA,EAAkD,OAAO,WAAW,CAAA,CAAA;AAAA,OACtE;AAAA;AAEF,IAAO,OAAA,MAAA;AAAA;AACT,EAEA,MAAM,cAAe,CAAA,EAAA,EAAY,MAAqB,EAAA;AACpD,IAAA,MAAM,QAAW,GAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,YAAY,EAAE,CAAA;AAElD,IAAM,MAAA,MAAA,GAAS,QAAW,GAAA,QAAA,CAAS,MAAS,GAAA,KAAA,CAAA;AAC5C,IAAA,IAAI,QAAW,GAAA,QAAA,GAAW,QAAS,CAAA,QAAA,GAAW,CAAI,GAAA,CAAA;AAElD,IAAM,MAAA,KAAA,GAAQC,uBAAY,GAAI,EAAA;AAC9B,IAAA,IAAI,KAAQ,GAAA,CAAA;AACZ,IAAA,IAAI,IAAO,GAAA,KAAA;AACX,IAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,MAClB,kCAAkC,EAAE,CAAA,iBAAA;AAAA,KACtC;AAEA,IAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,QAAS,CAAA,MAAA,CAAO,OAAO,OAAqB,KAAA;AAC7D,MAAA,IAAI,OAAO,MAAM,IAAA,CAAK,QAAQ,QAAS,CAAA,IAAA,CAAK,SAAS,MAAM,CAAA;AAC3D,MAAA,KAAA,EAAA;AACA,MAAS,WAAA;AACP,QAAA,IAAA,GAAO,IAAK,CAAA,IAAA;AACZ,QAAA,MAAM,KAAK,IAAK,CAAA;AAAA,UACd,EAAA;AAAA,UACA,QAAA;AAAA,UACA,UAAU,IAAM,EAAA,QAAA;AAAA,UAChB,MAAM,IAAK,CAAA,IAAA;AAAA,UACX,QAAQ,IAAM,EAAA;AAAA,SACf,CAAA;AACD,QAAI,IAAA,MAAA,CAAO,OAAW,IAAA,IAAA,CAAK,IAAM,EAAA;AAC/B,UAAA;AAAA,SACK,MAAA;AACL,UAAA,IAAA,GAAO,MAAM,IAAK,CAAA,OAAA,CAAQ,SAAS,IAAK,CAAA,OAAA,EAAS,KAAK,MAAM,CAAA;AAC5D,UAAA,KAAA,EAAA;AACA,UAAA,QAAA,EAAA;AAAA;AACF;AACF,KACD,CAAA;AAED,IAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,MAClB,CAAkC,+BAAA,EAAA,EAAE,CAAsB,mBAAA,EAAA,KAAK,eAAe,IAAK,CAAA,KAAA;AAAA,QACjFA,sBAAA,CAAY,KAAQ,GAAA;AAAA,OACrB,CAAA,IAAA;AAAA,KACH;AACA,IAAO,OAAA,IAAA;AAAA;AACT,EAEA,MAAM,KAAK,OAMR,EAAA;AACD,IAAA,MAAM,EAAE,EAAI,EAAA,QAAA,EAAU,QAAU,EAAA,IAAA,EAAM,QAAW,GAAA,OAAA;AACjD,IAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,MAClB,CAAkC,+BAAA,EAAA,EAAE,CAClC,QAAA,EAAA,QAAA,GAAW,SAAS,MAAS,GAAA,CAC/B,CACE,mBAAA,EAAA,MAAA,GAAS,KAAK,SAAU,CAAA,MAAM,CAAI,GAAA,MACpC,WAAW,IAAI,CAAA;AAAA,KACjB;AACA,IAAA,MAAM,SAASC,OAAG,EAAA;AAElB,IAAM,MAAA,IAAA,CAAK,QAAQ,UAAW,CAAA;AAAA,MAC5B,MAAQ,EAAA;AAAA,QACN,EAAI,EAAA,MAAA;AAAA,QACJ,YAAc,EAAA,EAAA;AAAA,QACd,MAAA;AAAA,QACA;AAAA;AACF,KACD,CAAA;AAED,IAAI,IAAA,QAAA,IAAY,QAAS,CAAA,MAAA,GAAS,CAAG,EAAA;AACnC,MAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,kBAAmB,CAAA,MAAA,EAAQ,QAAQ,CAAA;AAAA;AAGxD,IAAM,MAAA,KAAA,GACJ,QAAU,EAAA,GAAA,CAAI,CAAa,QAAA,MAAA;AAAA,MACzB,GAAG,QAAA;AAAA,MACH,MAAQ,EAAA;AAAA,QACN,GAAG,QAAS,CAAA,MAAA;AAAA,QACZ,QAAU,EAAA;AAAA,UACR,GAAG,SAAS,MAAO,CAAA,QAAA;AAAA,UACnB,WAAa,EAAA;AAAA,YACX,GAAG,QAAS,CAAA,MAAA,CAAO,QAAS,CAAA;AAAA;AAC9B;AACF;AACF,KACF,CAAE,KAAK,EAAC;AAEV,IAAA,MAAM,UAAmC,EAAC;AAE1C,IAAA,IAAI,IAAM,EAAA;AACR,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,QAClB,kCAAkC,EAAE,CAAA,mDAAA;AAAA,OACtC;AACA,MAAM,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,OAAQ,CAAA,cAAA;AAAA,QAChC,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,eAAgB,EAAA;AAAA,QACtC;AAAA,OACF;AAEA,MAAM,MAAA,EAAE,OAAU,GAAA,MAAA;AAElB,MAAA,IAAI,SAAY,GAAA,IAAA;AAChB,MAAI,IAAA,IAAA,CAAK,QAAQ,4BAA8B,EAAA;AAC7C,QAAA,IAAI,UAAU,CAAG,EAAA;AACf,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,YAClB,kCAAkC,EAAE,CAAA,qCAAA;AAAA,WACtC;AACA,UAAY,SAAA,GAAA,KAAA;AAAA;AACd;AAGF,MAAI,IAAA,IAAA,CAAK,QAAQ,6BAA+B,EAAA;AAG9C,QAAA,MAAM,iBACJ,KAAQ,GAAA,CAAA,GAAK,OAAO,OAAQ,CAAA,MAAA,GAAS,QAAS,GAAM,GAAA,GAAA;AACtD,QAAI,IAAA,cAAA,IAAkB,IAAK,CAAA,OAAA,CAAQ,6BAA+B,EAAA;AAChE,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,YAClB,CAAkC,+BAAA,EAAA,EAAE,CAAe,YAAA,EAAA,MAAA,CAAO,QAAQ,MAAM,CAAA,sCAAA;AAAA,WAC1E;AAAA,SACK,MAAA;AACL,UAAM,MAAA,MAAA,GAAS,uBAAuB,cAAc,CAAA,uBAAA,CAAA;AACpD,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,GAAA,EAAM,MAAM,CAAA;AAAA,WAClD;AACA,UAAM,MAAA,IAAA,CAAK,QAAQ,yBAA0B,CAAA;AAAA,YAC3C,WAAa,EAAA,EAAA;AAAA,YACb,MAAQ,EAAA;AAAA,cACN,UAAY,EAAA,CAAA,6CAAA,EAAgD,MAAM,CAAA,EAAA,EAAK,MAAM,CAAA;AAAA;AAC/E,WACD,CAAA;AACD,UAAY,SAAA,GAAA,KAAA;AAAA;AACd;AAEF,MAAA,IAAI,SAAW,EAAA;AACb,QAAW,KAAA,MAAA,SAAA,IAAa,OAAO,OAAS,EAAA;AACtC,UAAA,OAAA,CAAQ,KAAK,SAAS,CAAA;AAAA;AACxB;AACF;AAGF,IAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,UAAA,CAAW,aAAc,CAAA;AAAA,MAC1C,IAAM,EAAA,OAAA;AAAA,MACN,KAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA;AACH,EAEA,MAAM,QAAQ,MAAoC,EAAA;AAChD,IAAM,MAAA,EAAE,OAAU,GAAA,MAAA;AAClB,IAAA,IAAI,CAAC,IAAK,CAAA,mBAAA,EAAsB,CAAA,QAAA,CAAS,KAAK,CAAG,EAAA;AAC/C,MAAA;AAAA;AAGF,IAAA,MAAM,EAAE,MAAA,EAAQ,QAAU,EAAA,UAAA,KAAe,IAAK,CAAA,OAAA;AAC9C,IAAM,MAAA,YAAA,GAAe,SAAS,eAAgB,EAAA;AAC9C,IAAA,MAAA,CAAO,KAAM,CAAA,CAAA,oBAAA,EAAuB,YAAY,CAAA,UAAA,EAAa,KAAK,CAAQ,MAAA,CAAA,CAAA;AAE1E,IAAI,IAAA,CAAC,SAAS,YAAc,EAAA;AAC1B,MAAA;AAAA;AAGF,IAAA,MAAM,MAAS,GAAA,MAAM,QAAS,CAAA,YAAA,CAAa,QAAQ,MAAM,CAAA;AAEzD,IAAI,IAAA,MAAA,CAAO,SAAS,OAAS,EAAA;AAC3B,MAAI,IAAA,MAAA,CAAO,KAAM,CAAA,MAAA,GAAS,CAAG,EAAA;AAC3B,QAAM,MAAA,eAAA,GAAkB,MAAM,IAAA,CAAK,OAAQ,CAAA,yBAAA;AAAA,UACzC;AAAA,SACF;AAEA,QAAA,IAAI,CAAC,eAAiB,EAAA;AACpB,UAAO,MAAA,CAAA,KAAA;AAAA,YACL,uBAAuB,YAAY,CAAA,qEAAA;AAAA,WACrC;AAAA,SACK,MAAA;AACL,UAAA,MAAM,OACJ,eAAgB,CAAA,MAAA,KAAW,SACvB,GAAA,MAAM,KAAK,OAAQ,CAAA,WAAA,CAAY,eAAgB,CAAA,EAAE,IACjD,MAAM,IAAA,CAAK,OAAQ,CAAA,YAAA,CAAa,gBAAgB,EAAE,CAAA;AAExD,UAAA,IAAI,CAAC,IAAM,EAAA;AACT,YAAA,MAAM,IAAI,KAAA;AAAA,cACR,yFAAyF,YAAY,CAAA,CAAA;AAAA,aACvG;AAAA;AAEF,UAAA,MAAM,KAAK,OAAQ,CAAA,kBAAA,CAAmB,IAAK,CAAA,EAAA,EAAI,OAAO,KAAK,CAAA;AAAA;AAC7D;AAGF,MAAI,IAAA,MAAA,CAAO,OAAQ,CAAA,MAAA,GAAS,CAAG,EAAA;AAC7B,QAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,wBAAyB,CAAA,MAAA,CAAO,OAAO,CAAA;AAAA;AAG5D,MAAM,MAAA,UAAA,CAAW,cAAc,MAAM,CAAA;AACrC,MAAO,MAAA,CAAA,KAAA;AAAA,QACL,CAAA,oBAAA,EAAuB,YAAY,CAAA,uBAAA,EAA0B,KAAK,CAAA,OAAA;AAAA,OACpE;AAAA,KACK,MAAA;AACL,MAAO,MAAA,CAAA,KAAA;AAAA,QACL,CAAA,oBAAA,EAAuB,YAAY,CAAA,2BAAA,EAA8B,KAAK,CAAA,CAAA;AAAA,OACxE;AAAA;AACF;AACF,EAEA,mBAAgC,GAAA;AAC9B,IAAM,MAAA,EAAE,QAAS,EAAA,GAAI,IAAK,CAAA,OAAA;AAC1B,IAAA,MAAM,SAAS,QAAS,CAAA,YAAA,GACpB,SAAS,YAAa,CAAA,mBAAA,KACtB,EAAC;AACL,IAAO,OAAA,MAAA;AAAA;AAEX;;;;"}
1	+ {"version":3,"file":"IncrementalIngestionEngine.cjs.js","sources":["../../src/engine/IncrementalIngestionEngine.ts"],"sourcesContent":["/\n Copyright 2022 The Backstage Authors\n \n Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n \n http://www.apache.org/licenses/LICENSE-2.0\n \n Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n /\n\nimport type { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { Gauge, metrics } from '@opentelemetry/api';\nimport { IterationEngine, IterationEngineOptions } from '../types';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { performance } from 'perf_hooks';\nimport { Duration } from 'luxon';\nimport { v4 } from 'uuid';\nimport { stringifyError } from '@backstage/errors';\nimport { EventParams } from '@backstage/plugin-events-node';\nimport { HumanDuration } from '@backstage/types';\n\nexport class IncrementalIngestionEngine implements IterationEngine {\n private readonly restLength: Duration;\n private readonly backoff: HumanDuration[];\n private readonly lastStarted: Gauge;\n private readonly lastCompleted: Gauge;\n\n private manager: IncrementalIngestionDatabaseManager;\n\n constructor(private options: IterationEngineOptions) {\n const meter = metrics.getMeter('default');\n\n this.manager = options.manager;\n this.restLength = Duration.fromObject(options.restLength);\n this.backoff = options.backoff ?? [\n { minutes: 1 },\n { minutes: 5 },\n { minutes: 30 },\n { hours: 3 },\n ];\n\n this.lastStarted = meter.createGauge(\n 'catalog_incremental.ingestions.started',\n {\n description:\n 'Epoch timestamp seconds when the ingestion was last started',\n unit: 'seconds',\n },\n );\n this.lastCompleted = meter.createGauge(\n 'catalog_incremental.ingestions.completed',\n {\n description:\n 'Epoch timestamp seconds when the ingestion was last completed',\n unit: 'seconds',\n },\n );\n }\n\n async taskFn(signal: AbortSignal) {\n try {\n this.options.logger.debug('Begin tick');\n await this.handleNextAction(signal);\n } catch (error) {\n this.options.logger.error(`${error}`);\n throw error;\n } finally {\n this.options.logger.debug('End tick');\n }\n }\n\n async handleNextAction(signal: AbortSignal) {\n await this.options.ready;\n\n const result = await this.getCurrentAction();\n if (result) {\n const { ingestionId, nextActionAt, nextAction, attempts } = result;\n\n switch (nextAction) {\n case 'rest':\n if (Date.now() > nextActionAt) {\n await this.manager.clearFinishedIngestions(\n this.options.provider.getProviderName(),\n );\n this.options.logger.debug(\n `incremental-engine: Ingestion ${ingestionId} rest period complete. Ingestion will start again`,\n );\n\n this.lastStarted.record(Date.now() / 1000, {\n providerName: this.options.provider.getProviderName(),\n });\n await this.manager.setProviderComplete(ingestionId);\n } else {\n this.options.logger.debug(\n `incremental-engine: Ingestion '${ingestionId}' rest period continuing`,\n );\n }\n break;\n case 'ingest':\n try {\n await this.manager.setProviderBursting(ingestionId);\n const done = await this.ingestOneBurst(ingestionId, signal);\n if (done) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${ingestionId}' complete, transitioning to rest period of ${this.restLength.toHuman()}`,\n );\n this.lastCompleted.record(Date.now() / 1000, {\n providerName: this.options.provider.getProviderName(),\n status: 'completed',\n });\n await this.manager.setProviderResting(\n ingestionId,\n this.restLength,\n );\n } else {\n await this.manager.setProviderInterstitial(ingestionId);\n this.options.logger.debug(\n `incremental-engine: Ingestion '${ingestionId}' continuing`,\n );\n }\n } catch (error) {\n if (\n (error as Error).message &&\n (error as Error).message === 'CANCEL'\n ) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${ingestionId}' canceled`,\n );\n await this.manager.setProviderCanceling(\n ingestionId,\n (error as Error).message,\n );\n } else {\n const currentBackoff = Duration.fromObject(\n this.backoff[Math.min(this.backoff.length - 1, attempts)],\n );\n\n const backoffLength = currentBackoff.as('milliseconds');\n this.options.logger.error(\n `incremental-engine: Ingestion '${ingestionId}' failed`,\n error,\n );\n\n const truncatedError = stringifyError(error).substring(0, 700);\n this.options.logger.error(\n `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`,\n );\n this.lastCompleted.record(Date.now() / 1000, {\n providerName: this.options.provider.getProviderName(),\n status: 'failed',\n });\n\n await this.manager.setProviderBackoff(\n ingestionId,\n attempts,\n error as Error,\n backoffLength,\n );\n }\n }\n break;\n case 'backoff':\n if (Date.now() > nextActionAt) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${ingestionId}' backoff complete, will attempt to resume`,\n );\n await this.manager.setProviderIngesting(ingestionId);\n } else {\n this.options.logger.debug(\n `incremental-engine: Ingestion '${ingestionId}' backoff continuing`,\n );\n }\n break;\n case 'cancel':\n this.options.logger.info(\n `incremental-engine: Ingestion '${ingestionId}' canceling, will restart`,\n );\n await this.manager.setProviderCanceled(ingestionId);\n break;\n default:\n this.options.logger.error(\n `incremental-engine: Ingestion '${ingestionId}' received unknown action '${nextAction}'`,\n );\n }\n } else {\n this.options.logger.error(\n `incremental-engine: Engine tried to create duplicate ingestion record for provider '${this.options.provider.getProviderName()}'.`,\n );\n }\n }\n\n async getCurrentAction() {\n const providerName = this.options.provider.getProviderName();\n const record = await this.manager.getCurrentIngestionRecord(providerName);\n if (record) {\n this.options.logger.debug(\n `incremental-engine: Ingestion record found: '${record.id}'`,\n );\n return {\n ingestionId: record.id,\n nextAction: record.next_action as 'rest' \| 'ingest' \| 'backoff',\n attempts: record.attempts as number,\n nextActionAt: record.next_action_at.valueOf() as number,\n };\n }\n const result = await this.manager.createProviderIngestionRecord(\n providerName,\n );\n if (result) {\n this.options.logger.info(\n `incremental-engine: Ingestion record created: '${result.ingestionId}'`,\n );\n }\n return result;\n }\n\n async ingestOneBurst(id: string, signal: AbortSignal) {\n const lastMark = await this.manager.getLastMark(id);\n\n const cursor = lastMark ? lastMark.cursor : undefined;\n let sequence = lastMark ? lastMark.sequence + 1 : 0;\n\n const start = performance.now();\n let count = 0;\n let done = false;\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}' burst initiated`,\n );\n\n await this.options.provider.around(async (context: unknown) => {\n let next = await this.options.provider.next(context, cursor);\n count++;\n for (;;) {\n done = next.done;\n await this.mark({\n id,\n sequence,\n entities: next?.entities,\n done: next.done,\n cursor: next?.cursor,\n });\n if (signal.aborted \|\| next.done) {\n break;\n } else {\n next = await this.options.provider.next(context, next.cursor);\n count++;\n sequence++;\n }\n }\n });\n\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}' burst complete. (${count} batches in ${Math.round(\n performance.now() - start,\n )}ms).`,\n );\n return done;\n }\n\n async mark(options: {\n id: string;\n sequence: number;\n entities?: DeferredEntity[];\n done: boolean;\n cursor?: unknown;\n }) {\n const { id, sequence, entities, done, cursor } = options;\n this.options.logger.debug(\n `incremental-engine: Ingestion '${id}': MARK ${\n entities ? entities.length : 0\n } entities, cursor: ${\n cursor ? JSON.stringify(cursor) : 'none'\n }, done: ${done}`,\n );\n const markId = v4();\n\n await this.manager.createMark({\n record: {\n id: markId,\n ingestion_id: id,\n cursor,\n sequence,\n },\n });\n\n if (entities && entities.length > 0) {\n await this.manager.createMarkEntities(markId, entities);\n }\n\n const added =\n entities?.map(deferred => ({\n ...deferred,\n entity: {\n ...deferred.entity,\n metadata: {\n ...deferred.entity.metadata,\n annotations: {\n ...deferred.entity.metadata.annotations,\n },\n },\n },\n })) ?? [];\n\n const removed: { entityRef: string }[] = [];\n\n if (done) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}': Final page reached, calculating removed entities`,\n );\n const result = await this.manager.computeRemoved(\n this.options.provider.getProviderName(),\n id,\n );\n\n const { total } = result;\n\n let doRemoval = true;\n if (this.options.rejectEmptySourceCollections) {\n if (total === 0) {\n this.options.logger.error(\n `incremental-engine: Ingestion '${id}': Rejecting empty entity collection!`,\n );\n doRemoval = false;\n }\n }\n\n if (this.options.rejectRemovalsAbovePercentage) {\n // If the total entities upserted in this ingestion is 0, then\n // 100% of entities are stale and marked for removal.\n const percentRemoved =\n total > 0 ? (result.removed.length / total) 100 : 100;\n if (percentRemoved <= this.options.rejectRemovalsAbovePercentage) {\n this.options.logger.info(\n `incremental-engine: Ingestion '${id}': Removing ${result.removed.length} entities that have no matching assets`,\n );\n } else {\n const notice = `Attempted to remove ${percentRemoved}% of matching entities!`;\n this.options.logger.error(\n `incremental-engine: Ingestion '${id}': ${notice}`,\n );\n await this.manager.updateIngestionRecordById({\n ingestionId: id,\n update: {\n last_error: `REMOVAL_THRESHOLD exceeded on ingestion mark ${markId}: ${notice}`,\n },\n });\n doRemoval = false;\n }\n }\n if (doRemoval) {\n for (const entityRef of result.removed) {\n removed.push(entityRef);\n }\n }\n }\n\n await this.options.connection.applyMutation({\n type: 'delta',\n added,\n removed,\n });\n }\n\n async onEvent(params: EventParams): Promise<void> {\n const { topic } = params;\n if (!this.supportsEventTopics().includes(topic)) {\n return;\n }\n\n const { logger, provider, connection } = this.options;\n const providerName = provider.getProviderName();\n logger.debug(`incremental-engine: ${providerName} received ${topic} event`);\n\n if (!provider.eventHandler) {\n return;\n }\n\n const result = await provider.eventHandler.onEvent(params);\n\n if (result.type === 'delta') {\n if (result.added.length > 0) {\n const ingestionRecord = await this.manager.getCurrentIngestionRecord(\n providerName,\n );\n\n if (!ingestionRecord) {\n logger.debug(\n `incremental-engine: ${providerName} skipping delta addition because incremental ingestion is restarting.`,\n );\n } else {\n const mark =\n ingestionRecord.status === 'resting'\n ? await this.manager.getLastMark(ingestionRecord.id)\n : await this.manager.getFirstMark(ingestionRecord.id);\n\n if (!mark) {\n throw new Error(\n `Cannot apply delta, page records are missing! Please re-run incremental ingestion for ${providerName}.`,\n );\n }\n await this.manager.createMarkEntities(mark.id, result.added);\n }\n }\n\n if (result.removed.length > 0) {\n await this.manager.deleteEntityRecordsByRef(result.removed);\n }\n\n await connection.applyMutation(result);\n logger.debug(\n `incremental-engine: ${providerName} processed delta from '${topic}' event`,\n );\n } else {\n logger.debug(\n `incremental-engine: ${providerName} ignored event from topic '${topic}'`,\n );\n }\n }\n\n supportsEventTopics(): string[] {\n const { provider } = this.options;\n const topics = provider.eventHandler\n ? provider.eventHandler.supportsEventTopics()\n : [];\n return topics;\n }\n}\n"],"names":["metrics","Duration","stringifyError","performance","v4"],"mappings":";;;;;;;;AA2BO,MAAM,0BAAsD,CAAA;AAAA,EAQjE,YAAoB,OAAiC,EAAA;AAAjC,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAClB,IAAM,MAAA,KAAA,GAAQA,WAAQ,CAAA,QAAA,CAAS,SAAS,CAAA;AAExC,IAAA,IAAA,CAAK,UAAU,OAAQ,CAAA,OAAA;AACvB,IAAA,IAAA,CAAK,UAAa,GAAAC,cAAA,CAAS,UAAW,CAAA,OAAA,CAAQ,UAAU,CAAA;AACxD,IAAK,IAAA,CAAA,OAAA,GAAU,QAAQ,OAAW,IAAA;AAAA,MAChC,EAAE,SAAS,CAAE,EAAA;AAAA,MACb,EAAE,SAAS,CAAE,EAAA;AAAA,MACb,EAAE,SAAS,EAAG,EAAA;AAAA,MACd,EAAE,OAAO,CAAE;AAAA,KACb;AAEA,IAAA,IAAA,CAAK,cAAc,KAAM,CAAA,WAAA;AAAA,MACvB,wCAAA;AAAA,MACA;AAAA,QACE,WACE,EAAA,6DAAA;AAAA,QACF,IAAM,EAAA;AAAA;AACR,KACF;AACA,IAAA,IAAA,CAAK,gBAAgB,KAAM,CAAA,WAAA;AAAA,MACzB,0CAAA;AAAA,MACA;AAAA,QACE,WACE,EAAA,+DAAA;AAAA,QACF,IAAM,EAAA;AAAA;AACR,KACF;AAAA;AACF,EAnCiB,UAAA;AAAA,EACA,OAAA;AAAA,EACA,WAAA;AAAA,EACA,aAAA;AAAA,EAET,OAAA;AAAA,EAgCR,MAAM,OAAO,MAAqB,EAAA;AAChC,IAAI,IAAA;AACF,MAAK,IAAA,CAAA,OAAA,CAAQ,MAAO,CAAA,KAAA,CAAM,YAAY,CAAA;AACtC,MAAM,MAAA,IAAA,CAAK,iBAAiB,MAAM,CAAA;AAAA,aAC3B,KAAO,EAAA;AACd,MAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,CAAO,KAAM,CAAA,CAAA,EAAG,KAAK,CAAE,CAAA,CAAA;AACpC,MAAM,MAAA,KAAA;AAAA,KACN,SAAA;AACA,MAAK,IAAA,CAAA,OAAA,CAAQ,MAAO,CAAA,KAAA,CAAM,UAAU,CAAA;AAAA;AACtC;AACF,EAEA,MAAM,iBAAiB,MAAqB,EAAA;AAC1C,IAAA,MAAM,KAAK,OAAQ,CAAA,KAAA;AAEnB,IAAM,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,gBAAiB,EAAA;AAC3C,IAAA,IAAI,MAAQ,EAAA;AACV,MAAA,MAAM,EAAE,WAAA,EAAa,YAAc,EAAA,UAAA,EAAY,UAAa,GAAA,MAAA;AAE5D,MAAA,QAAQ,UAAY;AAAA,QAClB,KAAK,MAAA;AACH,UAAI,IAAA,IAAA,CAAK,GAAI,EAAA,GAAI,YAAc,EAAA;AAC7B,YAAA,MAAM,KAAK,OAAQ,CAAA,uBAAA;AAAA,cACjB,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,eAAgB;AAAA,aACxC;AACA,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,cAClB,iCAAiC,WAAW,CAAA,iDAAA;AAAA,aAC9C;AAEA,YAAA,IAAA,CAAK,WAAY,CAAA,MAAA,CAAO,IAAK,CAAA,GAAA,KAAQ,GAAM,EAAA;AAAA,cACzC,YAAc,EAAA,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,eAAgB;AAAA,aACrD,CAAA;AACD,YAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,mBAAA,CAAoB,WAAW,CAAA;AAAA,WAC7C,MAAA;AACL,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,wBAAA;AAAA,aAC/C;AAAA;AAEF,UAAA;AAAA,QACF,KAAK,QAAA;AACH,UAAI,IAAA;AACF,YAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,mBAAA,CAAoB,WAAW,CAAA;AAClD,YAAA,MAAM,IAAO,GAAA,MAAM,IAAK,CAAA,cAAA,CAAe,aAAa,MAAM,CAAA;AAC1D,YAAA,IAAI,IAAM,EAAA;AACR,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,4CAAA,EAA+C,IAAK,CAAA,UAAA,CAAW,SAAS,CAAA;AAAA,eACvH;AACA,cAAA,IAAA,CAAK,aAAc,CAAA,MAAA,CAAO,IAAK,CAAA,GAAA,KAAQ,GAAM,EAAA;AAAA,gBAC3C,YAAc,EAAA,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,eAAgB,EAAA;AAAA,gBACpD,MAAQ,EAAA;AAAA,eACT,CAAA;AACD,cAAA,MAAM,KAAK,OAAQ,CAAA,kBAAA;AAAA,gBACjB,WAAA;AAAA,gBACA,IAAK,CAAA;AAAA,eACP;AAAA,aACK,MAAA;AACL,cAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,uBAAA,CAAwB,WAAW,CAAA;AACtD,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,YAAA;AAAA,eAC/C;AAAA;AACF,mBACO,KAAO,EAAA;AACd,YAAA,IACG,KAAgB,CAAA,OAAA,IAChB,KAAgB,CAAA,OAAA,KAAY,QAC7B,EAAA;AACA,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,UAAA;AAAA,eAC/C;AACA,cAAA,MAAM,KAAK,OAAQ,CAAA,oBAAA;AAAA,gBACjB,WAAA;AAAA,gBACC,KAAgB,CAAA;AAAA,eACnB;AAAA,aACK,MAAA;AACL,cAAA,MAAM,iBAAiBA,cAAS,CAAA,UAAA;AAAA,gBAC9B,IAAA,CAAK,QAAQ,IAAK,CAAA,GAAA,CAAI,KAAK,OAAQ,CAAA,MAAA,GAAS,CAAG,EAAA,QAAQ,CAAC;AAAA,eAC1D;AAEA,cAAM,MAAA,aAAA,GAAgB,cAAe,CAAA,EAAA,CAAG,cAAc,CAAA;AACtD,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,QAAA,CAAA;AAAA,gBAC7C;AAAA,eACF;AAEA,cAAA,MAAM,iBAAiBC,qBAAe,CAAA,KAAK,CAAE,CAAA,SAAA,CAAU,GAAG,GAAG,CAAA;AAC7D,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,oEAAA,EAAuE,eAAe,OAAQ,EAAC,KAAK,cAAc,CAAA,CAAA;AAAA,eACjK;AACA,cAAA,IAAA,CAAK,aAAc,CAAA,MAAA,CAAO,IAAK,CAAA,GAAA,KAAQ,GAAM,EAAA;AAAA,gBAC3C,YAAc,EAAA,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,eAAgB,EAAA;AAAA,gBACpD,MAAQ,EAAA;AAAA,eACT,CAAA;AAED,cAAA,MAAM,KAAK,OAAQ,CAAA,kBAAA;AAAA,gBACjB,WAAA;AAAA,gBACA,QAAA;AAAA,gBACA,KAAA;AAAA,gBACA;AAAA,eACF;AAAA;AACF;AAEF,UAAA;AAAA,QACF,KAAK,SAAA;AACH,UAAI,IAAA,IAAA,CAAK,GAAI,EAAA,GAAI,YAAc,EAAA;AAC7B,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,0CAAA;AAAA,aAC/C;AACA,YAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,oBAAA,CAAqB,WAAW,CAAA;AAAA,WAC9C,MAAA;AACL,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,oBAAA;AAAA,aAC/C;AAAA;AAEF,UAAA;AAAA,QACF,KAAK,QAAA;AACH,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,YAClB,kCAAkC,WAAW,CAAA,yBAAA;AAAA,WAC/C;AACA,UAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,mBAAA,CAAoB,WAAW,CAAA;AAClD,UAAA;AAAA,QACF;AACE,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,WAAW,CAAA,2BAAA,EAA8B,UAAU,CAAA,CAAA;AAAA,WACvF;AAAA;AACJ,KACK,MAAA;AACL,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,QAClB,CAAuF,oFAAA,EAAA,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,iBAAiB,CAAA,EAAA;AAAA,OAChI;AAAA;AACF;AACF,EAEA,MAAM,gBAAmB,GAAA;AACvB,IAAA,MAAM,YAAe,GAAA,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,eAAgB,EAAA;AAC3D,IAAA,MAAM,MAAS,GAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,0BAA0B,YAAY,CAAA;AACxE,IAAA,IAAI,MAAQ,EAAA;AACV,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,QAClB,CAAA,6CAAA,EAAgD,OAAO,EAAE,CAAA,CAAA;AAAA,OAC3D;AACA,MAAO,OAAA;AAAA,QACL,aAAa,MAAO,CAAA,EAAA;AAAA,QACpB,YAAY,MAAO,CAAA,WAAA;AAAA,QACnB,UAAU,MAAO,CAAA,QAAA;AAAA,QACjB,YAAA,EAAc,MAAO,CAAA,cAAA,CAAe,OAAQ;AAAA,OAC9C;AAAA;AAEF,IAAM,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,OAAQ,CAAA,6BAAA;AAAA,MAChC;AAAA,KACF;AACA,IAAA,IAAI,MAAQ,EAAA;AACV,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,QAClB,CAAA,+CAAA,EAAkD,OAAO,WAAW,CAAA,CAAA;AAAA,OACtE;AAAA;AAEF,IAAO,OAAA,MAAA;AAAA;AACT,EAEA,MAAM,cAAe,CAAA,EAAA,EAAY,MAAqB,EAAA;AACpD,IAAA,MAAM,QAAW,GAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,YAAY,EAAE,CAAA;AAElD,IAAM,MAAA,MAAA,GAAS,QAAW,GAAA,QAAA,CAAS,MAAS,GAAA,KAAA,CAAA;AAC5C,IAAA,IAAI,QAAW,GAAA,QAAA,GAAW,QAAS,CAAA,QAAA,GAAW,CAAI,GAAA,CAAA;AAElD,IAAM,MAAA,KAAA,GAAQC,uBAAY,GAAI,EAAA;AAC9B,IAAA,IAAI,KAAQ,GAAA,CAAA;AACZ,IAAA,IAAI,IAAO,GAAA,KAAA;AACX,IAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,MAClB,kCAAkC,EAAE,CAAA,iBAAA;AAAA,KACtC;AAEA,IAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,QAAS,CAAA,MAAA,CAAO,OAAO,OAAqB,KAAA;AAC7D,MAAA,IAAI,OAAO,MAAM,IAAA,CAAK,QAAQ,QAAS,CAAA,IAAA,CAAK,SAAS,MAAM,CAAA;AAC3D,MAAA,KAAA,EAAA;AACA,MAAS,WAAA;AACP,QAAA,IAAA,GAAO,IAAK,CAAA,IAAA;AACZ,QAAA,MAAM,KAAK,IAAK,CAAA;AAAA,UACd,EAAA;AAAA,UACA,QAAA;AAAA,UACA,UAAU,IAAM,EAAA,QAAA;AAAA,UAChB,MAAM,IAAK,CAAA,IAAA;AAAA,UACX,QAAQ,IAAM,EAAA;AAAA,SACf,CAAA;AACD,QAAI,IAAA,MAAA,CAAO,OAAW,IAAA,IAAA,CAAK,IAAM,EAAA;AAC/B,UAAA;AAAA,SACK,MAAA;AACL,UAAA,IAAA,GAAO,MAAM,IAAK,CAAA,OAAA,CAAQ,SAAS,IAAK,CAAA,OAAA,EAAS,KAAK,MAAM,CAAA;AAC5D,UAAA,KAAA,EAAA;AACA,UAAA,QAAA,EAAA;AAAA;AACF;AACF,KACD,CAAA;AAED,IAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,MAClB,CAAkC,+BAAA,EAAA,EAAE,CAAsB,mBAAA,EAAA,KAAK,eAAe,IAAK,CAAA,KAAA;AAAA,QACjFA,sBAAA,CAAY,KAAQ,GAAA;AAAA,OACrB,CAAA,IAAA;AAAA,KACH;AACA,IAAO,OAAA,IAAA;AAAA;AACT,EAEA,MAAM,KAAK,OAMR,EAAA;AACD,IAAA,MAAM,EAAE,EAAI,EAAA,QAAA,EAAU,QAAU,EAAA,IAAA,EAAM,QAAW,GAAA,OAAA;AACjD,IAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,MAClB,CAAkC,+BAAA,EAAA,EAAE,CAClC,QAAA,EAAA,QAAA,GAAW,SAAS,MAAS,GAAA,CAC/B,CACE,mBAAA,EAAA,MAAA,GAAS,KAAK,SAAU,CAAA,MAAM,CAAI,GAAA,MACpC,WAAW,IAAI,CAAA;AAAA,KACjB;AACA,IAAA,MAAM,SAASC,OAAG,EAAA;AAElB,IAAM,MAAA,IAAA,CAAK,QAAQ,UAAW,CAAA;AAAA,MAC5B,MAAQ,EAAA;AAAA,QACN,EAAI,EAAA,MAAA;AAAA,QACJ,YAAc,EAAA,EAAA;AAAA,QACd,MAAA;AAAA,QACA;AAAA;AACF,KACD,CAAA;AAED,IAAI,IAAA,QAAA,IAAY,QAAS,CAAA,MAAA,GAAS,CAAG,EAAA;AACnC,MAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,kBAAmB,CAAA,MAAA,EAAQ,QAAQ,CAAA;AAAA;AAGxD,IAAM,MAAA,KAAA,GACJ,QAAU,EAAA,GAAA,CAAI,CAAa,QAAA,MAAA;AAAA,MACzB,GAAG,QAAA;AAAA,MACH,MAAQ,EAAA;AAAA,QACN,GAAG,QAAS,CAAA,MAAA;AAAA,QACZ,QAAU,EAAA;AAAA,UACR,GAAG,SAAS,MAAO,CAAA,QAAA;AAAA,UACnB,WAAa,EAAA;AAAA,YACX,GAAG,QAAS,CAAA,MAAA,CAAO,QAAS,CAAA;AAAA;AAC9B;AACF;AACF,KACF,CAAE,KAAK,EAAC;AAEV,IAAA,MAAM,UAAmC,EAAC;AAE1C,IAAA,IAAI,IAAM,EAAA;AACR,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,QAClB,kCAAkC,EAAE,CAAA,mDAAA;AAAA,OACtC;AACA,MAAM,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,OAAQ,CAAA,cAAA;AAAA,QAChC,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,eAAgB,EAAA;AAAA,QACtC;AAAA,OACF;AAEA,MAAM,MAAA,EAAE,OAAU,GAAA,MAAA;AAElB,MAAA,IAAI,SAAY,GAAA,IAAA;AAChB,MAAI,IAAA,IAAA,CAAK,QAAQ,4BAA8B,EAAA;AAC7C,QAAA,IAAI,UAAU,CAAG,EAAA;AACf,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,YAClB,kCAAkC,EAAE,CAAA,qCAAA;AAAA,WACtC;AACA,UAAY,SAAA,GAAA,KAAA;AAAA;AACd;AAGF,MAAI,IAAA,IAAA,CAAK,QAAQ,6BAA+B,EAAA;AAG9C,QAAA,MAAM,iBACJ,KAAQ,GAAA,CAAA,GAAK,OAAO,OAAQ,CAAA,MAAA,GAAS,QAAS,GAAM,GAAA,GAAA;AACtD,QAAI,IAAA,cAAA,IAAkB,IAAK,CAAA,OAAA,CAAQ,6BAA+B,EAAA;AAChE,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,YAClB,CAAkC,+BAAA,EAAA,EAAE,CAAe,YAAA,EAAA,MAAA,CAAO,QAAQ,MAAM,CAAA,sCAAA;AAAA,WAC1E;AAAA,SACK,MAAA;AACL,UAAM,MAAA,MAAA,GAAS,uBAAuB,cAAc,CAAA,uBAAA,CAAA;AACpD,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,GAAA,EAAM,MAAM,CAAA;AAAA,WAClD;AACA,UAAM,MAAA,IAAA,CAAK,QAAQ,yBAA0B,CAAA;AAAA,YAC3C,WAAa,EAAA,EAAA;AAAA,YACb,MAAQ,EAAA;AAAA,cACN,UAAY,EAAA,CAAA,6CAAA,EAAgD,MAAM,CAAA,EAAA,EAAK,MAAM,CAAA;AAAA;AAC/E,WACD,CAAA;AACD,UAAY,SAAA,GAAA,KAAA;AAAA;AACd;AAEF,MAAA,IAAI,SAAW,EAAA;AACb,QAAW,KAAA,MAAA,SAAA,IAAa,OAAO,OAAS,EAAA;AACtC,UAAA,OAAA,CAAQ,KAAK,SAAS,CAAA;AAAA;AACxB;AACF;AAGF,IAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,UAAA,CAAW,aAAc,CAAA;AAAA,MAC1C,IAAM,EAAA,OAAA;AAAA,MACN,KAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA;AACH,EAEA,MAAM,QAAQ,MAAoC,EAAA;AAChD,IAAM,MAAA,EAAE,OAAU,GAAA,MAAA;AAClB,IAAA,IAAI,CAAC,IAAK,CAAA,mBAAA,EAAsB,CAAA,QAAA,CAAS,KAAK,CAAG,EAAA;AAC/C,MAAA;AAAA;AAGF,IAAA,MAAM,EAAE,MAAA,EAAQ,QAAU,EAAA,UAAA,KAAe,IAAK,CAAA,OAAA;AAC9C,IAAM,MAAA,YAAA,GAAe,SAAS,eAAgB,EAAA;AAC9C,IAAA,MAAA,CAAO,KAAM,CAAA,CAAA,oBAAA,EAAuB,YAAY,CAAA,UAAA,EAAa,KAAK,CAAQ,MAAA,CAAA,CAAA;AAE1E,IAAI,IAAA,CAAC,SAAS,YAAc,EAAA;AAC1B,MAAA;AAAA;AAGF,IAAA,MAAM,MAAS,GAAA,MAAM,QAAS,CAAA,YAAA,CAAa,QAAQ,MAAM,CAAA;AAEzD,IAAI,IAAA,MAAA,CAAO,SAAS,OAAS,EAAA;AAC3B,MAAI,IAAA,MAAA,CAAO,KAAM,CAAA,MAAA,GAAS,CAAG,EAAA;AAC3B,QAAM,MAAA,eAAA,GAAkB,MAAM,IAAA,CAAK,OAAQ,CAAA,yBAAA;AAAA,UACzC;AAAA,SACF;AAEA,QAAA,IAAI,CAAC,eAAiB,EAAA;AACpB,UAAO,MAAA,CAAA,KAAA;AAAA,YACL,uBAAuB,YAAY,CAAA,qEAAA;AAAA,WACrC;AAAA,SACK,MAAA;AACL,UAAA,MAAM,OACJ,eAAgB,CAAA,MAAA,KAAW,SACvB,GAAA,MAAM,KAAK,OAAQ,CAAA,WAAA,CAAY,eAAgB,CAAA,EAAE,IACjD,MAAM,IAAA,CAAK,OAAQ,CAAA,YAAA,CAAa,gBAAgB,EAAE,CAAA;AAExD,UAAA,IAAI,CAAC,IAAM,EAAA;AACT,YAAA,MAAM,IAAI,KAAA;AAAA,cACR,yFAAyF,YAAY,CAAA,CAAA;AAAA,aACvG;AAAA;AAEF,UAAA,MAAM,KAAK,OAAQ,CAAA,kBAAA,CAAmB,IAAK,CAAA,EAAA,EAAI,OAAO,KAAK,CAAA;AAAA;AAC7D;AAGF,MAAI,IAAA,MAAA,CAAO,OAAQ,CAAA,MAAA,GAAS,CAAG,EAAA;AAC7B,QAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,wBAAyB,CAAA,MAAA,CAAO,OAAO,CAAA;AAAA;AAG5D,MAAM,MAAA,UAAA,CAAW,cAAc,MAAM,CAAA;AACrC,MAAO,MAAA,CAAA,KAAA;AAAA,QACL,CAAA,oBAAA,EAAuB,YAAY,CAAA,uBAAA,EAA0B,KAAK,CAAA,OAAA;AAAA,OACpE;AAAA,KACK,MAAA;AACL,MAAO,MAAA,CAAA,KAAA;AAAA,QACL,CAAA,oBAAA,EAAuB,YAAY,CAAA,2BAAA,EAA8B,KAAK,CAAA,CAAA;AAAA,OACxE;AAAA;AACF;AACF,EAEA,mBAAgC,GAAA;AAC9B,IAAM,MAAA,EAAE,QAAS,EAAA,GAAI,IAAK,CAAA,OAAA;AAC1B,IAAA,MAAM,SAAS,QAAS,CAAA,YAAA,GACpB,SAAS,YAAa,CAAA,mBAAA,KACtB,EAAC;AACL,IAAO,OAAA,MAAA;AAAA;AAEX;;;;"}

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend-module-incremental-ingestion",
-  "version": "0.6.1-next.0",
+  "version": "0.6.1-next.1",
   "description": "An entity provider for streaming large asset sources into the catalog",
   "backstage": {
     "role": "backend-plugin-module",
@@ -62,15 +62,16 @@
   },
   "dependencies": {
     "@backstage/backend-common": "^0.25.0",
-    "@backstage/backend-plugin-api": "1.0.3-next.0",
+    "@backstage/backend-plugin-api": "1.1.0-next.1",
     "@backstage/catalog-model": "1.7.1",
     "@backstage/config": "1.3.0",
     "@backstage/errors": "1.2.5",
-    "@backstage/plugin-catalog-backend": "1.28.1-next.0",
-    "@backstage/plugin-catalog-node": "1.14.1-next.0",
-    "@backstage/plugin-events-node": "0.4.6-next.0",
+    "@backstage/plugin-catalog-backend": "1.29.0-next.1",
+    "@backstage/plugin-catalog-node": "1.15.0-next.1",
+    "@backstage/plugin-events-node": "0.4.6-next.1",
     "@backstage/plugin-permission-common": "0.8.2",
     "@backstage/types": "1.2.0",
+    "@opentelemetry/api": "^1.3.0",
     "@types/express": "^4.17.6",
     "express": "^4.17.1",
     "express-promise-router": "^4.1.0",
@@ -79,9 +80,9 @@
     "uuid": "^11.0.0"
   },
   "devDependencies": {
-    "@backstage/backend-defaults": "0.6.0-next.0",
-    "@backstage/backend-test-utils": "1.2.0-next.0",
-    "@backstage/cli": "0.29.3-next.0",
+    "@backstage/backend-defaults": "0.6.0-next.1",
+    "@backstage/backend-test-utils": "1.2.0-next.1",
+    "@backstage/cli": "0.29.3-next.1",
     "@types/luxon": "^3.0.0"
   }
 }