@backstage/plugin-catalog-backend-module-incremental-ingestion 0.6.0 → 0.6.1-next.0

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @backstage/plugin-catalog-backend-module-incremental-ingestion
 
+## 0.6.1-next.0
+
+### Patch Changes
+
+- cce9cae: Deprecate old-backend-system `IncrementalCatalogBuilder`
+- 5aa44d2: Wire up the events together in the new backend system
+- cbfc69e: Create a `dev/index.ts` entrypoint for `yarn start`
+- 3ca5f70: Ensure that the scheduled worker task doesn't run at an unreasonably high frequency
+- Updated dependencies
+  - @backstage/backend-plugin-api@1.0.3-next.0
+  - @backstage/plugin-catalog-backend@1.28.1-next.0
+  - @backstage/plugin-events-node@0.4.6-next.0
+  - @backstage/catalog-model@1.7.1
+  - @backstage/config@1.3.0
+  - @backstage/errors@1.2.5
+  - @backstage/types@1.2.0
+  - @backstage/plugin-catalog-node@1.14.1-next.0
+  - @backstage/plugin-permission-common@0.8.2
+
 ## 0.6.0
 
 ### Minor Changes

package/dist/alpha.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"alpha.cjs.js","sources":["../src/alpha.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { default as feature } from './module';\n\n/** @alpha */\nconst _feature = feature;\nexport default _feature;\n"],"names":["feature"],"mappings":";;;;;;AAmBA,MAAM,QAAW,GAAAA;;;;"}
+{"version":3,"file":"alpha.cjs.js","sources":["../src/alpha.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { default as feature } from './module';\n\n/**\n * @alpha\n * @deprecated Please import from `@backstage/plugin-catalog-backend-module-incremental-ingestion` instead\n */\nconst _feature = feature;\nexport default _feature;\n"],"names":["feature"],"mappings":";;;;;;AAsBA,MAAM,QAAW,GAAAA;;;;"}

package/dist/alpha.d.ts

@@ -1,6 +1,9 @@
 import * as _backstage_backend_plugin_api from '@backstage/backend-plugin-api';
 
-/** @alpha */
+/**
+ * @alpha
+ * @deprecated Please import from `@backstage/plugin-catalog-backend-module-incremental-ingestion` instead
+ */
 declare const _feature: _backstage_backend_plugin_api.BackendFeature;
 
 export { _feature as default };

package/dist/engine/IncrementalIngestionEngine.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"IncrementalIngestionEngine.cjs.js","sources":["../../src/engine/IncrementalIngestionEngine.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport type { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { IterationEngine, IterationEngineOptions } from '../types';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { performance } from 'perf_hooks';\nimport { Duration } from 'luxon';\nimport { v4 } from 'uuid';\nimport { stringifyError } from '@backstage/errors';\nimport { EventParams, EventSubscriber } from '@backstage/plugin-events-node';\nimport { HumanDuration } from '@backstage/types';\n\nexport class IncrementalIngestionEngine\n  implements IterationEngine, EventSubscriber\n{\n  private readonly restLength: Duration;\n  private readonly backoff: HumanDuration[];\n\n  private manager: IncrementalIngestionDatabaseManager;\n\n  constructor(private options: IterationEngineOptions) {\n    this.manager = options.manager;\n    this.restLength = Duration.fromObject(options.restLength);\n    this.backoff = options.backoff ?? [\n      { minutes: 1 },\n      { minutes: 5 },\n      { minutes: 30 },\n      { hours: 3 },\n    ];\n  }\n\n  async taskFn(signal: AbortSignal) {\n    try {\n      this.options.logger.debug('Begin tick');\n      await this.handleNextAction(signal);\n    } catch (error) {\n      this.options.logger.error(`${error}`);\n      throw error;\n    } finally {\n      this.options.logger.debug('End tick');\n    }\n  }\n\n  async handleNextAction(signal: AbortSignal) {\n    await this.options.ready;\n\n    const result = await this.getCurrentAction();\n    if (result) {\n      const { ingestionId, nextActionAt, nextAction, attempts } = result;\n\n      switch (nextAction) {\n        case 'rest':\n          if (Date.now() > nextActionAt) {\n            await this.manager.clearFinishedIngestions(\n              this.options.provider.getProviderName(),\n            );\n            this.options.logger.debug(\n              `incremental-engine: Ingestion ${ingestionId} rest period complete. Ingestion will start again`,\n            );\n\n            await this.manager.setProviderComplete(ingestionId);\n          } else {\n            this.options.logger.debug(\n              `incremental-engine: Ingestion '${ingestionId}' rest period continuing`,\n            );\n          }\n          break;\n        case 'ingest':\n          try {\n            await this.manager.setProviderBursting(ingestionId);\n            const done = await this.ingestOneBurst(ingestionId, signal);\n            if (done) {\n              this.options.logger.info(\n                `incremental-engine: Ingestion '${ingestionId}' complete, transitioning to rest period of ${this.restLength.toHuman()}`,\n              );\n              await this.manager.setProviderResting(\n                ingestionId,\n                this.restLength,\n              );\n            } else {\n              await this.manager.setProviderInterstitial(ingestionId);\n              this.options.logger.debug(\n                `incremental-engine: Ingestion '${ingestionId}' continuing`,\n              );\n            }\n          } catch (error) {\n            if (\n              (error as Error).message &&\n              (error as Error).message === 'CANCEL'\n            ) {\n              this.options.logger.info(\n                `incremental-engine: Ingestion '${ingestionId}' canceled`,\n              );\n              await this.manager.setProviderCanceling(\n                ingestionId,\n                (error as Error).message,\n              );\n            } else {\n              const currentBackoff = Duration.fromObject(\n                this.backoff[Math.min(this.backoff.length - 1, attempts)],\n              );\n\n              const backoffLength = currentBackoff.as('milliseconds');\n              this.options.logger.error(\n                `incremental-engine: Ingestion '${ingestionId}' failed`,\n                error,\n              );\n\n              const truncatedError = stringifyError(error).substring(0, 700);\n              this.options.logger.error(\n                `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`,\n              );\n\n              await this.manager.setProviderBackoff(\n                ingestionId,\n                attempts,\n                error as Error,\n                backoffLength,\n              );\n            }\n          }\n          break;\n        case 'backoff':\n          if (Date.now() > nextActionAt) {\n            this.options.logger.info(\n              `incremental-engine: Ingestion '${ingestionId}' backoff complete, will attempt to resume`,\n            );\n            await this.manager.setProviderIngesting(ingestionId);\n          } else {\n            this.options.logger.debug(\n              `incremental-engine: Ingestion '${ingestionId}' backoff continuing`,\n            );\n          }\n          break;\n        case 'cancel':\n          this.options.logger.info(\n            `incremental-engine: Ingestion '${ingestionId}' canceling, will restart`,\n          );\n          await this.manager.setProviderCanceled(ingestionId);\n          break;\n        default:\n          this.options.logger.error(\n            `incremental-engine: Ingestion '${ingestionId}' received unknown action '${nextAction}'`,\n          );\n      }\n    } else {\n      this.options.logger.error(\n        `incremental-engine: Engine tried to create duplicate ingestion record for provider '${this.options.provider.getProviderName()}'.`,\n      );\n    }\n  }\n\n  async getCurrentAction() {\n    const providerName = this.options.provider.getProviderName();\n    const record = await this.manager.getCurrentIngestionRecord(providerName);\n    if (record) {\n      this.options.logger.debug(\n        `incremental-engine: Ingestion record found: '${record.id}'`,\n      );\n      return {\n        ingestionId: record.id,\n        nextAction: record.next_action as 'rest' | 'ingest' | 'backoff',\n        attempts: record.attempts as number,\n        nextActionAt: record.next_action_at.valueOf() as number,\n      };\n    }\n    const result = await this.manager.createProviderIngestionRecord(\n      providerName,\n    );\n    if (result) {\n      this.options.logger.info(\n        `incremental-engine: Ingestion record created: '${result.ingestionId}'`,\n      );\n    }\n    return result;\n  }\n\n  async ingestOneBurst(id: string, signal: AbortSignal) {\n    const lastMark = await this.manager.getLastMark(id);\n\n    const cursor = lastMark ? lastMark.cursor : undefined;\n    let sequence = lastMark ? lastMark.sequence + 1 : 0;\n\n    const start = performance.now();\n    let count = 0;\n    let done = false;\n    this.options.logger.info(\n      `incremental-engine: Ingestion '${id}' burst initiated`,\n    );\n\n    await this.options.provider.around(async (context: unknown) => {\n      let next = await this.options.provider.next(context, cursor);\n      count++;\n      for (;;) {\n        done = next.done;\n        await this.mark({\n          id,\n          sequence,\n          entities: next?.entities,\n          done: next.done,\n          cursor: next?.cursor,\n        });\n        if (signal.aborted || next.done) {\n          break;\n        } else {\n          next = await this.options.provider.next(context, next.cursor);\n          count++;\n          sequence++;\n        }\n      }\n    });\n\n    this.options.logger.info(\n      `incremental-engine: Ingestion '${id}' burst complete. (${count} batches in ${Math.round(\n        performance.now() - start,\n      )}ms).`,\n    );\n    return done;\n  }\n\n  async mark(options: {\n    id: string;\n    sequence: number;\n    entities?: DeferredEntity[];\n    done: boolean;\n    cursor?: unknown;\n  }) {\n    const { id, sequence, entities, done, cursor } = options;\n    this.options.logger.debug(\n      `incremental-engine: Ingestion '${id}': MARK ${\n        entities ? entities.length : 0\n      } entities, cursor: ${\n        cursor ? JSON.stringify(cursor) : 'none'\n      }, done: ${done}`,\n    );\n    const markId = v4();\n\n    await this.manager.createMark({\n      record: {\n        id: markId,\n        ingestion_id: id,\n        cursor,\n        sequence,\n      },\n    });\n\n    if (entities && entities.length > 0) {\n      await this.manager.createMarkEntities(markId, entities);\n    }\n\n    const added =\n      entities?.map(deferred => ({\n        ...deferred,\n        entity: {\n          ...deferred.entity,\n          metadata: {\n            ...deferred.entity.metadata,\n            annotations: {\n              ...deferred.entity.metadata.annotations,\n            },\n          },\n        },\n      })) ?? [];\n\n    const removed: { entityRef: string }[] = [];\n\n    if (done) {\n      this.options.logger.info(\n        `incremental-engine: Ingestion '${id}': Final page reached, calculating removed entities`,\n      );\n      const result = await this.manager.computeRemoved(\n        this.options.provider.getProviderName(),\n        id,\n      );\n\n      const { total } = result;\n\n      let doRemoval = true;\n      if (this.options.rejectEmptySourceCollections) {\n        if (total === 0) {\n          this.options.logger.error(\n            `incremental-engine: Ingestion '${id}': Rejecting empty entity collection!`,\n          );\n          doRemoval = false;\n        }\n      }\n\n      if (this.options.rejectRemovalsAbovePercentage) {\n        // If the total entities upserted in this ingestion is 0, then\n        // 100% of entities are stale and marked for removal.\n        const percentRemoved =\n          total > 0 ? (result.removed.length / total) * 100 : 100;\n        if (percentRemoved <= this.options.rejectRemovalsAbovePercentage) {\n          this.options.logger.info(\n            `incremental-engine: Ingestion '${id}': Removing ${result.removed.length} entities that have no matching assets`,\n          );\n        } else {\n          const notice = `Attempted to remove ${percentRemoved}% of matching entities!`;\n          this.options.logger.error(\n            `incremental-engine: Ingestion '${id}': ${notice}`,\n          );\n          await this.manager.updateIngestionRecordById({\n            ingestionId: id,\n            update: {\n              last_error: `REMOVAL_THRESHOLD exceeded on ingestion mark ${markId}: ${notice}`,\n            },\n          });\n          doRemoval = false;\n        }\n      }\n      if (doRemoval) {\n        for (const entityRef of result.removed) {\n          removed.push(entityRef);\n        }\n      }\n    }\n\n    await this.options.connection.applyMutation({\n      type: 'delta',\n      added,\n      removed,\n    });\n  }\n\n  async onEvent(params: EventParams): Promise<void> {\n    const { topic } = params;\n    if (!this.supportsEventTopics().includes(topic)) {\n      return;\n    }\n\n    const { logger, provider, connection } = this.options;\n    const providerName = provider.getProviderName();\n    logger.debug(`incremental-engine: ${providerName} received ${topic} event`);\n\n    if (!provider.eventHandler) {\n      return;\n    }\n\n    const result = await provider.eventHandler.onEvent(params);\n\n    if (result.type === 'delta') {\n      if (result.added.length > 0) {\n        const ingestionRecord = await this.manager.getCurrentIngestionRecord(\n          providerName,\n        );\n\n        if (!ingestionRecord) {\n          logger.debug(\n            `incremental-engine: ${providerName} skipping delta addition because incremental ingestion is restarting.`,\n          );\n        } else {\n          const mark =\n            ingestionRecord.status === 'resting'\n              ? await this.manager.getLastMark(ingestionRecord.id)\n              : await this.manager.getFirstMark(ingestionRecord.id);\n\n          if (!mark) {\n            throw new Error(\n              `Cannot apply delta, page records are missing! Please re-run incremental ingestion for ${providerName}.`,\n            );\n          }\n          await this.manager.createMarkEntities(mark.id, result.added);\n        }\n      }\n\n      if (result.removed.length > 0) {\n        await this.manager.deleteEntityRecordsByRef(result.removed);\n      }\n\n      await connection.applyMutation(result);\n      logger.debug(\n        `incremental-engine: ${providerName} processed delta from '${topic}' event`,\n      );\n    } else {\n      logger.debug(\n        `incremental-engine: ${providerName} ignored event from topic '${topic}'`,\n      );\n    }\n  }\n\n  supportsEventTopics(): string[] {\n    const { provider } = this.options;\n    const topics = provider.eventHandler\n      ? provider.eventHandler.supportsEventTopics()\n      : [];\n    return topics;\n  }\n}\n"],"names":["Duration","stringifyError","performance","v4"],"mappings":";;;;;;;AA0BO,MAAM,0BAEb,CAAA;AAAA,EAME,YAAoB,OAAiC,EAAA;AAAjC,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAClB,IAAA,IAAA,CAAK,UAAU,OAAQ,CAAA,OAAA;AACvB,IAAA,IAAA,CAAK,UAAa,GAAAA,cAAA,CAAS,UAAW,CAAA,OAAA,CAAQ,UAAU,CAAA;AACxD,IAAK,IAAA,CAAA,OAAA,GAAU,QAAQ,OAAW,IAAA;AAAA,MAChC,EAAE,SAAS,CAAE,EAAA;AAAA,MACb,EAAE,SAAS,CAAE,EAAA;AAAA,MACb,EAAE,SAAS,EAAG,EAAA;AAAA,MACd,EAAE,OAAO,CAAE;AAAA,KACb;AAAA;AACF,EAdiB,UAAA;AAAA,EACA,OAAA;AAAA,EAET,OAAA;AAAA,EAaR,MAAM,OAAO,MAAqB,EAAA;AAChC,IAAI,IAAA;AACF,MAAK,IAAA,CAAA,OAAA,CAAQ,MAAO,CAAA,KAAA,CAAM,YAAY,CAAA;AACtC,MAAM,MAAA,IAAA,CAAK,iBAAiB,MAAM,CAAA;AAAA,aAC3B,KAAO,EAAA;AACd,MAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,CAAO,KAAM,CAAA,CAAA,EAAG,KAAK,CAAE,CAAA,CAAA;AACpC,MAAM,MAAA,KAAA;AAAA,KACN,SAAA;AACA,MAAK,IAAA,CAAA,OAAA,CAAQ,MAAO,CAAA,KAAA,CAAM,UAAU,CAAA;AAAA;AACtC;AACF,EAEA,MAAM,iBAAiB,MAAqB,EAAA;AAC1C,IAAA,MAAM,KAAK,OAAQ,CAAA,KAAA;AAEnB,IAAM,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,gBAAiB,EAAA;AAC3C,IAAA,IAAI,MAAQ,EAAA;AACV,MAAA,MAAM,EAAE,WAAA,EAAa,YAAc,EAAA,UAAA,EAAY,UAAa,GAAA,MAAA;AAE5D,MAAA,QAAQ,UAAY;AAAA,QAClB,KAAK,MAAA;AACH,UAAI,IAAA,IAAA,CAAK,GAAI,EAAA,GAAI,YAAc,EAAA;AAC7B,YAAA,MAAM,KAAK,OAAQ,CAAA,uBAAA;AAAA,cACjB,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,eAAgB;AAAA,aACxC;AACA,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,cAClB,iCAAiC,WAAW,CAAA,iDAAA;AAAA,aAC9C;AAEA,YAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,mBAAA,CAAoB,WAAW,CAAA;AAAA,WAC7C,MAAA;AACL,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,wBAAA;AAAA,aAC/C;AAAA;AAEF,UAAA;AAAA,QACF,KAAK,QAAA;AACH,UAAI,IAAA;AACF,YAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,mBAAA,CAAoB,WAAW,CAAA;AAClD,YAAA,MAAM,IAAO,GAAA,MAAM,IAAK,CAAA,cAAA,CAAe,aAAa,MAAM,CAAA;AAC1D,YAAA,IAAI,IAAM,EAAA;AACR,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,4CAAA,EAA+C,IAAK,CAAA,UAAA,CAAW,SAAS,CAAA;AAAA,eACvH;AACA,cAAA,MAAM,KAAK,OAAQ,CAAA,kBAAA;AAAA,gBACjB,WAAA;AAAA,gBACA,IAAK,CAAA;AAAA,eACP;AAAA,aACK,MAAA;AACL,cAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,uBAAA,CAAwB,WAAW,CAAA;AACtD,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,YAAA;AAAA,eAC/C;AAAA;AACF,mBACO,KAAO,EAAA;AACd,YAAA,IACG,KAAgB,CAAA,OAAA,IAChB,KAAgB,CAAA,OAAA,KAAY,QAC7B,EAAA;AACA,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,UAAA;AAAA,eAC/C;AACA,cAAA,MAAM,KAAK,OAAQ,CAAA,oBAAA;AAAA,gBACjB,WAAA;AAAA,gBACC,KAAgB,CAAA;AAAA,eACnB;AAAA,aACK,MAAA;AACL,cAAA,MAAM,iBAAiBA,cAAS,CAAA,UAAA;AAAA,gBAC9B,IAAA,CAAK,QAAQ,IAAK,CAAA,GAAA,CAAI,KAAK,OAAQ,CAAA,MAAA,GAAS,CAAG,EAAA,QAAQ,CAAC;AAAA,eAC1D;AAEA,cAAM,MAAA,aAAA,GAAgB,cAAe,CAAA,EAAA,CAAG,cAAc,CAAA;AACtD,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,QAAA,CAAA;AAAA,gBAC7C;AAAA,eACF;AAEA,cAAA,MAAM,iBAAiBC,qBAAe,CAAA,KAAK,CAAE,CAAA,SAAA,CAAU,GAAG,GAAG,CAAA;AAC7D,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,oEAAA,EAAuE,eAAe,OAAQ,EAAC,KAAK,cAAc,CAAA,CAAA;AAAA,eACjK;AAEA,cAAA,MAAM,KAAK,OAAQ,CAAA,kBAAA;AAAA,gBACjB,WAAA;AAAA,gBACA,QAAA;AAAA,gBACA,KAAA;AAAA,gBACA;AAAA,eACF;AAAA;AACF;AAEF,UAAA;AAAA,QACF,KAAK,SAAA;AACH,UAAI,IAAA,IAAA,CAAK,GAAI,EAAA,GAAI,YAAc,EAAA;AAC7B,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,0CAAA;AAAA,aAC/C;AACA,YAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,oBAAA,CAAqB,WAAW,CAAA;AAAA,WAC9C,MAAA;AACL,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,oBAAA;AAAA,aAC/C;AAAA;AAEF,UAAA;AAAA,QACF,KAAK,QAAA;AACH,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,YAClB,kCAAkC,WAAW,CAAA,yBAAA;AAAA,WAC/C;AACA,UAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,mBAAA,CAAoB,WAAW,CAAA;AAClD,UAAA;AAAA,QACF;AACE,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,WAAW,CAAA,2BAAA,EAA8B,UAAU,CAAA,CAAA;AAAA,WACvF;AAAA;AACJ,KACK,MAAA;AACL,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,QAClB,CAAuF,oFAAA,EAAA,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,iBAAiB,CAAA,EAAA;AAAA,OAChI;AAAA;AACF;AACF,EAEA,MAAM,gBAAmB,GAAA;AACvB,IAAA,MAAM,YAAe,GAAA,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,eAAgB,EAAA;AAC3D,IAAA,MAAM,MAAS,GAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,0BAA0B,YAAY,CAAA;AACxE,IAAA,IAAI,MAAQ,EAAA;AACV,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,QAClB,CAAA,6CAAA,EAAgD,OAAO,EAAE,CAAA,CAAA;AAAA,OAC3D;AACA,MAAO,OAAA;AAAA,QACL,aAAa,MAAO,CAAA,EAAA;AAAA,QACpB,YAAY,MAAO,CAAA,WAAA;AAAA,QACnB,UAAU,MAAO,CAAA,QAAA;AAAA,QACjB,YAAA,EAAc,MAAO,CAAA,cAAA,CAAe,OAAQ;AAAA,OAC9C;AAAA;AAEF,IAAM,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,OAAQ,CAAA,6BAAA;AAAA,MAChC;AAAA,KACF;AACA,IAAA,IAAI,MAAQ,EAAA;AACV,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,QAClB,CAAA,+CAAA,EAAkD,OAAO,WAAW,CAAA,CAAA;AAAA,OACtE;AAAA;AAEF,IAAO,OAAA,MAAA;AAAA;AACT,EAEA,MAAM,cAAe,CAAA,EAAA,EAAY,MAAqB,EAAA;AACpD,IAAA,MAAM,QAAW,GAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,YAAY,EAAE,CAAA;AAElD,IAAM,MAAA,MAAA,GAAS,QAAW,GAAA,QAAA,CAAS,MAAS,GAAA,KAAA,CAAA;AAC5C,IAAA,IAAI,QAAW,GAAA,QAAA,GAAW,QAAS,CAAA,QAAA,GAAW,CAAI,GAAA,CAAA;AAElD,IAAM,MAAA,KAAA,GAAQC,uBAAY,GAAI,EAAA;AAC9B,IAAA,IAAI,KAAQ,GAAA,CAAA;AACZ,IAAA,IAAI,IAAO,GAAA,KAAA;AACX,IAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,MAClB,kCAAkC,EAAE,CAAA,iBAAA;AAAA,KACtC;AAEA,IAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,QAAS,CAAA,MAAA,CAAO,OAAO,OAAqB,KAAA;AAC7D,MAAA,IAAI,OAAO,MAAM,IAAA,CAAK,QAAQ,QAAS,CAAA,IAAA,CAAK,SAAS,MAAM,CAAA;AAC3D,MAAA,KAAA,EAAA;AACA,MAAS,WAAA;AACP,QAAA,IAAA,GAAO,IAAK,CAAA,IAAA;AACZ,QAAA,MAAM,KAAK,IAAK,CAAA;AAAA,UACd,EAAA;AAAA,UACA,QAAA;AAAA,UACA,UAAU,IAAM,EAAA,QAAA;AAAA,UAChB,MAAM,IAAK,CAAA,IAAA;AAAA,UACX,QAAQ,IAAM,EAAA;AAAA,SACf,CAAA;AACD,QAAI,IAAA,MAAA,CAAO,OAAW,IAAA,IAAA,CAAK,IAAM,EAAA;AAC/B,UAAA;AAAA,SACK,MAAA;AACL,UAAA,IAAA,GAAO,MAAM,IAAK,CAAA,OAAA,CAAQ,SAAS,IAAK,CAAA,OAAA,EAAS,KAAK,MAAM,CAAA;AAC5D,UAAA,KAAA,EAAA;AACA,UAAA,QAAA,EAAA;AAAA;AACF;AACF,KACD,CAAA;AAED,IAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,MAClB,CAAkC,+BAAA,EAAA,EAAE,CAAsB,mBAAA,EAAA,KAAK,eAAe,IAAK,CAAA,KAAA;AAAA,QACjFA,sBAAA,CAAY,KAAQ,GAAA;AAAA,OACrB,CAAA,IAAA;AAAA,KACH;AACA,IAAO,OAAA,IAAA;AAAA;AACT,EAEA,MAAM,KAAK,OAMR,EAAA;AACD,IAAA,MAAM,EAAE,EAAI,EAAA,QAAA,EAAU,QAAU,EAAA,IAAA,EAAM,QAAW,GAAA,OAAA;AACjD,IAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,MAClB,CAAkC,+BAAA,EAAA,EAAE,CAClC,QAAA,EAAA,QAAA,GAAW,SAAS,MAAS,GAAA,CAC/B,CACE,mBAAA,EAAA,MAAA,GAAS,KAAK,SAAU,CAAA,MAAM,CAAI,GAAA,MACpC,WAAW,IAAI,CAAA;AAAA,KACjB;AACA,IAAA,MAAM,SAASC,OAAG,EAAA;AAElB,IAAM,MAAA,IAAA,CAAK,QAAQ,UAAW,CAAA;AAAA,MAC5B,MAAQ,EAAA;AAAA,QACN,EAAI,EAAA,MAAA;AAAA,QACJ,YAAc,EAAA,EAAA;AAAA,QACd,MAAA;AAAA,QACA;AAAA;AACF,KACD,CAAA;AAED,IAAI,IAAA,QAAA,IAAY,QAAS,CAAA,MAAA,GAAS,CAAG,EAAA;AACnC,MAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,kBAAmB,CAAA,MAAA,EAAQ,QAAQ,CAAA;AAAA;AAGxD,IAAM,MAAA,KAAA,GACJ,QAAU,EAAA,GAAA,CAAI,CAAa,QAAA,MAAA;AAAA,MACzB,GAAG,QAAA;AAAA,MACH,MAAQ,EAAA;AAAA,QACN,GAAG,QAAS,CAAA,MAAA;AAAA,QACZ,QAAU,EAAA;AAAA,UACR,GAAG,SAAS,MAAO,CAAA,QAAA;AAAA,UACnB,WAAa,EAAA;AAAA,YACX,GAAG,QAAS,CAAA,MAAA,CAAO,QAAS,CAAA;AAAA;AAC9B;AACF;AACF,KACF,CAAE,KAAK,EAAC;AAEV,IAAA,MAAM,UAAmC,EAAC;AAE1C,IAAA,IAAI,IAAM,EAAA;AACR,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,QAClB,kCAAkC,EAAE,CAAA,mDAAA;AAAA,OACtC;AACA,MAAM,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,OAAQ,CAAA,cAAA;AAAA,QAChC,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,eAAgB,EAAA;AAAA,QACtC;AAAA,OACF;AAEA,MAAM,MAAA,EAAE,OAAU,GAAA,MAAA;AAElB,MAAA,IAAI,SAAY,GAAA,IAAA;AAChB,MAAI,IAAA,IAAA,CAAK,QAAQ,4BAA8B,EAAA;AAC7C,QAAA,IAAI,UAAU,CAAG,EAAA;AACf,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,YAClB,kCAAkC,EAAE,CAAA,qCAAA;AAAA,WACtC;AACA,UAAY,SAAA,GAAA,KAAA;AAAA;AACd;AAGF,MAAI,IAAA,IAAA,CAAK,QAAQ,6BAA+B,EAAA;AAG9C,QAAA,MAAM,iBACJ,KAAQ,GAAA,CAAA,GAAK,OAAO,OAAQ,CAAA,MAAA,GAAS,QAAS,GAAM,GAAA,GAAA;AACtD,QAAI,IAAA,cAAA,IAAkB,IAAK,CAAA,OAAA,CAAQ,6BAA+B,EAAA;AAChE,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,YAClB,CAAkC,+BAAA,EAAA,EAAE,CAAe,YAAA,EAAA,MAAA,CAAO,QAAQ,MAAM,CAAA,sCAAA;AAAA,WAC1E;AAAA,SACK,MAAA;AACL,UAAM,MAAA,MAAA,GAAS,uBAAuB,cAAc,CAAA,uBAAA,CAAA;AACpD,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,GAAA,EAAM,MAAM,CAAA;AAAA,WAClD;AACA,UAAM,MAAA,IAAA,CAAK,QAAQ,yBAA0B,CAAA;AAAA,YAC3C,WAAa,EAAA,EAAA;AAAA,YACb,MAAQ,EAAA;AAAA,cACN,UAAY,EAAA,CAAA,6CAAA,EAAgD,MAAM,CAAA,EAAA,EAAK,MAAM,CAAA;AAAA;AAC/E,WACD,CAAA;AACD,UAAY,SAAA,GAAA,KAAA;AAAA;AACd;AAEF,MAAA,IAAI,SAAW,EAAA;AACb,QAAW,KAAA,MAAA,SAAA,IAAa,OAAO,OAAS,EAAA;AACtC,UAAA,OAAA,CAAQ,KAAK,SAAS,CAAA;AAAA;AACxB;AACF;AAGF,IAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,UAAA,CAAW,aAAc,CAAA;AAAA,MAC1C,IAAM,EAAA,OAAA;AAAA,MACN,KAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA;AACH,EAEA,MAAM,QAAQ,MAAoC,EAAA;AAChD,IAAM,MAAA,EAAE,OAAU,GAAA,MAAA;AAClB,IAAA,IAAI,CAAC,IAAK,CAAA,mBAAA,EAAsB,CAAA,QAAA,CAAS,KAAK,CAAG,EAAA;AAC/C,MAAA;AAAA;AAGF,IAAA,MAAM,EAAE,MAAA,EAAQ,QAAU,EAAA,UAAA,KAAe,IAAK,CAAA,OAAA;AAC9C,IAAM,MAAA,YAAA,GAAe,SAAS,eAAgB,EAAA;AAC9C,IAAA,MAAA,CAAO,KAAM,CAAA,CAAA,oBAAA,EAAuB,YAAY,CAAA,UAAA,EAAa,KAAK,CAAQ,MAAA,CAAA,CAAA;AAE1E,IAAI,IAAA,CAAC,SAAS,YAAc,EAAA;AAC1B,MAAA;AAAA;AAGF,IAAA,MAAM,MAAS,GAAA,MAAM,QAAS,CAAA,YAAA,CAAa,QAAQ,MAAM,CAAA;AAEzD,IAAI,IAAA,MAAA,CAAO,SAAS,OAAS,EAAA;AAC3B,MAAI,IAAA,MAAA,CAAO,KAAM,CAAA,MAAA,GAAS,CAAG,EAAA;AAC3B,QAAM,MAAA,eAAA,GAAkB,MAAM,IAAA,CAAK,OAAQ,CAAA,yBAAA;AAAA,UACzC;AAAA,SACF;AAEA,QAAA,IAAI,CAAC,eAAiB,EAAA;AACpB,UAAO,MAAA,CAAA,KAAA;AAAA,YACL,uBAAuB,YAAY,CAAA,qEAAA;AAAA,WACrC;AAAA,SACK,MAAA;AACL,UAAA,MAAM,OACJ,eAAgB,CAAA,MAAA,KAAW,SACvB,GAAA,MAAM,KAAK,OAAQ,CAAA,WAAA,CAAY,eAAgB,CAAA,EAAE,IACjD,MAAM,IAAA,CAAK,OAAQ,CAAA,YAAA,CAAa,gBAAgB,EAAE,CAAA;AAExD,UAAA,IAAI,CAAC,IAAM,EAAA;AACT,YAAA,MAAM,IAAI,KAAA;AAAA,cACR,yFAAyF,YAAY,CAAA,CAAA;AAAA,aACvG;AAAA;AAEF,UAAA,MAAM,KAAK,OAAQ,CAAA,kBAAA,CAAmB,IAAK,CAAA,EAAA,EAAI,OAAO,KAAK,CAAA;AAAA;AAC7D;AAGF,MAAI,IAAA,MAAA,CAAO,OAAQ,CAAA,MAAA,GAAS,CAAG,EAAA;AAC7B,QAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,wBAAyB,CAAA,MAAA,CAAO,OAAO,CAAA;AAAA;AAG5D,MAAM,MAAA,UAAA,CAAW,cAAc,MAAM,CAAA;AACrC,MAAO,MAAA,CAAA,KAAA;AAAA,QACL,CAAA,oBAAA,EAAuB,YAAY,CAAA,uBAAA,EAA0B,KAAK,CAAA,OAAA;AAAA,OACpE;AAAA,KACK,MAAA;AACL,MAAO,MAAA,CAAA,KAAA;AAAA,QACL,CAAA,oBAAA,EAAuB,YAAY,CAAA,2BAAA,EAA8B,KAAK,CAAA,CAAA;AAAA,OACxE;AAAA;AACF;AACF,EAEA,mBAAgC,GAAA;AAC9B,IAAM,MAAA,EAAE,QAAS,EAAA,GAAI,IAAK,CAAA,OAAA;AAC1B,IAAA,MAAM,SAAS,QAAS,CAAA,YAAA,GACpB,SAAS,YAAa,CAAA,mBAAA,KACtB,EAAC;AACL,IAAO,OAAA,MAAA;AAAA;AAEX;;;;"}
+{"version":3,"file":"IncrementalIngestionEngine.cjs.js","sources":["../../src/engine/IncrementalIngestionEngine.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport type { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { IterationEngine, IterationEngineOptions } from '../types';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { performance } from 'perf_hooks';\nimport { Duration } from 'luxon';\nimport { v4 } from 'uuid';\nimport { stringifyError } from '@backstage/errors';\nimport { EventParams } from '@backstage/plugin-events-node';\nimport { HumanDuration } from '@backstage/types';\n\nexport class IncrementalIngestionEngine implements IterationEngine {\n  private readonly restLength: Duration;\n  private readonly backoff: HumanDuration[];\n\n  private manager: IncrementalIngestionDatabaseManager;\n\n  constructor(private options: IterationEngineOptions) {\n    this.manager = options.manager;\n    this.restLength = Duration.fromObject(options.restLength);\n    this.backoff = options.backoff ?? [\n      { minutes: 1 },\n      { minutes: 5 },\n      { minutes: 30 },\n      { hours: 3 },\n    ];\n  }\n\n  async taskFn(signal: AbortSignal) {\n    try {\n      this.options.logger.debug('Begin tick');\n      await this.handleNextAction(signal);\n    } catch (error) {\n      this.options.logger.error(`${error}`);\n      throw error;\n    } finally {\n      this.options.logger.debug('End tick');\n    }\n  }\n\n  async handleNextAction(signal: AbortSignal) {\n    await this.options.ready;\n\n    const result = await this.getCurrentAction();\n    if (result) {\n      const { ingestionId, nextActionAt, nextAction, attempts } = result;\n\n      switch (nextAction) {\n        case 'rest':\n          if (Date.now() > nextActionAt) {\n            await this.manager.clearFinishedIngestions(\n              this.options.provider.getProviderName(),\n            );\n            this.options.logger.debug(\n              `incremental-engine: Ingestion ${ingestionId} rest period complete. Ingestion will start again`,\n            );\n\n            await this.manager.setProviderComplete(ingestionId);\n          } else {\n            this.options.logger.debug(\n              `incremental-engine: Ingestion '${ingestionId}' rest period continuing`,\n            );\n          }\n          break;\n        case 'ingest':\n          try {\n            await this.manager.setProviderBursting(ingestionId);\n            const done = await this.ingestOneBurst(ingestionId, signal);\n            if (done) {\n              this.options.logger.info(\n                `incremental-engine: Ingestion '${ingestionId}' complete, transitioning to rest period of ${this.restLength.toHuman()}`,\n              );\n              await this.manager.setProviderResting(\n                ingestionId,\n                this.restLength,\n              );\n            } else {\n              await this.manager.setProviderInterstitial(ingestionId);\n              this.options.logger.debug(\n                `incremental-engine: Ingestion '${ingestionId}' continuing`,\n              );\n            }\n          } catch (error) {\n            if (\n              (error as Error).message &&\n              (error as Error).message === 'CANCEL'\n            ) {\n              this.options.logger.info(\n                `incremental-engine: Ingestion '${ingestionId}' canceled`,\n              );\n              await this.manager.setProviderCanceling(\n                ingestionId,\n                (error as Error).message,\n              );\n            } else {\n              const currentBackoff = Duration.fromObject(\n                this.backoff[Math.min(this.backoff.length - 1, attempts)],\n              );\n\n              const backoffLength = currentBackoff.as('milliseconds');\n              this.options.logger.error(\n                `incremental-engine: Ingestion '${ingestionId}' failed`,\n                error,\n              );\n\n              const truncatedError = stringifyError(error).substring(0, 700);\n              this.options.logger.error(\n                `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`,\n              );\n\n              await this.manager.setProviderBackoff(\n                ingestionId,\n                attempts,\n                error as Error,\n                backoffLength,\n              );\n            }\n          }\n          break;\n        case 'backoff':\n          if (Date.now() > nextActionAt) {\n            this.options.logger.info(\n              `incremental-engine: Ingestion '${ingestionId}' backoff complete, will attempt to resume`,\n            );\n            await this.manager.setProviderIngesting(ingestionId);\n          } else {\n            this.options.logger.debug(\n              `incremental-engine: Ingestion '${ingestionId}' backoff continuing`,\n            );\n          }\n          break;\n        case 'cancel':\n          this.options.logger.info(\n            `incremental-engine: Ingestion '${ingestionId}' canceling, will restart`,\n          );\n          await this.manager.setProviderCanceled(ingestionId);\n          break;\n        default:\n          this.options.logger.error(\n            `incremental-engine: Ingestion '${ingestionId}' received unknown action '${nextAction}'`,\n          );\n      }\n    } else {\n      this.options.logger.error(\n        `incremental-engine: Engine tried to create duplicate ingestion record for provider '${this.options.provider.getProviderName()}'.`,\n      );\n    }\n  }\n\n  async getCurrentAction() {\n    const providerName = this.options.provider.getProviderName();\n    const record = await this.manager.getCurrentIngestionRecord(providerName);\n    if (record) {\n      this.options.logger.debug(\n        `incremental-engine: Ingestion record found: '${record.id}'`,\n      );\n      return {\n        ingestionId: record.id,\n        nextAction: record.next_action as 'rest' | 'ingest' | 'backoff',\n        attempts: record.attempts as number,\n        nextActionAt: record.next_action_at.valueOf() as number,\n      };\n    }\n    const result = await this.manager.createProviderIngestionRecord(\n      providerName,\n    );\n    if (result) {\n      this.options.logger.info(\n        `incremental-engine: Ingestion record created: '${result.ingestionId}'`,\n      );\n    }\n    return result;\n  }\n\n  async ingestOneBurst(id: string, signal: AbortSignal) {\n    const lastMark = await this.manager.getLastMark(id);\n\n    const cursor = lastMark ? lastMark.cursor : undefined;\n    let sequence = lastMark ? lastMark.sequence + 1 : 0;\n\n    const start = performance.now();\n    let count = 0;\n    let done = false;\n    this.options.logger.info(\n      `incremental-engine: Ingestion '${id}' burst initiated`,\n    );\n\n    await this.options.provider.around(async (context: unknown) => {\n      let next = await this.options.provider.next(context, cursor);\n      count++;\n      for (;;) {\n        done = next.done;\n        await this.mark({\n          id,\n          sequence,\n          entities: next?.entities,\n          done: next.done,\n          cursor: next?.cursor,\n        });\n        if (signal.aborted || next.done) {\n          break;\n        } else {\n          next = await this.options.provider.next(context, next.cursor);\n          count++;\n          sequence++;\n        }\n      }\n    });\n\n    this.options.logger.info(\n      `incremental-engine: Ingestion '${id}' burst complete. (${count} batches in ${Math.round(\n        performance.now() - start,\n      )}ms).`,\n    );\n    return done;\n  }\n\n  async mark(options: {\n    id: string;\n    sequence: number;\n    entities?: DeferredEntity[];\n    done: boolean;\n    cursor?: unknown;\n  }) {\n    const { id, sequence, entities, done, cursor } = options;\n    this.options.logger.debug(\n      `incremental-engine: Ingestion '${id}': MARK ${\n        entities ? entities.length : 0\n      } entities, cursor: ${\n        cursor ? JSON.stringify(cursor) : 'none'\n      }, done: ${done}`,\n    );\n    const markId = v4();\n\n    await this.manager.createMark({\n      record: {\n        id: markId,\n        ingestion_id: id,\n        cursor,\n        sequence,\n      },\n    });\n\n    if (entities && entities.length > 0) {\n      await this.manager.createMarkEntities(markId, entities);\n    }\n\n    const added =\n      entities?.map(deferred => ({\n        ...deferred,\n        entity: {\n          ...deferred.entity,\n          metadata: {\n            ...deferred.entity.metadata,\n            annotations: {\n              ...deferred.entity.metadata.annotations,\n            },\n          },\n        },\n      })) ?? [];\n\n    const removed: { entityRef: string }[] = [];\n\n    if (done) {\n      this.options.logger.info(\n        `incremental-engine: Ingestion '${id}': Final page reached, calculating removed entities`,\n      );\n      const result = await this.manager.computeRemoved(\n        this.options.provider.getProviderName(),\n        id,\n      );\n\n      const { total } = result;\n\n      let doRemoval = true;\n      if (this.options.rejectEmptySourceCollections) {\n        if (total === 0) {\n          this.options.logger.error(\n            `incremental-engine: Ingestion '${id}': Rejecting empty entity collection!`,\n          );\n          doRemoval = false;\n        }\n      }\n\n      if (this.options.rejectRemovalsAbovePercentage) {\n        // If the total entities upserted in this ingestion is 0, then\n        // 100% of entities are stale and marked for removal.\n        const percentRemoved =\n          total > 0 ? (result.removed.length / total) * 100 : 100;\n        if (percentRemoved <= this.options.rejectRemovalsAbovePercentage) {\n          this.options.logger.info(\n            `incremental-engine: Ingestion '${id}': Removing ${result.removed.length} entities that have no matching assets`,\n          );\n        } else {\n          const notice = `Attempted to remove ${percentRemoved}% of matching entities!`;\n          this.options.logger.error(\n            `incremental-engine: Ingestion '${id}': ${notice}`,\n          );\n          await this.manager.updateIngestionRecordById({\n            ingestionId: id,\n            update: {\n              last_error: `REMOVAL_THRESHOLD exceeded on ingestion mark ${markId}: ${notice}`,\n            },\n          });\n          doRemoval = false;\n        }\n      }\n      if (doRemoval) {\n        for (const entityRef of result.removed) {\n          removed.push(entityRef);\n        }\n      }\n    }\n\n    await this.options.connection.applyMutation({\n      type: 'delta',\n      added,\n      removed,\n    });\n  }\n\n  async onEvent(params: EventParams): Promise<void> {\n    const { topic } = params;\n    if (!this.supportsEventTopics().includes(topic)) {\n      return;\n    }\n\n    const { logger, provider, connection } = this.options;\n    const providerName = provider.getProviderName();\n    logger.debug(`incremental-engine: ${providerName} received ${topic} event`);\n\n    if (!provider.eventHandler) {\n      return;\n    }\n\n    const result = await provider.eventHandler.onEvent(params);\n\n    if (result.type === 'delta') {\n      if (result.added.length > 0) {\n        const ingestionRecord = await this.manager.getCurrentIngestionRecord(\n          providerName,\n        );\n\n        if (!ingestionRecord) {\n          logger.debug(\n            `incremental-engine: ${providerName} skipping delta addition because incremental ingestion is restarting.`,\n          );\n        } else {\n          const mark =\n            ingestionRecord.status === 'resting'\n              ? await this.manager.getLastMark(ingestionRecord.id)\n              : await this.manager.getFirstMark(ingestionRecord.id);\n\n          if (!mark) {\n            throw new Error(\n              `Cannot apply delta, page records are missing! Please re-run incremental ingestion for ${providerName}.`,\n            );\n          }\n          await this.manager.createMarkEntities(mark.id, result.added);\n        }\n      }\n\n      if (result.removed.length > 0) {\n        await this.manager.deleteEntityRecordsByRef(result.removed);\n      }\n\n      await connection.applyMutation(result);\n      logger.debug(\n        `incremental-engine: ${providerName} processed delta from '${topic}' event`,\n      );\n    } else {\n      logger.debug(\n        `incremental-engine: ${providerName} ignored event from topic '${topic}'`,\n      );\n    }\n  }\n\n  supportsEventTopics(): string[] {\n    const { provider } = this.options;\n    const topics = provider.eventHandler\n      ? provider.eventHandler.supportsEventTopics()\n      : [];\n    return topics;\n  }\n}\n"],"names":["Duration","stringifyError","performance","v4"],"mappings":";;;;;;;AA0BO,MAAM,0BAAsD,CAAA;AAAA,EAMjE,YAAoB,OAAiC,EAAA;AAAjC,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAClB,IAAA,IAAA,CAAK,UAAU,OAAQ,CAAA,OAAA;AACvB,IAAA,IAAA,CAAK,UAAa,GAAAA,cAAA,CAAS,UAAW,CAAA,OAAA,CAAQ,UAAU,CAAA;AACxD,IAAK,IAAA,CAAA,OAAA,GAAU,QAAQ,OAAW,IAAA;AAAA,MAChC,EAAE,SAAS,CAAE,EAAA;AAAA,MACb,EAAE,SAAS,CAAE,EAAA;AAAA,MACb,EAAE,SAAS,EAAG,EAAA;AAAA,MACd,EAAE,OAAO,CAAE;AAAA,KACb;AAAA;AACF,EAdiB,UAAA;AAAA,EACA,OAAA;AAAA,EAET,OAAA;AAAA,EAaR,MAAM,OAAO,MAAqB,EAAA;AAChC,IAAI,IAAA;AACF,MAAK,IAAA,CAAA,OAAA,CAAQ,MAAO,CAAA,KAAA,CAAM,YAAY,CAAA;AACtC,MAAM,MAAA,IAAA,CAAK,iBAAiB,MAAM,CAAA;AAAA,aAC3B,KAAO,EAAA;AACd,MAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,CAAO,KAAM,CAAA,CAAA,EAAG,KAAK,CAAE,CAAA,CAAA;AACpC,MAAM,MAAA,KAAA;AAAA,KACN,SAAA;AACA,MAAK,IAAA,CAAA,OAAA,CAAQ,MAAO,CAAA,KAAA,CAAM,UAAU,CAAA;AAAA;AACtC;AACF,EAEA,MAAM,iBAAiB,MAAqB,EAAA;AAC1C,IAAA,MAAM,KAAK,OAAQ,CAAA,KAAA;AAEnB,IAAM,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,gBAAiB,EAAA;AAC3C,IAAA,IAAI,MAAQ,EAAA;AACV,MAAA,MAAM,EAAE,WAAA,EAAa,YAAc,EAAA,UAAA,EAAY,UAAa,GAAA,MAAA;AAE5D,MAAA,QAAQ,UAAY;AAAA,QAClB,KAAK,MAAA;AACH,UAAI,IAAA,IAAA,CAAK,GAAI,EAAA,GAAI,YAAc,EAAA;AAC7B,YAAA,MAAM,KAAK,OAAQ,CAAA,uBAAA;AAAA,cACjB,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,eAAgB;AAAA,aACxC;AACA,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,cAClB,iCAAiC,WAAW,CAAA,iDAAA;AAAA,aAC9C;AAEA,YAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,mBAAA,CAAoB,WAAW,CAAA;AAAA,WAC7C,MAAA;AACL,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,wBAAA;AAAA,aAC/C;AAAA;AAEF,UAAA;AAAA,QACF,KAAK,QAAA;AACH,UAAI,IAAA;AACF,YAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,mBAAA,CAAoB,WAAW,CAAA;AAClD,YAAA,MAAM,IAAO,GAAA,MAAM,IAAK,CAAA,cAAA,CAAe,aAAa,MAAM,CAAA;AAC1D,YAAA,IAAI,IAAM,EAAA;AACR,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,4CAAA,EAA+C,IAAK,CAAA,UAAA,CAAW,SAAS,CAAA;AAAA,eACvH;AACA,cAAA,MAAM,KAAK,OAAQ,CAAA,kBAAA;AAAA,gBACjB,WAAA;AAAA,gBACA,IAAK,CAAA;AAAA,eACP;AAAA,aACK,MAAA;AACL,cAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,uBAAA,CAAwB,WAAW,CAAA;AACtD,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,YAAA;AAAA,eAC/C;AAAA;AACF,mBACO,KAAO,EAAA;AACd,YAAA,IACG,KAAgB,CAAA,OAAA,IAChB,KAAgB,CAAA,OAAA,KAAY,QAC7B,EAAA;AACA,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,UAAA;AAAA,eAC/C;AACA,cAAA,MAAM,KAAK,OAAQ,CAAA,oBAAA;AAAA,gBACjB,WAAA;AAAA,gBACC,KAAgB,CAAA;AAAA,eACnB;AAAA,aACK,MAAA;AACL,cAAA,MAAM,iBAAiBA,cAAS,CAAA,UAAA;AAAA,gBAC9B,IAAA,CAAK,QAAQ,IAAK,CAAA,GAAA,CAAI,KAAK,OAAQ,CAAA,MAAA,GAAS,CAAG,EAAA,QAAQ,CAAC;AAAA,eAC1D;AAEA,cAAM,MAAA,aAAA,GAAgB,cAAe,CAAA,EAAA,CAAG,cAAc,CAAA;AACtD,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,QAAA,CAAA;AAAA,gBAC7C;AAAA,eACF;AAEA,cAAA,MAAM,iBAAiBC,qBAAe,CAAA,KAAK,CAAE,CAAA,SAAA,CAAU,GAAG,GAAG,CAAA;AAC7D,cAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,oEAAA,EAAuE,eAAe,OAAQ,EAAC,KAAK,cAAc,CAAA,CAAA;AAAA,eACjK;AAEA,cAAA,MAAM,KAAK,OAAQ,CAAA,kBAAA;AAAA,gBACjB,WAAA;AAAA,gBACA,QAAA;AAAA,gBACA,KAAA;AAAA,gBACA;AAAA,eACF;AAAA;AACF;AAEF,UAAA;AAAA,QACF,KAAK,SAAA;AACH,UAAI,IAAA,IAAA,CAAK,GAAI,EAAA,GAAI,YAAc,EAAA;AAC7B,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,0CAAA;AAAA,aAC/C;AACA,YAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,oBAAA,CAAqB,WAAW,CAAA;AAAA,WAC9C,MAAA;AACL,YAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,oBAAA;AAAA,aAC/C;AAAA;AAEF,UAAA;AAAA,QACF,KAAK,QAAA;AACH,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,YAClB,kCAAkC,WAAW,CAAA,yBAAA;AAAA,WAC/C;AACA,UAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,mBAAA,CAAoB,WAAW,CAAA;AAClD,UAAA;AAAA,QACF;AACE,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,WAAW,CAAA,2BAAA,EAA8B,UAAU,CAAA,CAAA;AAAA,WACvF;AAAA;AACJ,KACK,MAAA;AACL,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,QAClB,CAAuF,oFAAA,EAAA,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,iBAAiB,CAAA,EAAA;AAAA,OAChI;AAAA;AACF;AACF,EAEA,MAAM,gBAAmB,GAAA;AACvB,IAAA,MAAM,YAAe,GAAA,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,eAAgB,EAAA;AAC3D,IAAA,MAAM,MAAS,GAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,0BAA0B,YAAY,CAAA;AACxE,IAAA,IAAI,MAAQ,EAAA;AACV,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,QAClB,CAAA,6CAAA,EAAgD,OAAO,EAAE,CAAA,CAAA;AAAA,OAC3D;AACA,MAAO,OAAA;AAAA,QACL,aAAa,MAAO,CAAA,EAAA;AAAA,QACpB,YAAY,MAAO,CAAA,WAAA;AAAA,QACnB,UAAU,MAAO,CAAA,QAAA;AAAA,QACjB,YAAA,EAAc,MAAO,CAAA,cAAA,CAAe,OAAQ;AAAA,OAC9C;AAAA;AAEF,IAAM,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,OAAQ,CAAA,6BAAA;AAAA,MAChC;AAAA,KACF;AACA,IAAA,IAAI,MAAQ,EAAA;AACV,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,QAClB,CAAA,+CAAA,EAAkD,OAAO,WAAW,CAAA,CAAA;AAAA,OACtE;AAAA;AAEF,IAAO,OAAA,MAAA;AAAA;AACT,EAEA,MAAM,cAAe,CAAA,EAAA,EAAY,MAAqB,EAAA;AACpD,IAAA,MAAM,QAAW,GAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,YAAY,EAAE,CAAA;AAElD,IAAM,MAAA,MAAA,GAAS,QAAW,GAAA,QAAA,CAAS,MAAS,GAAA,KAAA,CAAA;AAC5C,IAAA,IAAI,QAAW,GAAA,QAAA,GAAW,QAAS,CAAA,QAAA,GAAW,CAAI,GAAA,CAAA;AAElD,IAAM,MAAA,KAAA,GAAQC,uBAAY,GAAI,EAAA;AAC9B,IAAA,IAAI,KAAQ,GAAA,CAAA;AACZ,IAAA,IAAI,IAAO,GAAA,KAAA;AACX,IAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,MAClB,kCAAkC,EAAE,CAAA,iBAAA;AAAA,KACtC;AAEA,IAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,QAAS,CAAA,MAAA,CAAO,OAAO,OAAqB,KAAA;AAC7D,MAAA,IAAI,OAAO,MAAM,IAAA,CAAK,QAAQ,QAAS,CAAA,IAAA,CAAK,SAAS,MAAM,CAAA;AAC3D,MAAA,KAAA,EAAA;AACA,MAAS,WAAA;AACP,QAAA,IAAA,GAAO,IAAK,CAAA,IAAA;AACZ,QAAA,MAAM,KAAK,IAAK,CAAA;AAAA,UACd,EAAA;AAAA,UACA,QAAA;AAAA,UACA,UAAU,IAAM,EAAA,QAAA;AAAA,UAChB,MAAM,IAAK,CAAA,IAAA;AAAA,UACX,QAAQ,IAAM,EAAA;AAAA,SACf,CAAA;AACD,QAAI,IAAA,MAAA,CAAO,OAAW,IAAA,IAAA,CAAK,IAAM,EAAA;AAC/B,UAAA;AAAA,SACK,MAAA;AACL,UAAA,IAAA,GAAO,MAAM,IAAK,CAAA,OAAA,CAAQ,SAAS,IAAK,CAAA,OAAA,EAAS,KAAK,MAAM,CAAA;AAC5D,UAAA,KAAA,EAAA;AACA,UAAA,QAAA,EAAA;AAAA;AACF;AACF,KACD,CAAA;AAED,IAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,MAClB,CAAkC,+BAAA,EAAA,EAAE,CAAsB,mBAAA,EAAA,KAAK,eAAe,IAAK,CAAA,KAAA;AAAA,QACjFA,sBAAA,CAAY,KAAQ,GAAA;AAAA,OACrB,CAAA,IAAA;AAAA,KACH;AACA,IAAO,OAAA,IAAA;AAAA;AACT,EAEA,MAAM,KAAK,OAMR,EAAA;AACD,IAAA,MAAM,EAAE,EAAI,EAAA,QAAA,EAAU,QAAU,EAAA,IAAA,EAAM,QAAW,GAAA,OAAA;AACjD,IAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,MAClB,CAAkC,+BAAA,EAAA,EAAE,CAClC,QAAA,EAAA,QAAA,GAAW,SAAS,MAAS,GAAA,CAC/B,CACE,mBAAA,EAAA,MAAA,GAAS,KAAK,SAAU,CAAA,MAAM,CAAI,GAAA,MACpC,WAAW,IAAI,CAAA;AAAA,KACjB;AACA,IAAA,MAAM,SAASC,OAAG,EAAA;AAElB,IAAM,MAAA,IAAA,CAAK,QAAQ,UAAW,CAAA;AAAA,MAC5B,MAAQ,EAAA;AAAA,QACN,EAAI,EAAA,MAAA;AAAA,QACJ,YAAc,EAAA,EAAA;AAAA,QACd,MAAA;AAAA,QACA;AAAA;AACF,KACD,CAAA;AAED,IAAI,IAAA,QAAA,IAAY,QAAS,CAAA,MAAA,GAAS,CAAG,EAAA;AACnC,MAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,kBAAmB,CAAA,MAAA,EAAQ,QAAQ,CAAA;AAAA;AAGxD,IAAM,MAAA,KAAA,GACJ,QAAU,EAAA,GAAA,CAAI,CAAa,QAAA,MAAA;AAAA,MACzB,GAAG,QAAA;AAAA,MACH,MAAQ,EAAA;AAAA,QACN,GAAG,QAAS,CAAA,MAAA;AAAA,QACZ,QAAU,EAAA;AAAA,UACR,GAAG,SAAS,MAAO,CAAA,QAAA;AAAA,UACnB,WAAa,EAAA;AAAA,YACX,GAAG,QAAS,CAAA,MAAA,CAAO,QAAS,CAAA;AAAA;AAC9B;AACF;AACF,KACF,CAAE,KAAK,EAAC;AAEV,IAAA,MAAM,UAAmC,EAAC;AAE1C,IAAA,IAAI,IAAM,EAAA;AACR,MAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,QAClB,kCAAkC,EAAE,CAAA,mDAAA;AAAA,OACtC;AACA,MAAM,MAAA,MAAA,GAAS,MAAM,IAAA,CAAK,OAAQ,CAAA,cAAA;AAAA,QAChC,IAAA,CAAK,OAAQ,CAAA,QAAA,CAAS,eAAgB,EAAA;AAAA,QACtC;AAAA,OACF;AAEA,MAAM,MAAA,EAAE,OAAU,GAAA,MAAA;AAElB,MAAA,IAAI,SAAY,GAAA,IAAA;AAChB,MAAI,IAAA,IAAA,CAAK,QAAQ,4BAA8B,EAAA;AAC7C,QAAA,IAAI,UAAU,CAAG,EAAA;AACf,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,YAClB,kCAAkC,EAAE,CAAA,qCAAA;AAAA,WACtC;AACA,UAAY,SAAA,GAAA,KAAA;AAAA;AACd;AAGF,MAAI,IAAA,IAAA,CAAK,QAAQ,6BAA+B,EAAA;AAG9C,QAAA,MAAM,iBACJ,KAAQ,GAAA,CAAA,GAAK,OAAO,OAAQ,CAAA,MAAA,GAAS,QAAS,GAAM,GAAA,GAAA;AACtD,QAAI,IAAA,cAAA,IAAkB,IAAK,CAAA,OAAA,CAAQ,6BAA+B,EAAA;AAChE,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,IAAA;AAAA,YAClB,CAAkC,+BAAA,EAAA,EAAE,CAAe,YAAA,EAAA,MAAA,CAAO,QAAQ,MAAM,CAAA,sCAAA;AAAA,WAC1E;AAAA,SACK,MAAA;AACL,UAAM,MAAA,MAAA,GAAS,uBAAuB,cAAc,CAAA,uBAAA,CAAA;AACpD,UAAA,IAAA,CAAK,QAAQ,MAAO,CAAA,KAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,GAAA,EAAM,MAAM,CAAA;AAAA,WAClD;AACA,UAAM,MAAA,IAAA,CAAK,QAAQ,yBAA0B,CAAA;AAAA,YAC3C,WAAa,EAAA,EAAA;AAAA,YACb,MAAQ,EAAA;AAAA,cACN,UAAY,EAAA,CAAA,6CAAA,EAAgD,MAAM,CAAA,EAAA,EAAK,MAAM,CAAA;AAAA;AAC/E,WACD,CAAA;AACD,UAAY,SAAA,GAAA,KAAA;AAAA;AACd;AAEF,MAAA,IAAI,SAAW,EAAA;AACb,QAAW,KAAA,MAAA,SAAA,IAAa,OAAO,OAAS,EAAA;AACtC,UAAA,OAAA,CAAQ,KAAK,SAAS,CAAA;AAAA;AACxB;AACF;AAGF,IAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,UAAA,CAAW,aAAc,CAAA;AAAA,MAC1C,IAAM,EAAA,OAAA;AAAA,MACN,KAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA;AACH,EAEA,MAAM,QAAQ,MAAoC,EAAA;AAChD,IAAM,MAAA,EAAE,OAAU,GAAA,MAAA;AAClB,IAAA,IAAI,CAAC,IAAK,CAAA,mBAAA,EAAsB,CAAA,QAAA,CAAS,KAAK,CAAG,EAAA;AAC/C,MAAA;AAAA;AAGF,IAAA,MAAM,EAAE,MAAA,EAAQ,QAAU,EAAA,UAAA,KAAe,IAAK,CAAA,OAAA;AAC9C,IAAM,MAAA,YAAA,GAAe,SAAS,eAAgB,EAAA;AAC9C,IAAA,MAAA,CAAO,KAAM,CAAA,CAAA,oBAAA,EAAuB,YAAY,CAAA,UAAA,EAAa,KAAK,CAAQ,MAAA,CAAA,CAAA;AAE1E,IAAI,IAAA,CAAC,SAAS,YAAc,EAAA;AAC1B,MAAA;AAAA;AAGF,IAAA,MAAM,MAAS,GAAA,MAAM,QAAS,CAAA,YAAA,CAAa,QAAQ,MAAM,CAAA;AAEzD,IAAI,IAAA,MAAA,CAAO,SAAS,OAAS,EAAA;AAC3B,MAAI,IAAA,MAAA,CAAO,KAAM,CAAA,MAAA,GAAS,CAAG,EAAA;AAC3B,QAAM,MAAA,eAAA,GAAkB,MAAM,IAAA,CAAK,OAAQ,CAAA,yBAAA;AAAA,UACzC;AAAA,SACF;AAEA,QAAA,IAAI,CAAC,eAAiB,EAAA;AACpB,UAAO,MAAA,CAAA,KAAA;AAAA,YACL,uBAAuB,YAAY,CAAA,qEAAA;AAAA,WACrC;AAAA,SACK,MAAA;AACL,UAAA,MAAM,OACJ,eAAgB,CAAA,MAAA,KAAW,SACvB,GAAA,MAAM,KAAK,OAAQ,CAAA,WAAA,CAAY,eAAgB,CAAA,EAAE,IACjD,MAAM,IAAA,CAAK,OAAQ,CAAA,YAAA,CAAa,gBAAgB,EAAE,CAAA;AAExD,UAAA,IAAI,CAAC,IAAM,EAAA;AACT,YAAA,MAAM,IAAI,KAAA;AAAA,cACR,yFAAyF,YAAY,CAAA,CAAA;AAAA,aACvG;AAAA;AAEF,UAAA,MAAM,KAAK,OAAQ,CAAA,kBAAA,CAAmB,IAAK,CAAA,EAAA,EAAI,OAAO,KAAK,CAAA;AAAA;AAC7D;AAGF,MAAI,IAAA,MAAA,CAAO,OAAQ,CAAA,MAAA,GAAS,CAAG,EAAA;AAC7B,QAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,wBAAyB,CAAA,MAAA,CAAO,OAAO,CAAA;AAAA;AAG5D,MAAM,MAAA,UAAA,CAAW,cAAc,MAAM,CAAA;AACrC,MAAO,MAAA,CAAA,KAAA;AAAA,QACL,CAAA,oBAAA,EAAuB,YAAY,CAAA,uBAAA,EAA0B,KAAK,CAAA,OAAA;AAAA,OACpE;AAAA,KACK,MAAA;AACL,MAAO,MAAA,CAAA,KAAA;AAAA,QACL,CAAA,oBAAA,EAAuB,YAAY,CAAA,2BAAA,EAA8B,KAAK,CAAA,CAAA;AAAA,OACxE;AAAA;AACF;AACF,EAEA,mBAAgC,GAAA;AAC9B,IAAM,MAAA,EAAE,QAAS,EAAA,GAAI,IAAK,CAAA,OAAA;AAC1B,IAAA,MAAM,SAAS,QAAS,CAAA,YAAA,GACpB,SAAS,YAAa,CAAA,mBAAA,KACtB,EAAC;AACL,IAAO,OAAA,MAAA;AAAA;AAEX;;;;"}

package/dist/index.d.ts

@@ -209,7 +209,10 @@ type PluginEnvironment = {
     permissions: PermissionEvaluator;
 };
 
-/** @public */
+/**
+ * @public
+ * @deprecated Please migrate to the new backend system and import `@backstage/plugin-catalog-backend-module-incremental-ingestion` as a module, and add providers to the `incrementalIngestionProvidersExtensionPoint` instead
+ */
 declare class IncrementalCatalogBuilder {
     private env;
     private builder;

package/dist/module/WrapperProviders.cjs.js

@@ -60,14 +60,31 @@ class WrapperProviders {
         restLength,
         connection
       });
-      const frequency = luxon.Duration.isDuration(burstInterval) ? burstInterval : luxon.Duration.fromObject(burstInterval);
-      const length = luxon.Duration.isDuration(burstLength) ? burstLength : luxon.Duration.fromObject(burstLength);
+      let frequency = luxon.Duration.isDuration(burstInterval) ? burstInterval : luxon.Duration.fromObject(burstInterval);
+      if (frequency.as("milliseconds") < 5e3) {
+        frequency = luxon.Duration.fromObject({ seconds: 5 });
+      }
+      let length = luxon.Duration.isDuration(burstLength) ? burstLength : luxon.Duration.fromObject(burstLength);
+      length = length.plus(luxon.Duration.fromObject({ minutes: 1 }));
       await this.options.scheduler.scheduleTask({
         id: provider.getProviderName(),
         fn: engine.taskFn.bind(engine),
         frequency,
         timeout: length
       });
+      const topics = engine.supportsEventTopics();
+      if (topics.length > 0) {
+        logger.info(
+          `Provider ${provider.getProviderName()} subscribing to events for topics: ${topics.join(
+            ","
+          )}`
+        );
+        await this.options.events.subscribe({
+          topics,
+          id: `catalog-backend-module-incremental-ingestion:${provider.getProviderName()}`,
+          onEvent: (evt) => engine.onEvent(evt)
+        });
+      }
     } catch (error) {
       logger.warn(
         `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${errors.stringifyError(

package/dist/module/WrapperProviders.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"WrapperProviders.cjs.js","sources":["../../src/module/WrapperProviders.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  LoggerService,\n  RootConfigService,\n  SchedulerService,\n} from '@backstage/backend-plugin-api';\nimport { stringifyError } from '@backstage/errors';\nimport {\n  EntityProvider,\n  EntityProviderConnection,\n} from '@backstage/plugin-catalog-node';\nimport { createDeferred } from '@backstage/types';\nimport express from 'express';\nimport { Knex } from 'knex';\nimport { Duration } from 'luxon';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { applyDatabaseMigrations } from '../database/migrations';\nimport { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine';\nimport { IncrementalProviderRouter } from '../router/routes';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '../types';\n\n/**\n * Helps in the creation of the catalog entity providers that wrap the\n * incremental ones.\n */\nexport class WrapperProviders {\n  private migrate: Promise<void> | undefined;\n  private numberOfProvidersToConnect = 0;\n  private readonly readySignal = createDeferred();\n\n  constructor(\n    private readonly options: {\n      config: RootConfigService;\n      logger: LoggerService;\n      client: Knex;\n      scheduler: SchedulerService;\n      applyDatabaseMigrations?: typeof applyDatabaseMigrations;\n    },\n  ) {}\n\n  wrap(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    options: IncrementalEntityProviderOptions,\n  ): EntityProvider {\n    this.numberOfProvidersToConnect += 1;\n    return {\n      getProviderName: () => provider.getProviderName(),\n      connect: async connection => {\n        await this.startProvider(provider, options, connection);\n        this.numberOfProvidersToConnect -= 1;\n        if (this.numberOfProvidersToConnect === 0) {\n          this.readySignal.resolve();\n        }\n      },\n    };\n  }\n\n  async adminRouter(): Promise<express.Router> {\n    return await new IncrementalProviderRouter(\n      new IncrementalIngestionDatabaseManager({ client: this.options.client }),\n      this.options.logger,\n    ).createRouter();\n  }\n\n  private async startProvider(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    providerOptions: IncrementalEntityProviderOptions,\n    connection: EntityProviderConnection,\n  ) {\n    const logger = this.options.logger.child({\n      entityProvider: provider.getProviderName(),\n    });\n\n    try {\n      if (!this.migrate) {\n        this.migrate = Promise.resolve().then(async () => {\n          const apply =\n            this.options.applyDatabaseMigrations ?? applyDatabaseMigrations;\n          await apply(this.options.client);\n        });\n      }\n\n      await this.migrate;\n\n      const { burstInterval, burstLength, restLength } = providerOptions;\n\n      logger.info(`Connecting`);\n\n      const manager = new IncrementalIngestionDatabaseManager({\n        client: this.options.client,\n      });\n      const engine = new IncrementalIngestionEngine({\n        ...providerOptions,\n        ready: this.readySignal,\n        manager,\n        logger,\n        provider,\n        restLength,\n        connection,\n      });\n\n      const frequency = Duration.isDuration(burstInterval)\n        ? burstInterval\n        : Duration.fromObject(burstInterval);\n      const length = Duration.isDuration(burstLength)\n        ? burstLength\n        : Duration.fromObject(burstLength);\n\n      await this.options.scheduler.scheduleTask({\n        id: provider.getProviderName(),\n        fn: engine.taskFn.bind(engine),\n        frequency,\n        timeout: length,\n      });\n    } catch (error) {\n      logger.warn(\n        `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${stringifyError(\n          error,\n        )}`,\n      );\n      throw error;\n    }\n  }\n}\n"],"names":["createDeferred","IncrementalProviderRouter","IncrementalIngestionDatabaseManager","applyDatabaseMigrations","IncrementalIngestionEngine","Duration","stringifyError"],"mappings":";;;;;;;;;;AA2CO,MAAM,gBAAiB,CAAA;AAAA,EAK5B,YACmB,OAOjB,EAAA;AAPiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA;AAOhB,EAZK,OAAA;AAAA,EACA,0BAA6B,GAAA,CAAA;AAAA,EACpB,cAAcA,oBAAe,EAAA;AAAA,EAY9C,IAAA,CACE,UACA,OACgB,EAAA;AAChB,IAAA,IAAA,CAAK,0BAA8B,IAAA,CAAA;AACnC,IAAO,OAAA;AAAA,MACL,eAAA,EAAiB,MAAM,QAAA,CAAS,eAAgB,EAAA;AAAA,MAChD,OAAA,EAAS,OAAM,UAAc,KAAA;AAC3B,QAAA,MAAM,IAAK,CAAA,aAAA,CAAc,QAAU,EAAA,OAAA,EAAS,UAAU,CAAA;AACtD,QAAA,IAAA,CAAK,0BAA8B,IAAA,CAAA;AACnC,QAAI,IAAA,IAAA,CAAK,+BAA+B,CAAG,EAAA;AACzC,UAAA,IAAA,CAAK,YAAY,OAAQ,EAAA;AAAA;AAC3B;AACF,KACF;AAAA;AACF,EAEA,MAAM,WAAuC,GAAA;AAC3C,IAAA,OAAO,MAAM,IAAIC,gCAAA;AAAA,MACf,IAAIC,uEAAoC,CAAA,EAAE,QAAQ,IAAK,CAAA,OAAA,CAAQ,QAAQ,CAAA;AAAA,MACvE,KAAK,OAAQ,CAAA;AAAA,MACb,YAAa,EAAA;AAAA;AACjB,EAEA,MAAc,aAAA,CACZ,QACA,EAAA,eAAA,EACA,UACA,EAAA;AACA,IAAA,MAAM,MAAS,GAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,CAAO,KAAM,CAAA;AAAA,MACvC,cAAA,EAAgB,SAAS,eAAgB;AAAA,KAC1C,CAAA;AAED,IAAI,IAAA;AACF,MAAI,IAAA,CAAC,KAAK,OAAS,EAAA;AACjB,QAAA,IAAA,CAAK,OAAU,GAAA,OAAA,CAAQ,OAAQ,EAAA,CAAE,KAAK,YAAY;AAChD,UAAM,MAAA,KAAA,GACJ,IAAK,CAAA,OAAA,CAAQ,uBAA2B,IAAAC,kCAAA;AAC1C,UAAM,MAAA,KAAA,CAAM,IAAK,CAAA,OAAA,CAAQ,MAAM,CAAA;AAAA,SAChC,CAAA;AAAA;AAGH,MAAA,MAAM,IAAK,CAAA,OAAA;AAEX,MAAA,MAAM,EAAE,aAAA,EAAe,WAAa,EAAA,UAAA,EAAe,GAAA,eAAA;AAEnD,MAAA,MAAA,CAAO,KAAK,CAAY,UAAA,CAAA,CAAA;AAExB,MAAM,MAAA,OAAA,GAAU,IAAID,uEAAoC,CAAA;AAAA,QACtD,MAAA,EAAQ,KAAK,OAAQ,CAAA;AAAA,OACtB,CAAA;AACD,MAAM,MAAA,MAAA,GAAS,IAAIE,qDAA2B,CAAA;AAAA,QAC5C,GAAG,eAAA;AAAA,QACH,OAAO,IAAK,CAAA,WAAA;AAAA,QACZ,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA;AAAA,OACD,CAAA;AAED,MAAM,MAAA,SAAA,GAAYC,eAAS,UAAW,CAAA,aAAa,IAC/C,aACA,GAAAA,cAAA,CAAS,WAAW,aAAa,CAAA;AACrC,MAAM,MAAA,MAAA,GAASA,eAAS,UAAW,CAAA,WAAW,IAC1C,WACA,GAAAA,cAAA,CAAS,WAAW,WAAW,CAAA;AAEnC,MAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,SAAA,CAAU,YAAa,CAAA;AAAA,QACxC,EAAA,EAAI,SAAS,eAAgB,EAAA;AAAA,QAC7B,EAAI,EAAA,MAAA,CAAO,MAAO,CAAA,IAAA,CAAK,MAAM,CAAA;AAAA,QAC7B,SAAA;AAAA,QACA,OAAS,EAAA;AAAA,OACV,CAAA;AAAA,aACM,KAAO,EAAA;AACd,MAAO,MAAA,CAAA,IAAA;AAAA,QACL,CAAuD,oDAAA,EAAA,QAAA,CAAS,eAAgB,EAAC,CAAK,EAAA,EAAAC,qBAAA;AAAA,UACpF;AAAA,SACD,CAAA;AAAA,OACH;AACA,MAAM,MAAA,KAAA;AAAA;AACR;AAEJ;;;;"}
+{"version":3,"file":"WrapperProviders.cjs.js","sources":["../../src/module/WrapperProviders.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  LoggerService,\n  RootConfigService,\n  SchedulerService,\n} from '@backstage/backend-plugin-api';\nimport { stringifyError } from '@backstage/errors';\nimport {\n  EntityProvider,\n  EntityProviderConnection,\n} from '@backstage/plugin-catalog-node';\nimport { createDeferred } from '@backstage/types';\nimport express from 'express';\nimport { Knex } from 'knex';\nimport { Duration } from 'luxon';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { applyDatabaseMigrations } from '../database/migrations';\nimport { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine';\nimport { IncrementalProviderRouter } from '../router/routes';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '../types';\nimport { EventsService } from '@backstage/plugin-events-node';\n\n/**\n * Helps in the creation of the catalog entity providers that wrap the\n * incremental ones.\n */\nexport class WrapperProviders {\n  private migrate: Promise<void> | undefined;\n  private numberOfProvidersToConnect = 0;\n  private readonly readySignal = createDeferred();\n\n  constructor(\n    private readonly options: {\n      config: RootConfigService;\n      logger: LoggerService;\n      client: Knex;\n      scheduler: SchedulerService;\n      applyDatabaseMigrations?: typeof applyDatabaseMigrations;\n      events: EventsService;\n    },\n  ) {}\n\n  wrap(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    options: IncrementalEntityProviderOptions,\n  ): EntityProvider {\n    this.numberOfProvidersToConnect += 1;\n    return {\n      getProviderName: () => provider.getProviderName(),\n      connect: async connection => {\n        await this.startProvider(provider, options, connection);\n        this.numberOfProvidersToConnect -= 1;\n        if (this.numberOfProvidersToConnect === 0) {\n          this.readySignal.resolve();\n        }\n      },\n    };\n  }\n\n  async adminRouter(): Promise<express.Router> {\n    return await new IncrementalProviderRouter(\n      new IncrementalIngestionDatabaseManager({ client: this.options.client }),\n      this.options.logger,\n    ).createRouter();\n  }\n\n  private async startProvider(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    providerOptions: IncrementalEntityProviderOptions,\n    connection: EntityProviderConnection,\n  ) {\n    const logger = this.options.logger.child({\n      entityProvider: provider.getProviderName(),\n    });\n\n    try {\n      if (!this.migrate) {\n        this.migrate = Promise.resolve().then(async () => {\n          const apply =\n            this.options.applyDatabaseMigrations ?? applyDatabaseMigrations;\n          await apply(this.options.client);\n        });\n      }\n\n      await this.migrate;\n\n      const { burstInterval, burstLength, restLength } = providerOptions;\n\n      logger.info(`Connecting`);\n\n      const manager = new IncrementalIngestionDatabaseManager({\n        client: this.options.client,\n      });\n      const engine = new IncrementalIngestionEngine({\n        ...providerOptions,\n        ready: this.readySignal,\n        manager,\n        logger,\n        provider,\n        restLength,\n        connection,\n      });\n\n      let frequency = Duration.isDuration(burstInterval)\n        ? burstInterval\n        : Duration.fromObject(burstInterval);\n      if (frequency.as('milliseconds') < 5000) {\n        frequency = Duration.fromObject({ seconds: 5 }); // don't let it be silly low, to not overload the scheduler\n      }\n\n      let length = Duration.isDuration(burstLength)\n        ? burstLength\n        : Duration.fromObject(burstLength);\n      length = length.plus(Duration.fromObject({ minutes: 1 })); // some margin from the actual completion\n\n      await this.options.scheduler.scheduleTask({\n        id: provider.getProviderName(),\n        fn: engine.taskFn.bind(engine),\n        frequency,\n        timeout: length,\n      });\n\n      const topics = engine.supportsEventTopics();\n      if (topics.length > 0) {\n        logger.info(\n          `Provider ${provider.getProviderName()} subscribing to events for topics: ${topics.join(\n            ',',\n          )}`,\n        );\n        await this.options.events.subscribe({\n          topics,\n          id: `catalog-backend-module-incremental-ingestion:${provider.getProviderName()}`,\n          onEvent: evt => engine.onEvent(evt),\n        });\n      }\n    } catch (error) {\n      logger.warn(\n        `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${stringifyError(\n          error,\n        )}`,\n      );\n      throw error;\n    }\n  }\n}\n"],"names":["createDeferred","IncrementalProviderRouter","IncrementalIngestionDatabaseManager","applyDatabaseMigrations","IncrementalIngestionEngine","Duration","stringifyError"],"mappings":";;;;;;;;;;AA4CO,MAAM,gBAAiB,CAAA;AAAA,EAK5B,YACmB,OAQjB,EAAA;AARiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA;AAQhB,EAbK,OAAA;AAAA,EACA,0BAA6B,GAAA,CAAA;AAAA,EACpB,cAAcA,oBAAe,EAAA;AAAA,EAa9C,IAAA,CACE,UACA,OACgB,EAAA;AAChB,IAAA,IAAA,CAAK,0BAA8B,IAAA,CAAA;AACnC,IAAO,OAAA;AAAA,MACL,eAAA,EAAiB,MAAM,QAAA,CAAS,eAAgB,EAAA;AAAA,MAChD,OAAA,EAAS,OAAM,UAAc,KAAA;AAC3B,QAAA,MAAM,IAAK,CAAA,aAAA,CAAc,QAAU,EAAA,OAAA,EAAS,UAAU,CAAA;AACtD,QAAA,IAAA,CAAK,0BAA8B,IAAA,CAAA;AACnC,QAAI,IAAA,IAAA,CAAK,+BAA+B,CAAG,EAAA;AACzC,UAAA,IAAA,CAAK,YAAY,OAAQ,EAAA;AAAA;AAC3B;AACF,KACF;AAAA;AACF,EAEA,MAAM,WAAuC,GAAA;AAC3C,IAAA,OAAO,MAAM,IAAIC,gCAAA;AAAA,MACf,IAAIC,uEAAoC,CAAA,EAAE,QAAQ,IAAK,CAAA,OAAA,CAAQ,QAAQ,CAAA;AAAA,MACvE,KAAK,OAAQ,CAAA;AAAA,MACb,YAAa,EAAA;AAAA;AACjB,EAEA,MAAc,aAAA,CACZ,QACA,EAAA,eAAA,EACA,UACA,EAAA;AACA,IAAA,MAAM,MAAS,GAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,CAAO,KAAM,CAAA;AAAA,MACvC,cAAA,EAAgB,SAAS,eAAgB;AAAA,KAC1C,CAAA;AAED,IAAI,IAAA;AACF,MAAI,IAAA,CAAC,KAAK,OAAS,EAAA;AACjB,QAAA,IAAA,CAAK,OAAU,GAAA,OAAA,CAAQ,OAAQ,EAAA,CAAE,KAAK,YAAY;AAChD,UAAM,MAAA,KAAA,GACJ,IAAK,CAAA,OAAA,CAAQ,uBAA2B,IAAAC,kCAAA;AAC1C,UAAM,MAAA,KAAA,CAAM,IAAK,CAAA,OAAA,CAAQ,MAAM,CAAA;AAAA,SAChC,CAAA;AAAA;AAGH,MAAA,MAAM,IAAK,CAAA,OAAA;AAEX,MAAA,MAAM,EAAE,aAAA,EAAe,WAAa,EAAA,UAAA,EAAe,GAAA,eAAA;AAEnD,MAAA,MAAA,CAAO,KAAK,CAAY,UAAA,CAAA,CAAA;AAExB,MAAM,MAAA,OAAA,GAAU,IAAID,uEAAoC,CAAA;AAAA,QACtD,MAAA,EAAQ,KAAK,OAAQ,CAAA;AAAA,OACtB,CAAA;AACD,MAAM,MAAA,MAAA,GAAS,IAAIE,qDAA2B,CAAA;AAAA,QAC5C,GAAG,eAAA;AAAA,QACH,OAAO,IAAK,CAAA,WAAA;AAAA,QACZ,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA;AAAA,OACD,CAAA;AAED,MAAI,IAAA,SAAA,GAAYC,eAAS,UAAW,CAAA,aAAa,IAC7C,aACA,GAAAA,cAAA,CAAS,WAAW,aAAa,CAAA;AACrC,MAAA,IAAI,SAAU,CAAA,EAAA,CAAG,cAAc,CAAA,GAAI,GAAM,EAAA;AACvC,QAAA,SAAA,GAAYA,cAAS,CAAA,UAAA,CAAW,EAAE,OAAA,EAAS,GAAG,CAAA;AAAA;AAGhD,MAAI,IAAA,MAAA,GAASA,eAAS,UAAW,CAAA,WAAW,IACxC,WACA,GAAAA,cAAA,CAAS,WAAW,WAAW,CAAA;AACnC,MAAS,MAAA,GAAA,MAAA,CAAO,KAAKA,cAAS,CAAA,UAAA,CAAW,EAAE,OAAS,EAAA,CAAA,EAAG,CAAC,CAAA;AAExD,MAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,SAAA,CAAU,YAAa,CAAA;AAAA,QACxC,EAAA,EAAI,SAAS,eAAgB,EAAA;AAAA,QAC7B,EAAI,EAAA,MAAA,CAAO,MAAO,CAAA,IAAA,CAAK,MAAM,CAAA;AAAA,QAC7B,SAAA;AAAA,QACA,OAAS,EAAA;AAAA,OACV,CAAA;AAED,MAAM,MAAA,MAAA,GAAS,OAAO,mBAAoB,EAAA;AAC1C,MAAI,IAAA,MAAA,CAAO,SAAS,CAAG,EAAA;AACrB,QAAO,MAAA,CAAA,IAAA;AAAA,UACL,CAAY,SAAA,EAAA,QAAA,CAAS,eAAgB,EAAC,sCAAsC,MAAO,CAAA,IAAA;AAAA,YACjF;AAAA,WACD,CAAA;AAAA,SACH;AACA,QAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,CAAO,SAAU,CAAA;AAAA,UAClC,MAAA;AAAA,UACA,EAAI,EAAA,CAAA,6CAAA,EAAgD,QAAS,CAAA,eAAA,EAAiB,CAAA,CAAA;AAAA,UAC9E,OAAS,EAAA,CAAA,GAAA,KAAO,MAAO,CAAA,OAAA,CAAQ,GAAG;AAAA,SACnC,CAAA;AAAA;AACH,aACO,KAAO,EAAA;AACd,MAAO,MAAA,CAAA,IAAA;AAAA,QACL,CAAuD,oDAAA,EAAA,QAAA,CAAS,eAAgB,EAAC,CAAK,EAAA,EAAAC,qBAAA;AAAA,UACpF;AAAA,SACD,CAAA;AAAA,OACH;AACA,MAAM,MAAA,KAAA;AAAA;AACR;AAEJ;;;;"}

package/dist/module/catalogModuleIncrementalIngestionEntityProvider.cjs.js

@@ -3,6 +3,7 @@
 var backendPluginApi = require('@backstage/backend-plugin-api');
 var alpha = require('@backstage/plugin-catalog-node/alpha');
 var WrapperProviders = require('./WrapperProviders.cjs.js');
+var pluginEventsNode = require('@backstage/plugin-events-node');
 
 const incrementalIngestionProvidersExtensionPoint = backendPluginApi.createExtensionPoint({
   id: "catalog.incrementalIngestionProvider.providers"
@@ -24,7 +25,8 @@ const catalogModuleIncrementalIngestionEntityProvider = backendPluginApi.createB
         database: backendPluginApi.coreServices.database,
         httpRouter: backendPluginApi.coreServices.httpRouter,
         logger: backendPluginApi.coreServices.logger,
-        scheduler: backendPluginApi.coreServices.scheduler
+        scheduler: backendPluginApi.coreServices.scheduler,
+        events: pluginEventsNode.eventsServiceRef
       },
       async init({
         catalog,
@@ -32,14 +34,16 @@ const catalogModuleIncrementalIngestionEntityProvider = backendPluginApi.createB
         database,
         httpRouter,
         logger,
-        scheduler
+        scheduler,
+        events
       }) {
         const client = await database.getClient();
         const providers = new WrapperProviders.WrapperProviders({
           config,
           logger,
           client,
-          scheduler
+          scheduler,
+          events
         });
         for (const entry of addedProviders) {
           const wrapped = providers.wrap(entry.provider, entry.options);

package/dist/module/catalogModuleIncrementalIngestionEntityProvider.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"catalogModuleIncrementalIngestionEntityProvider.cjs.js","sources":["../../src/module/catalogModuleIncrementalIngestionEntityProvider.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  coreServices,\n  createBackendModule,\n  createExtensionPoint,\n} from '@backstage/backend-plugin-api';\nimport { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '@backstage/plugin-catalog-backend-module-incremental-ingestion';\nimport { WrapperProviders } from './WrapperProviders';\n\n/**\n * @public\n * Interface for {@link incrementalIngestionProvidersExtensionPoint}.\n */\nexport interface IncrementalIngestionProviderExtensionPoint {\n  /** Adds a new incremental entity provider */\n  addProvider<TCursor, TContext>(config: {\n    options: IncrementalEntityProviderOptions;\n    provider: IncrementalEntityProvider<TCursor, TContext>;\n  }): void;\n}\n\n/**\n * @public\n *\n * Extension point for registering incremental ingestion providers.\n * The `catalogModuleIncrementalIngestionEntityProvider` must be installed for these providers to work.\n *\n * @example\n *\n * ```ts\n * backend.add(createBackendModule({\n *   pluginId: 'catalog',\n *   moduleId: 'my-incremental-provider',\n *   register(env) {\n *     env.registerInit({\n *       deps: {\n *         extension: incrementalIngestionProvidersExtensionPoint,\n *       },\n *       async init({ extension }) {\n *         extension.addProvider({\n *           burstInterval: ...,\n *           burstLength: ...,\n *           restLength: ...,\n *         }, {\n *           next(context, cursor) {\n *             ...\n *           },\n *           ...\n *         })\n *       })\n *     })\n *   }\n * }))\n * ```\n */\nexport const incrementalIngestionProvidersExtensionPoint =\n  createExtensionPoint<IncrementalIngestionProviderExtensionPoint>({\n    id: 'catalog.incrementalIngestionProvider.providers',\n  });\n\n/**\n * Registers the incremental entity provider with the catalog processing extension point.\n *\n * @public\n */\nexport const catalogModuleIncrementalIngestionEntityProvider =\n  createBackendModule({\n    pluginId: 'catalog',\n    moduleId: 'incremental-ingestion-entity-provider',\n    register(env) {\n      const addedProviders = new Array<{\n        provider: IncrementalEntityProvider<unknown, unknown>;\n        options: IncrementalEntityProviderOptions;\n      }>();\n\n      env.registerExtensionPoint(incrementalIngestionProvidersExtensionPoint, {\n        addProvider({ options, provider }) {\n          addedProviders.push({ options, provider });\n        },\n      });\n\n      env.registerInit({\n        deps: {\n          catalog: catalogProcessingExtensionPoint,\n          config: coreServices.rootConfig,\n          database: coreServices.database,\n          httpRouter: coreServices.httpRouter,\n          logger: coreServices.logger,\n          scheduler: coreServices.scheduler,\n        },\n        async init({\n          catalog,\n          config,\n          database,\n          httpRouter,\n          logger,\n          scheduler,\n        }) {\n          const client = await database.getClient();\n\n          const providers = new WrapperProviders({\n            config,\n            logger,\n            client,\n            scheduler,\n          });\n\n          for (const entry of addedProviders) {\n            const wrapped = providers.wrap(entry.provider, entry.options);\n            catalog.addEntityProvider(wrapped);\n          }\n\n          httpRouter.use(await providers.adminRouter());\n        },\n      });\n    },\n  });\n"],"names":["createExtensionPoint","createBackendModule","catalogProcessingExtensionPoint","coreServices","WrapperProviders"],"mappings":";;;;;;AA0EO,MAAM,8CACXA,qCAAiE,CAAA;AAAA,EAC/D,EAAI,EAAA;AACN,CAAC;AAOI,MAAM,kDACXC,oCAAoB,CAAA;AAAA,EAClB,QAAU,EAAA,SAAA;AAAA,EACV,QAAU,EAAA,uCAAA;AAAA,EACV,SAAS,GAAK,EAAA;AACZ,IAAM,MAAA,cAAA,GAAiB,IAAI,KAGxB,EAAA;AAEH,IAAA,GAAA,CAAI,uBAAuB,2CAA6C,EAAA;AAAA,MACtE,WAAY,CAAA,EAAE,OAAS,EAAA,QAAA,EAAY,EAAA;AACjC,QAAA,cAAA,CAAe,IAAK,CAAA,EAAE,OAAS,EAAA,QAAA,EAAU,CAAA;AAAA;AAC3C,KACD,CAAA;AAED,IAAA,GAAA,CAAI,YAAa,CAAA;AAAA,MACf,IAAM,EAAA;AAAA,QACJ,OAAS,EAAAC,qCAAA;AAAA,QACT,QAAQC,6BAAa,CAAA,UAAA;AAAA,QACrB,UAAUA,6BAAa,CAAA,QAAA;AAAA,QACvB,YAAYA,6BAAa,CAAA,UAAA;AAAA,QACzB,QAAQA,6BAAa,CAAA,MAAA;AAAA,QACrB,WAAWA,6BAAa,CAAA;AAAA,OAC1B;AAAA,MACA,MAAM,IAAK,CAAA;AAAA,QACT,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,MAAA;AAAA,QACA;AAAA,OACC,EAAA;AACD,QAAM,MAAA,MAAA,GAAS,MAAM,QAAA,CAAS,SAAU,EAAA;AAExC,QAAM,MAAA,SAAA,GAAY,IAAIC,iCAAiB,CAAA;AAAA,UACrC,MAAA;AAAA,UACA,MAAA;AAAA,UACA,MAAA;AAAA,UACA;AAAA,SACD,CAAA;AAED,QAAA,KAAA,MAAW,SAAS,cAAgB,EAAA;AAClC,UAAA,MAAM,UAAU,SAAU,CAAA,IAAA,CAAK,KAAM,CAAA,QAAA,EAAU,MAAM,OAAO,CAAA;AAC5D,UAAA,OAAA,CAAQ,kBAAkB,OAAO,CAAA;AAAA;AAGnC,QAAA,UAAA,CAAW,GAAI,CAAA,MAAM,SAAU,CAAA,WAAA,EAAa,CAAA;AAAA;AAC9C,KACD,CAAA;AAAA;AAEL,CAAC;;;;;"}
+{"version":3,"file":"catalogModuleIncrementalIngestionEntityProvider.cjs.js","sources":["../../src/module/catalogModuleIncrementalIngestionEntityProvider.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  coreServices,\n  createBackendModule,\n  createExtensionPoint,\n} from '@backstage/backend-plugin-api';\nimport { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '@backstage/plugin-catalog-backend-module-incremental-ingestion';\nimport { WrapperProviders } from './WrapperProviders';\nimport { eventsServiceRef } from '@backstage/plugin-events-node';\n\n/**\n * @public\n * Interface for {@link incrementalIngestionProvidersExtensionPoint}.\n */\nexport interface IncrementalIngestionProviderExtensionPoint {\n  /** Adds a new incremental entity provider */\n  addProvider<TCursor, TContext>(config: {\n    options: IncrementalEntityProviderOptions;\n    provider: IncrementalEntityProvider<TCursor, TContext>;\n  }): void;\n}\n\n/**\n * @public\n *\n * Extension point for registering incremental ingestion providers.\n * The `catalogModuleIncrementalIngestionEntityProvider` must be installed for these providers to work.\n *\n * @example\n *\n * ```ts\n * backend.add(createBackendModule({\n *   pluginId: 'catalog',\n *   moduleId: 'my-incremental-provider',\n *   register(env) {\n *     env.registerInit({\n *       deps: {\n *         extension: incrementalIngestionProvidersExtensionPoint,\n *       },\n *       async init({ extension }) {\n *         extension.addProvider({\n *           burstInterval: ...,\n *           burstLength: ...,\n *           restLength: ...,\n *         }, {\n *           next(context, cursor) {\n *             ...\n *           },\n *           ...\n *         })\n *       })\n *     })\n *   }\n * }))\n * ```\n */\nexport const incrementalIngestionProvidersExtensionPoint =\n  createExtensionPoint<IncrementalIngestionProviderExtensionPoint>({\n    id: 'catalog.incrementalIngestionProvider.providers',\n  });\n\n/**\n * Registers the incremental entity provider with the catalog processing extension point.\n *\n * @public\n */\nexport const catalogModuleIncrementalIngestionEntityProvider =\n  createBackendModule({\n    pluginId: 'catalog',\n    moduleId: 'incremental-ingestion-entity-provider',\n    register(env) {\n      const addedProviders = new Array<{\n        provider: IncrementalEntityProvider<unknown, unknown>;\n        options: IncrementalEntityProviderOptions;\n      }>();\n\n      env.registerExtensionPoint(incrementalIngestionProvidersExtensionPoint, {\n        addProvider({ options, provider }) {\n          addedProviders.push({ options, provider });\n        },\n      });\n\n      env.registerInit({\n        deps: {\n          catalog: catalogProcessingExtensionPoint,\n          config: coreServices.rootConfig,\n          database: coreServices.database,\n          httpRouter: coreServices.httpRouter,\n          logger: coreServices.logger,\n          scheduler: coreServices.scheduler,\n          events: eventsServiceRef,\n        },\n        async init({\n          catalog,\n          config,\n          database,\n          httpRouter,\n          logger,\n          scheduler,\n          events,\n        }) {\n          const client = await database.getClient();\n\n          const providers = new WrapperProviders({\n            config,\n            logger,\n            client,\n            scheduler,\n            events,\n          });\n\n          for (const entry of addedProviders) {\n            const wrapped = providers.wrap(entry.provider, entry.options);\n            catalog.addEntityProvider(wrapped);\n          }\n\n          httpRouter.use(await providers.adminRouter());\n        },\n      });\n    },\n  });\n"],"names":["createExtensionPoint","createBackendModule","catalogProcessingExtensionPoint","coreServices","eventsServiceRef","WrapperProviders"],"mappings":";;;;;;;AA2EO,MAAM,8CACXA,qCAAiE,CAAA;AAAA,EAC/D,EAAI,EAAA;AACN,CAAC;AAOI,MAAM,kDACXC,oCAAoB,CAAA;AAAA,EAClB,QAAU,EAAA,SAAA;AAAA,EACV,QAAU,EAAA,uCAAA;AAAA,EACV,SAAS,GAAK,EAAA;AACZ,IAAM,MAAA,cAAA,GAAiB,IAAI,KAGxB,EAAA;AAEH,IAAA,GAAA,CAAI,uBAAuB,2CAA6C,EAAA;AAAA,MACtE,WAAY,CAAA,EAAE,OAAS,EAAA,QAAA,EAAY,EAAA;AACjC,QAAA,cAAA,CAAe,IAAK,CAAA,EAAE,OAAS,EAAA,QAAA,EAAU,CAAA;AAAA;AAC3C,KACD,CAAA;AAED,IAAA,GAAA,CAAI,YAAa,CAAA;AAAA,MACf,IAAM,EAAA;AAAA,QACJ,OAAS,EAAAC,qCAAA;AAAA,QACT,QAAQC,6BAAa,CAAA,UAAA;AAAA,QACrB,UAAUA,6BAAa,CAAA,QAAA;AAAA,QACvB,YAAYA,6BAAa,CAAA,UAAA;AAAA,QACzB,QAAQA,6BAAa,CAAA,MAAA;AAAA,QACrB,WAAWA,6BAAa,CAAA,SAAA;AAAA,QACxB,MAAQ,EAAAC;AAAA,OACV;AAAA,MACA,MAAM,IAAK,CAAA;AAAA,QACT,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,MAAA;AAAA,QACA,SAAA;AAAA,QACA;AAAA,OACC,EAAA;AACD,QAAM,MAAA,MAAA,GAAS,MAAM,QAAA,CAAS,SAAU,EAAA;AAExC,QAAM,MAAA,SAAA,GAAY,IAAIC,iCAAiB,CAAA;AAAA,UACrC,MAAA;AAAA,UACA,MAAA;AAAA,UACA,MAAA;AAAA,UACA,SAAA;AAAA,UACA;AAAA,SACD,CAAA;AAED,QAAA,KAAA,MAAW,SAAS,cAAgB,EAAA;AAClC,UAAA,MAAM,UAAU,SAAU,CAAA,IAAA,CAAK,KAAM,CAAA,QAAA,EAAU,MAAM,OAAO,CAAA;AAC5D,UAAA,OAAA,CAAQ,kBAAkB,OAAO,CAAA;AAAA;AAGnC,QAAA,UAAA,CAAW,GAAI,CAAA,MAAM,SAAU,CAAA,WAAA,EAAa,CAAA;AAAA;AAC9C,KACD,CAAA;AAAA;AAEL,CAAC;;;;;"}

package/dist/service/IncrementalCatalogBuilder.cjs.js

@@ -59,8 +59,12 @@ class IncrementalCatalogBuilder {
           restLength,
           connection
         });
-        const frequency = luxon.Duration.isDuration(burstInterval) ? burstInterval : luxon.Duration.fromObject(burstInterval);
-        const length = luxon.Duration.isDuration(burstLength) ? burstLength : luxon.Duration.fromObject(burstLength);
+        let frequency = luxon.Duration.isDuration(burstInterval) ? burstInterval : luxon.Duration.fromObject(burstInterval);
+        if (frequency.as("milliseconds") < 5e3) {
+          frequency = luxon.Duration.fromObject({ seconds: 5 });
+        }
+        let length = luxon.Duration.isDuration(burstLength) ? burstLength : luxon.Duration.fromObject(burstLength);
+        length = length.plus(luxon.Duration.fromObject({ minutes: 1 }));
         await scheduler.scheduleTask({
           id: provider.getProviderName(),
           fn: engine.taskFn.bind(engine),

package/dist/service/IncrementalCatalogBuilder.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"IncrementalCatalogBuilder.cjs.js","sources":["../../src/service/IncrementalCatalogBuilder.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n  PluginEnvironment,\n} from '../types';\nimport { CatalogBuilder as CoreCatalogBuilder } from '@backstage/plugin-catalog-backend';\nimport { createDeferred } from '@backstage/types';\nimport { Duration } from 'luxon';\nimport { Knex } from 'knex';\nimport { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine';\nimport { applyDatabaseMigrations } from '../database/migrations';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { IncrementalProviderRouter } from '../router/routes';\nimport { EventParams, EventSubscriber } from '@backstage/plugin-events-node';\n\n/** @public */\nexport class IncrementalCatalogBuilder {\n  /**\n   * Creates the incremental catalog builder, which extends the regular catalog builder.\n   * @param env - PluginEnvironment\n   * @param builder - CatalogBuilder\n   * @returns IncrementalCatalogBuilder\n   */\n  static async create(env: PluginEnvironment, builder: CoreCatalogBuilder) {\n    const client = await env.database.getClient();\n    const manager = new IncrementalIngestionDatabaseManager({ client });\n    return new IncrementalCatalogBuilder(env, builder, client, manager);\n  }\n\n  private ready = createDeferred();\n\n  private constructor(\n    private env: PluginEnvironment,\n    private builder: CoreCatalogBuilder,\n    private client: Knex,\n    private manager: IncrementalIngestionDatabaseManager,\n  ) {}\n\n  async build() {\n    await applyDatabaseMigrations(this.client);\n    this.ready.resolve();\n\n    const routerLogger = this.env.logger.child({\n      router: 'IncrementalProviderAdmin',\n    });\n\n    const incrementalAdminRouter = await new IncrementalProviderRouter(\n      this.manager,\n      routerLogger,\n    ).createRouter();\n\n    return { incrementalAdminRouter };\n  }\n\n  addIncrementalEntityProvider<TCursor, TContext>(\n    provider: IncrementalEntityProvider<TCursor, TContext>,\n    options: IncrementalEntityProviderOptions,\n  ): EventSubscriber {\n    const { burstInterval, burstLength, restLength } = options;\n    const { logger: catalogLogger, scheduler } = this.env;\n    const ready = this.ready;\n\n    const manager = this.manager;\n\n    let engine: IncrementalIngestionEngine;\n\n    this.builder.addEntityProvider({\n      getProviderName: provider.getProviderName.bind(provider),\n      async connect(connection) {\n        const logger = catalogLogger.child({\n          entityProvider: provider.getProviderName(),\n        });\n\n        engine = new IncrementalIngestionEngine({\n          ...options,\n          ready,\n          manager,\n          logger,\n          provider,\n          restLength,\n          connection,\n        });\n\n        const frequency = Duration.isDuration(burstInterval)\n          ? burstInterval\n          : Duration.fromObject(burstInterval);\n        const length = Duration.isDuration(burstLength)\n          ? burstLength\n          : Duration.fromObject(burstLength);\n\n        await scheduler.scheduleTask({\n          id: provider.getProviderName(),\n          fn: engine.taskFn.bind(engine),\n          frequency,\n          timeout: length,\n        });\n      },\n    });\n\n    return {\n      onEvent: (params: EventParams) => engine.onEvent(params),\n      supportsEventTopics() {\n        return engine.supportsEventTopics();\n      },\n    };\n  }\n}\n"],"names":["IncrementalIngestionDatabaseManager","createDeferred","applyDatabaseMigrations","IncrementalProviderRouter","IncrementalIngestionEngine","Duration"],"mappings":";;;;;;;;;AAgCO,MAAM,yBAA0B,CAAA;AAAA,EAe7B,WACE,CAAA,GAAA,EACA,OACA,EAAA,MAAA,EACA,OACR,EAAA;AAJQ,IAAA,IAAA,CAAA,GAAA,GAAA,GAAA;AACA,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AACA,IAAA,IAAA,CAAA,MAAA,GAAA,MAAA;AACA,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA;AACP;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAbH,aAAa,MAAO,CAAA,GAAA,EAAwB,OAA6B,EAAA;AACvE,IAAA,MAAM,MAAS,GAAA,MAAM,GAAI,CAAA,QAAA,CAAS,SAAU,EAAA;AAC5C,IAAA,MAAM,OAAU,GAAA,IAAIA,uEAAoC,CAAA,EAAE,QAAQ,CAAA;AAClE,IAAA,OAAO,IAAI,yBAAA,CAA0B,GAAK,EAAA,OAAA,EAAS,QAAQ,OAAO,CAAA;AAAA;AACpE,EAEQ,QAAQC,oBAAe,EAAA;AAAA,EAS/B,MAAM,KAAQ,GAAA;AACZ,IAAM,MAAAC,kCAAA,CAAwB,KAAK,MAAM,CAAA;AACzC,IAAA,IAAA,CAAK,MAAM,OAAQ,EAAA;AAEnB,IAAA,MAAM,YAAe,GAAA,IAAA,CAAK,GAAI,CAAA,MAAA,CAAO,KAAM,CAAA;AAAA,MACzC,MAAQ,EAAA;AAAA,KACT,CAAA;AAED,IAAM,MAAA,sBAAA,GAAyB,MAAM,IAAIC,gCAAA;AAAA,MACvC,IAAK,CAAA,OAAA;AAAA,MACL;AAAA,MACA,YAAa,EAAA;AAEf,IAAA,OAAO,EAAE,sBAAuB,EAAA;AAAA;AAClC,EAEA,4BAAA,CACE,UACA,OACiB,EAAA;AACjB,IAAA,MAAM,EAAE,aAAA,EAAe,WAAa,EAAA,UAAA,EAAe,GAAA,OAAA;AACnD,IAAA,MAAM,EAAE,MAAA,EAAQ,aAAe,EAAA,SAAA,KAAc,IAAK,CAAA,GAAA;AAClD,IAAA,MAAM,QAAQ,IAAK,CAAA,KAAA;AAEnB,IAAA,MAAM,UAAU,IAAK,CAAA,OAAA;AAErB,IAAI,IAAA,MAAA;AAEJ,IAAA,IAAA,CAAK,QAAQ,iBAAkB,CAAA;AAAA,MAC7B,eAAiB,EAAA,QAAA,CAAS,eAAgB,CAAA,IAAA,CAAK,QAAQ,CAAA;AAAA,MACvD,MAAM,QAAQ,UAAY,EAAA;AACxB,QAAM,MAAA,MAAA,GAAS,cAAc,KAAM,CAAA;AAAA,UACjC,cAAA,EAAgB,SAAS,eAAgB;AAAA,SAC1C,CAAA;AAED,QAAA,MAAA,GAAS,IAAIC,qDAA2B,CAAA;AAAA,UACtC,GAAG,OAAA;AAAA,UACH,KAAA;AAAA,UACA,OAAA;AAAA,UACA,MAAA;AAAA,UACA,QAAA;AAAA,UACA,UAAA;AAAA,UACA;AAAA,SACD,CAAA;AAED,QAAM,MAAA,SAAA,GAAYC,eAAS,UAAW,CAAA,aAAa,IAC/C,aACA,GAAAA,cAAA,CAAS,WAAW,aAAa,CAAA;AACrC,QAAM,MAAA,MAAA,GAASA,eAAS,UAAW,CAAA,WAAW,IAC1C,WACA,GAAAA,cAAA,CAAS,WAAW,WAAW,CAAA;AAEnC,QAAA,MAAM,UAAU,YAAa,CAAA;AAAA,UAC3B,EAAA,EAAI,SAAS,eAAgB,EAAA;AAAA,UAC7B,EAAI,EAAA,MAAA,CAAO,MAAO,CAAA,IAAA,CAAK,MAAM,CAAA;AAAA,UAC7B,SAAA;AAAA,UACA,OAAS,EAAA;AAAA,SACV,CAAA;AAAA;AACH,KACD,CAAA;AAED,IAAO,OAAA;AAAA,MACL,OAAS,EAAA,CAAC,MAAwB,KAAA,MAAA,CAAO,QAAQ,MAAM,CAAA;AAAA,MACvD,mBAAsB,GAAA;AACpB,QAAA,OAAO,OAAO,mBAAoB,EAAA;AAAA;AACpC,KACF;AAAA;AAEJ;;;;"}
+{"version":3,"file":"IncrementalCatalogBuilder.cjs.js","sources":["../../src/service/IncrementalCatalogBuilder.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n  PluginEnvironment,\n} from '../types';\nimport { CatalogBuilder as CoreCatalogBuilder } from '@backstage/plugin-catalog-backend';\nimport { createDeferred } from '@backstage/types';\nimport { Duration } from 'luxon';\nimport { Knex } from 'knex';\nimport { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine';\nimport { applyDatabaseMigrations } from '../database/migrations';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { IncrementalProviderRouter } from '../router/routes';\nimport { EventParams, EventSubscriber } from '@backstage/plugin-events-node';\n\n/**\n * @public\n * @deprecated Please migrate to the new backend system and import `@backstage/plugin-catalog-backend-module-incremental-ingestion` as a module, and add providers to the `incrementalIngestionProvidersExtensionPoint` instead\n */\nexport class IncrementalCatalogBuilder {\n  /**\n   * Creates the incremental catalog builder, which extends the regular catalog builder.\n   * @param env - PluginEnvironment\n   * @param builder - CatalogBuilder\n   * @returns IncrementalCatalogBuilder\n   */\n  static async create(env: PluginEnvironment, builder: CoreCatalogBuilder) {\n    const client = await env.database.getClient();\n    const manager = new IncrementalIngestionDatabaseManager({ client });\n    return new IncrementalCatalogBuilder(env, builder, client, manager);\n  }\n\n  private ready = createDeferred();\n\n  private constructor(\n    private env: PluginEnvironment,\n    private builder: CoreCatalogBuilder,\n    private client: Knex,\n    private manager: IncrementalIngestionDatabaseManager,\n  ) {}\n\n  async build() {\n    await applyDatabaseMigrations(this.client);\n    this.ready.resolve();\n\n    const routerLogger = this.env.logger.child({\n      router: 'IncrementalProviderAdmin',\n    });\n\n    const incrementalAdminRouter = await new IncrementalProviderRouter(\n      this.manager,\n      routerLogger,\n    ).createRouter();\n\n    return { incrementalAdminRouter };\n  }\n\n  addIncrementalEntityProvider<TCursor, TContext>(\n    provider: IncrementalEntityProvider<TCursor, TContext>,\n    options: IncrementalEntityProviderOptions,\n  ): EventSubscriber {\n    const { burstInterval, burstLength, restLength } = options;\n    const { logger: catalogLogger, scheduler } = this.env;\n    const ready = this.ready;\n\n    const manager = this.manager;\n\n    let engine: IncrementalIngestionEngine;\n\n    this.builder.addEntityProvider({\n      getProviderName: provider.getProviderName.bind(provider),\n      async connect(connection) {\n        const logger = catalogLogger.child({\n          entityProvider: provider.getProviderName(),\n        });\n\n        engine = new IncrementalIngestionEngine({\n          ...options,\n          ready,\n          manager,\n          logger,\n          provider,\n          restLength,\n          connection,\n        });\n\n        let frequency = Duration.isDuration(burstInterval)\n          ? burstInterval\n          : Duration.fromObject(burstInterval);\n        if (frequency.as('milliseconds') < 5000) {\n          frequency = Duration.fromObject({ seconds: 5 }); // don't let it be silly low, to not overload the scheduler\n        }\n\n        let length = Duration.isDuration(burstLength)\n          ? burstLength\n          : Duration.fromObject(burstLength);\n        length = length.plus(Duration.fromObject({ minutes: 1 })); // some margin from the actual completion\n\n        await scheduler.scheduleTask({\n          id: provider.getProviderName(),\n          fn: engine.taskFn.bind(engine),\n          frequency,\n          timeout: length,\n        });\n      },\n    });\n\n    return {\n      onEvent: (params: EventParams) => engine.onEvent(params),\n      supportsEventTopics() {\n        return engine.supportsEventTopics();\n      },\n    };\n  }\n}\n"],"names":["IncrementalIngestionDatabaseManager","createDeferred","applyDatabaseMigrations","IncrementalProviderRouter","IncrementalIngestionEngine","Duration"],"mappings":";;;;;;;;;AAmCO,MAAM,yBAA0B,CAAA;AAAA,EAe7B,WACE,CAAA,GAAA,EACA,OACA,EAAA,MAAA,EACA,OACR,EAAA;AAJQ,IAAA,IAAA,CAAA,GAAA,GAAA,GAAA;AACA,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AACA,IAAA,IAAA,CAAA,MAAA,GAAA,MAAA;AACA,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA;AACP;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAbH,aAAa,MAAO,CAAA,GAAA,EAAwB,OAA6B,EAAA;AACvE,IAAA,MAAM,MAAS,GAAA,MAAM,GAAI,CAAA,QAAA,CAAS,SAAU,EAAA;AAC5C,IAAA,MAAM,OAAU,GAAA,IAAIA,uEAAoC,CAAA,EAAE,QAAQ,CAAA;AAClE,IAAA,OAAO,IAAI,yBAAA,CAA0B,GAAK,EAAA,OAAA,EAAS,QAAQ,OAAO,CAAA;AAAA;AACpE,EAEQ,QAAQC,oBAAe,EAAA;AAAA,EAS/B,MAAM,KAAQ,GAAA;AACZ,IAAM,MAAAC,kCAAA,CAAwB,KAAK,MAAM,CAAA;AACzC,IAAA,IAAA,CAAK,MAAM,OAAQ,EAAA;AAEnB,IAAA,MAAM,YAAe,GAAA,IAAA,CAAK,GAAI,CAAA,MAAA,CAAO,KAAM,CAAA;AAAA,MACzC,MAAQ,EAAA;AAAA,KACT,CAAA;AAED,IAAM,MAAA,sBAAA,GAAyB,MAAM,IAAIC,gCAAA;AAAA,MACvC,IAAK,CAAA,OAAA;AAAA,MACL;AAAA,MACA,YAAa,EAAA;AAEf,IAAA,OAAO,EAAE,sBAAuB,EAAA;AAAA;AAClC,EAEA,4BAAA,CACE,UACA,OACiB,EAAA;AACjB,IAAA,MAAM,EAAE,aAAA,EAAe,WAAa,EAAA,UAAA,EAAe,GAAA,OAAA;AACnD,IAAA,MAAM,EAAE,MAAA,EAAQ,aAAe,EAAA,SAAA,KAAc,IAAK,CAAA,GAAA;AAClD,IAAA,MAAM,QAAQ,IAAK,CAAA,KAAA;AAEnB,IAAA,MAAM,UAAU,IAAK,CAAA,OAAA;AAErB,IAAI,IAAA,MAAA;AAEJ,IAAA,IAAA,CAAK,QAAQ,iBAAkB,CAAA;AAAA,MAC7B,eAAiB,EAAA,QAAA,CAAS,eAAgB,CAAA,IAAA,CAAK,QAAQ,CAAA;AAAA,MACvD,MAAM,QAAQ,UAAY,EAAA;AACxB,QAAM,MAAA,MAAA,GAAS,cAAc,KAAM,CAAA;AAAA,UACjC,cAAA,EAAgB,SAAS,eAAgB;AAAA,SAC1C,CAAA;AAED,QAAA,MAAA,GAAS,IAAIC,qDAA2B,CAAA;AAAA,UACtC,GAAG,OAAA;AAAA,UACH,KAAA;AAAA,UACA,OAAA;AAAA,UACA,MAAA;AAAA,UACA,QAAA;AAAA,UACA,UAAA;AAAA,UACA;AAAA,SACD,CAAA;AAED,QAAI,IAAA,SAAA,GAAYC,eAAS,UAAW,CAAA,aAAa,IAC7C,aACA,GAAAA,cAAA,CAAS,WAAW,aAAa,CAAA;AACrC,QAAA,IAAI,SAAU,CAAA,EAAA,CAAG,cAAc,CAAA,GAAI,GAAM,EAAA;AACvC,UAAA,SAAA,GAAYA,cAAS,CAAA,UAAA,CAAW,EAAE,OAAA,EAAS,GAAG,CAAA;AAAA;AAGhD,QAAI,IAAA,MAAA,GAASA,eAAS,UAAW,CAAA,WAAW,IACxC,WACA,GAAAA,cAAA,CAAS,WAAW,WAAW,CAAA;AACnC,QAAS,MAAA,GAAA,MAAA,CAAO,KAAKA,cAAS,CAAA,UAAA,CAAW,EAAE,OAAS,EAAA,CAAA,EAAG,CAAC,CAAA;AAExD,QAAA,MAAM,UAAU,YAAa,CAAA;AAAA,UAC3B,EAAA,EAAI,SAAS,eAAgB,EAAA;AAAA,UAC7B,EAAI,EAAA,MAAA,CAAO,MAAO,CAAA,IAAA,CAAK,MAAM,CAAA;AAAA,UAC7B,SAAA;AAAA,UACA,OAAS,EAAA;AAAA,SACV,CAAA;AAAA;AACH,KACD,CAAA;AAED,IAAO,OAAA;AAAA,MACL,OAAS,EAAA,CAAC,MAAwB,KAAA,MAAA,CAAO,QAAQ,MAAM,CAAA;AAAA,MACvD,mBAAsB,GAAA;AACpB,QAAA,OAAO,OAAO,mBAAoB,EAAA;AAAA;AACpC,KACF;AAAA;AAEJ;;;;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend-module-incremental-ingestion",
-  "version": "0.6.0",
+  "version": "0.6.1-next.0",
   "description": "An entity provider for streaming large asset sources into the catalog",
   "backstage": {
     "role": "backend-plugin-module",
@@ -62,15 +62,15 @@
   },
   "dependencies": {
     "@backstage/backend-common": "^0.25.0",
-    "@backstage/backend-plugin-api": "^1.0.2",
-    "@backstage/catalog-model": "^1.7.1",
-    "@backstage/config": "^1.3.0",
-    "@backstage/errors": "^1.2.5",
-    "@backstage/plugin-catalog-backend": "^1.28.0",
-    "@backstage/plugin-catalog-node": "^1.14.0",
-    "@backstage/plugin-events-node": "^0.4.5",
-    "@backstage/plugin-permission-common": "^0.8.2",
-    "@backstage/types": "^1.2.0",
+    "@backstage/backend-plugin-api": "1.0.3-next.0",
+    "@backstage/catalog-model": "1.7.1",
+    "@backstage/config": "1.3.0",
+    "@backstage/errors": "1.2.5",
+    "@backstage/plugin-catalog-backend": "1.28.1-next.0",
+    "@backstage/plugin-catalog-node": "1.14.1-next.0",
+    "@backstage/plugin-events-node": "0.4.6-next.0",
+    "@backstage/plugin-permission-common": "0.8.2",
+    "@backstage/types": "1.2.0",
     "@types/express": "^4.17.6",
     "express": "^4.17.1",
     "express-promise-router": "^4.1.0",
@@ -79,9 +79,9 @@
     "uuid": "^11.0.0"
   },
   "devDependencies": {
-    "@backstage/backend-defaults": "^0.5.3",
-    "@backstage/backend-test-utils": "^1.1.0",
-    "@backstage/cli": "^0.29.0",
+    "@backstage/backend-defaults": "0.6.0-next.0",
+    "@backstage/backend-test-utils": "1.2.0-next.0",
+    "@backstage/cli": "0.29.3-next.0",
     "@types/luxon": "^3.0.0"
   }
 }