@backstage/plugin-catalog-backend-module-incremental-ingestion 0.0.0-nightly-20221125023412

package/dist/index.cjs.js

@@ -0,0 +1,979 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+var backendPluginApi = require('@backstage/backend-plugin-api');
+var pluginCatalogNode = require('@backstage/plugin-catalog-node');
+var errors = require('@backstage/errors');
+var luxon = require('luxon');
+var catalogModel = require('@backstage/catalog-model');
+var uuid = require('uuid');
+var backendCommon = require('@backstage/backend-common');
+var perf_hooks = require('perf_hooks');
+var express = require('express');
+var Router = require('express-promise-router');
+
+function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
+
+var express__default = /*#__PURE__*/_interopDefaultLegacy(express);
+var Router__default = /*#__PURE__*/_interopDefaultLegacy(Router);
+
+const INCREMENTAL_ENTITY_PROVIDER_ANNOTATION = "backstage.io/incremental-provider-name";
+
+class IncrementalIngestionDatabaseManager {
+  constructor(options) {
+    this.client = options.client;
+  }
+  async updateIngestionRecordById(options) {
+    await this.client.transaction(async (tx) => {
+      const { ingestionId, update } = options;
+      await tx("ingestions").where("id", ingestionId).update(update);
+    });
+  }
+  async updateIngestionRecordByProvider(provider, update) {
+    await this.client.transaction(async (tx) => {
+      await tx("ingestions").where("provider_name", provider).andWhere("completion_ticket", "open").update(update);
+    });
+  }
+  async insertIngestionRecord(record) {
+    await this.client.transaction(async (tx) => {
+      await tx("ingestions").insert(record);
+    });
+  }
+  async deleteMarkEntities(tx, ids) {
+    const chunks = [];
+    for (let i = 0; i < ids.length; i += 100) {
+      const chunk = ids.slice(i, i + 100);
+      chunks.push(chunk);
+    }
+    let deleted = 0;
+    for (const chunk of chunks) {
+      const chunkDeleted = await tx("ingestion_mark_entities").delete().whereIn(
+        "id",
+        chunk.map((entry) => entry.id)
+      );
+      deleted += chunkDeleted;
+    }
+    return deleted;
+  }
+  async getCurrentIngestionRecord(provider) {
+    return await this.client.transaction(async (tx) => {
+      const record = await tx("ingestions").where("provider_name", provider).andWhere("completion_ticket", "open").first();
+      return record;
+    });
+  }
+  async clearFinishedIngestions(provider) {
+    return await this.client.transaction(async (tx) => {
+      const markEntitiesDeleted = await tx("ingestion_mark_entities").delete().whereIn(
+        "ingestion_mark_id",
+        tx("ingestion_marks").select("id").whereIn(
+          "ingestion_id",
+          tx("ingestions").select("id").where("provider_name", provider).andWhereNot("completion_ticket", "open")
+        )
+      );
+      const marksDeleted = await tx("ingestion_marks").delete().whereIn(
+        "ingestion_id",
+        tx("ingestions").select("id").where("provider_name", provider).andWhereNot("completion_ticket", "open")
+      );
+      const ingestionsDeleted = await tx("ingestions").delete().where("provider_name", provider).andWhereNot("completion_ticket", "open");
+      return {
+        deletions: {
+          markEntitiesDeleted,
+          marksDeleted,
+          ingestionsDeleted
+        }
+      };
+    });
+  }
+  async clearDuplicateIngestions(ingestionId, provider) {
+    await this.client.transaction(async (tx) => {
+      const invalid = await tx("ingestions").where("provider_name", provider).andWhere("rest_completed_at", null).andWhereNot("id", ingestionId);
+      if (invalid.length > 0) {
+        await tx("ingestions").delete().whereIn("id", invalid);
+        await tx("ingestion_mark_entities").delete().whereIn(
+          "ingestion_mark_id",
+          tx("ingestion_marks").select("id").whereIn("ingestion_id", invalid)
+        );
+        await tx("ingestion_marks").delete().whereIn("ingestion_id", invalid);
+      }
+    });
+  }
+  async purgeAndResetProvider(provider) {
+    return await this.client.transaction(async (tx) => {
+      const ingestionIDs = await tx("ingestions").select("id").where("provider_name", provider);
+      const markIDs = ingestionIDs.length > 0 ? await tx("ingestion_marks").select("id").whereIn(
+        "ingestion_id",
+        ingestionIDs.map((entry) => entry.id)
+      ) : [];
+      const markEntityIDs = markIDs.length > 0 ? await tx("ingestion_mark_entities").select("id").whereIn(
+        "ingestion_mark_id",
+        markIDs.map((entry) => entry.id)
+      ) : [];
+      const markEntitiesDeleted = await this.deleteMarkEntities(
+        tx,
+        markEntityIDs
+      );
+      const marksDeleted = markIDs.length > 0 ? await tx("ingestion_marks").delete().whereIn(
+        "ingestion_id",
+        ingestionIDs.map((entry) => entry.id)
+      ) : 0;
+      const ingestionsDeleted = await tx("ingestions").delete().where("provider_name", provider);
+      const next_action_at = new Date();
+      next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1e3);
+      await this.insertIngestionRecord({
+        id: uuid.v4(),
+        next_action: "rest",
+        provider_name: provider,
+        next_action_at,
+        ingestion_completed_at: new Date(),
+        status: "resting",
+        completion_ticket: "open"
+      });
+      return { provider, ingestionsDeleted, marksDeleted, markEntitiesDeleted };
+    });
+  }
+  async createProviderIngestionRecord(provider) {
+    const ingestionId = uuid.v4();
+    const nextAction = "ingest";
+    try {
+      await this.insertIngestionRecord({
+        id: ingestionId,
+        next_action: nextAction,
+        provider_name: provider,
+        status: "bursting",
+        completion_ticket: "open"
+      });
+      return { ingestionId, nextAction, attempts: 0, nextActionAt: Date.now() };
+    } catch (_e) {
+      return void 0;
+    }
+  }
+  async computeRemoved(provider, ingestionId) {
+    return await this.client.transaction(async (tx) => {
+      const removed = await tx(
+        "final_entities"
+      ).select(
+        tx.ref("final_entity").as("entity"),
+        tx.ref("refresh_state.entity_ref").as("ref")
+      ).join(
+        "refresh_state",
+        "refresh_state.entity_id",
+        "final_entities.entity_id"
+      ).join("search", "search.entity_id", "final_entities.entity_id").whereNotIn(
+        "entity_ref",
+        tx("ingestion_marks").join(
+          "ingestion_mark_entities",
+          "ingestion_marks.id",
+          "ingestion_mark_entities.ingestion_mark_id"
+        ).select("ingestion_mark_entities.ref").where("ingestion_marks.ingestion_id", ingestionId)
+      ).andWhere(
+        "search.key",
+        `metadata.annotations.${INCREMENTAL_ENTITY_PROVIDER_ANNOTATION}`
+      ).andWhere("search.value", provider);
+      return removed.map((entity) => {
+        return { entity: JSON.parse(entity.entity) };
+      });
+    });
+  }
+  async healthcheck() {
+    return await this.client.transaction(async (tx) => {
+      const records = await tx(
+        "ingestions"
+      ).distinct("id", "provider_name").where("rest_completed_at", null);
+      return records;
+    });
+  }
+  async triggerNextProviderAction(provider) {
+    await this.updateIngestionRecordByProvider(provider, {
+      next_action_at: new Date()
+    });
+  }
+  async cleanupProviders() {
+    const providers = await this.listProviders();
+    const ingestionsDeleted = await this.purgeTable("ingestions");
+    const next_action_at = new Date();
+    next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1e3);
+    for (const provider of providers) {
+      await this.insertIngestionRecord({
+        id: uuid.v4(),
+        next_action: "rest",
+        provider_name: provider,
+        next_action_at,
+        ingestion_completed_at: new Date(),
+        status: "resting",
+        completion_ticket: "open"
+      });
+    }
+    const ingestionMarksDeleted = await this.purgeTable("ingestion_marks");
+    const markEntitiesDeleted = await this.purgeTable(
+      "ingestion_mark_entities"
+    );
+    return { ingestionsDeleted, ingestionMarksDeleted, markEntitiesDeleted };
+  }
+  async setProviderIngesting(ingestionId) {
+    await this.updateIngestionRecordById({
+      ingestionId,
+      update: { next_action: "ingest" }
+    });
+  }
+  async setProviderBursting(ingestionId) {
+    await this.updateIngestionRecordById({
+      ingestionId,
+      update: { status: "bursting" }
+    });
+  }
+  async setProviderComplete(ingestionId) {
+    await this.updateIngestionRecordById({
+      ingestionId,
+      update: {
+        next_action: "nothing (done)",
+        rest_completed_at: new Date(),
+        status: "complete",
+        completion_ticket: uuid.v4()
+      }
+    });
+  }
+  async setProviderResting(ingestionId, restLength) {
+    await this.updateIngestionRecordById({
+      ingestionId,
+      update: {
+        next_action: "rest",
+        next_action_at: new Date(Date.now() + restLength.as("milliseconds")),
+        ingestion_completed_at: new Date(),
+        status: "resting"
+      }
+    });
+  }
+  async setProviderInterstitial(ingestionId) {
+    await this.updateIngestionRecordById({
+      ingestionId,
+      update: { attempts: 0, status: "interstitial" }
+    });
+  }
+  async setProviderCanceling(ingestionId, message) {
+    const update = {
+      next_action: "cancel",
+      last_error: message ? message : void 0,
+      next_action_at: new Date(),
+      status: "canceling"
+    };
+    await this.updateIngestionRecordById({ ingestionId, update });
+  }
+  async setProviderCanceled(ingestionId) {
+    await this.updateIngestionRecordById({
+      ingestionId,
+      update: {
+        next_action: "nothing (canceled)",
+        rest_completed_at: new Date(),
+        status: "complete",
+        completion_ticket: uuid.v4()
+      }
+    });
+  }
+  async setProviderBackoff(ingestionId, attempts, error, backoffLength) {
+    await this.updateIngestionRecordById({
+      ingestionId,
+      update: {
+        next_action: "backoff",
+        attempts: attempts + 1,
+        last_error: String(error),
+        next_action_at: new Date(Date.now() + backoffLength),
+        status: "backing off"
+      }
+    });
+  }
+  async getLastMark(ingestionId) {
+    return await this.client.transaction(async (tx) => {
+      const mark = await tx("ingestion_marks").where("ingestion_id", ingestionId).orderBy("sequence", "desc").first();
+      return mark;
+    });
+  }
+  async getAllMarks(ingestionId) {
+    return await this.client.transaction(async (tx) => {
+      const marks = await tx("ingestion_marks").where("ingestion_id", ingestionId).orderBy("sequence", "desc");
+      return marks;
+    });
+  }
+  async createMark(options) {
+    const { record } = options;
+    await this.client.transaction(async (tx) => {
+      await tx("ingestion_marks").insert(record);
+    });
+  }
+  async createMarkEntities(markId, entities) {
+    await this.client.transaction(async (tx) => {
+      await tx("ingestion_mark_entities").insert(
+        entities.map((entity) => ({
+          id: uuid.v4(),
+          ingestion_mark_id: markId,
+          ref: catalogModel.stringifyEntityRef(entity.entity)
+        }))
+      );
+    });
+  }
+  async purgeTable(table) {
+    return await this.client.transaction(async (tx) => {
+      return await tx(table).delete();
+    });
+  }
+  async listProviders() {
+    return await this.client.transaction(async (tx) => {
+      const providers = await tx(
+        "ingestions"
+      ).distinct("provider_name");
+      return providers.map((entry) => entry.provider_name);
+    });
+  }
+  async updateByName(provider, update) {
+    await this.updateIngestionRecordByProvider(provider, update);
+  }
+}
+
+const DB_MIGRATIONS_TABLE = "incremental_ingestion__knex_migrations";
+
+async function applyDatabaseMigrations(knex) {
+  const migrationsDir = backendCommon.resolvePackagePath(
+    "@backstage/plugin-catalog-backend-module-incremental-ingestion",
+    "migrations"
+  );
+  await knex.migrate.latest({
+    directory: migrationsDir,
+    tableName: DB_MIGRATIONS_TABLE
+  });
+}
+
+class IncrementalIngestionEngine {
+  constructor(options) {
+    this.options = options;
+    var _a;
+    this.manager = options.manager;
+    this.restLength = luxon.Duration.fromObject(options.restLength);
+    this.backoff = (_a = options.backoff) != null ? _a : [
+      { minutes: 1 },
+      { minutes: 5 },
+      { minutes: 30 },
+      { hours: 3 }
+    ];
+  }
+  async taskFn(signal) {
+    try {
+      this.options.logger.debug("Begin tick");
+      await this.handleNextAction(signal);
+    } catch (error) {
+      this.options.logger.error(`${error}`);
+      throw error;
+    } finally {
+      this.options.logger.debug("End tick");
+    }
+  }
+  async handleNextAction(signal) {
+    await this.options.ready;
+    const result = await this.getCurrentAction();
+    if (result) {
+      const { ingestionId, nextActionAt, nextAction, attempts } = result;
+      switch (nextAction) {
+        case "rest":
+          if (Date.now() > nextActionAt) {
+            await this.manager.clearFinishedIngestions(
+              this.options.provider.getProviderName()
+            );
+            this.options.logger.info(
+              `incremental-engine: Ingestion ${ingestionId} rest period complete. Ingestion will start again`
+            );
+            await this.manager.setProviderComplete(ingestionId);
+          } else {
+            this.options.logger.info(
+              `incremental-engine: Ingestion '${ingestionId}' rest period continuing`
+            );
+          }
+          break;
+        case "ingest":
+          try {
+            await this.manager.setProviderBursting(ingestionId);
+            const done = await this.ingestOneBurst(ingestionId, signal);
+            if (done) {
+              this.options.logger.info(
+                `incremental-engine: Ingestion '${ingestionId}' complete, transitioning to rest period of ${this.restLength.toHuman()}`
+              );
+              await this.manager.setProviderResting(
+                ingestionId,
+                this.restLength
+              );
+            } else {
+              await this.manager.setProviderInterstitial(ingestionId);
+              this.options.logger.info(
+                `incremental-engine: Ingestion '${ingestionId}' continuing`
+              );
+            }
+          } catch (error) {
+            if (error.message && error.message === "CANCEL") {
+              this.options.logger.info(
+                `incremental-engine: Ingestion '${ingestionId}' canceled`
+              );
+              await this.manager.setProviderCanceling(
+                ingestionId,
+                error.message
+              );
+            } else {
+              const currentBackoff = luxon.Duration.fromObject(
+                this.backoff[Math.min(this.backoff.length - 1, attempts)]
+              );
+              const backoffLength = currentBackoff.as("milliseconds");
+              this.options.logger.error(error);
+              const truncatedError = errors.stringifyError(error).substring(0, 700);
+              this.options.logger.error(
+                `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`
+              );
+              await this.manager.setProviderBackoff(
+                ingestionId,
+                attempts,
+                error,
+                backoffLength
+              );
+            }
+          }
+          break;
+        case "backoff":
+          if (Date.now() > nextActionAt) {
+            this.options.logger.info(
+              `incremental-engine: Ingestion '${ingestionId}' backoff complete, will attempt to resume`
+            );
+            await this.manager.setProviderIngesting(ingestionId);
+          } else {
+            this.options.logger.info(
+              `incremental-engine: Ingestion '${ingestionId}' backoff continuing`
+            );
+          }
+          break;
+        case "cancel":
+          this.options.logger.info(
+            `incremental-engine: Ingestion '${ingestionId}' canceling, will restart`
+          );
+          await this.manager.setProviderCanceled(ingestionId);
+          break;
+        default:
+          this.options.logger.error(
+            `incremental-engine: Ingestion '${ingestionId}' received unknown action '${nextAction}'`
+          );
+      }
+    } else {
+      this.options.logger.error(
+        `incremental-engine: Engine tried to create duplicate ingestion record for provider '${this.options.provider.getProviderName()}'.`
+      );
+    }
+  }
+  async getCurrentAction() {
+    const providerName = this.options.provider.getProviderName();
+    const record = await this.manager.getCurrentIngestionRecord(providerName);
+    if (record) {
+      this.options.logger.info(
+        `incremental-engine: Ingestion record found: '${record.id}'`
+      );
+      return {
+        ingestionId: record.id,
+        nextAction: record.next_action,
+        attempts: record.attempts,
+        nextActionAt: record.next_action_at.valueOf()
+      };
+    }
+    const result = await this.manager.createProviderIngestionRecord(
+      providerName
+    );
+    if (result) {
+      this.options.logger.info(
+        `incremental-engine: Ingestion record created: '${result.ingestionId}'`
+      );
+    }
+    return result;
+  }
+  async ingestOneBurst(id, signal) {
+    const lastMark = await this.manager.getLastMark(id);
+    const cursor = lastMark ? lastMark.cursor : void 0;
+    let sequence = lastMark ? lastMark.sequence + 1 : 0;
+    const start = perf_hooks.performance.now();
+    let count = 0;
+    let done = false;
+    this.options.logger.info(
+      `incremental-engine: Ingestion '${id}' burst initiated`
+    );
+    await this.options.provider.around(async (context) => {
+      let next = await this.options.provider.next(context, cursor);
+      count++;
+      for (; ; ) {
+        done = next.done;
+        await this.mark({
+          id,
+          sequence,
+          entities: next == null ? void 0 : next.entities,
+          done: next.done,
+          cursor: next == null ? void 0 : next.cursor
+        });
+        if (signal.aborted || next.done) {
+          break;
+        } else {
+          next = await this.options.provider.next(context, next.cursor);
+          count++;
+          sequence++;
+        }
+      }
+    });
+    this.options.logger.info(
+      `incremental-engine: Ingestion '${id}' burst complete. (${count} batches in ${Math.round(
+        perf_hooks.performance.now() - start
+      )}ms).`
+    );
+    return done;
+  }
+  async mark(options) {
+    var _a;
+    const { id, sequence, entities, done, cursor } = options;
+    this.options.logger.debug(
+      `incremental-engine: Ingestion '${id}': MARK ${entities ? entities.length : 0} entities, cursor: ${cursor ? JSON.stringify(cursor) : "none"}, done: ${done}`
+    );
+    const markId = uuid.v4();
+    await this.manager.createMark({
+      record: {
+        id: markId,
+        ingestion_id: id,
+        cursor,
+        sequence
+      }
+    });
+    if (entities && entities.length > 0) {
+      await this.manager.createMarkEntities(markId, entities);
+    }
+    const added = (_a = entities == null ? void 0 : entities.map((deferred) => ({
+      ...deferred,
+      entity: {
+        ...deferred.entity,
+        metadata: {
+          ...deferred.entity.metadata,
+          annotations: {
+            ...deferred.entity.metadata.annotations,
+            [INCREMENTAL_ENTITY_PROVIDER_ANNOTATION]: this.options.provider.getProviderName()
+          }
+        }
+      }
+    }))) != null ? _a : [];
+    const removed = [];
+    if (done) {
+      removed.push(
+        ...await this.manager.computeRemoved(
+          this.options.provider.getProviderName(),
+          id
+        )
+      );
+    }
+    await this.options.connection.applyMutation({
+      type: "delta",
+      added,
+      removed
+    });
+  }
+}
+
+const PROVIDER_CLEANUP = "/incremental/cleanup";
+const PROVIDER_HEALTH = "/incremental/health";
+const PROVIDER_BASE_PATH = "/incremental/providers/:provider";
+
+const createIncrementalProviderRouter = async (manager, logger) => {
+  const router = Router__default["default"]();
+  router.use(express__default["default"].json());
+  router.get(PROVIDER_HEALTH, async (_, res) => {
+    const records = await manager.healthcheck();
+    const providers = records.map((record) => record.provider_name);
+    const duplicates = [
+      ...new Set(providers.filter((e, i, a) => a.indexOf(e) !== i))
+    ];
+    if (duplicates.length > 0) {
+      res.json({ healthy: false, duplicateIngestions: duplicates });
+    } else {
+      res.json({ healthy: true });
+    }
+  });
+  router.post(PROVIDER_CLEANUP, async (_, res) => {
+    const result = await manager.cleanupProviders();
+    res.json(result);
+  });
+  router.get(PROVIDER_BASE_PATH, async (req, res) => {
+    const { provider } = req.params;
+    const record = await manager.getCurrentIngestionRecord(provider);
+    if (record) {
+      res.json({
+        success: true,
+        status: {
+          current_action: record.status,
+          next_action_at: new Date(record.next_action_at)
+        },
+        last_error: record.last_error
+      });
+    } else {
+      const providers = await manager.listProviders();
+      if (providers.includes(provider)) {
+        res.json({
+          success: true,
+          status: {
+            current_action: "rest complete, waiting to start"
+          }
+        });
+      } else {
+        logger.error(
+          `${provider} - No ingestion record found in the database!`
+        );
+        res.status(404).json({
+          success: false,
+          status: {},
+          last_error: `Provider '${provider}' not found`
+        });
+      }
+    }
+  });
+  router.post(`${PROVIDER_BASE_PATH}/trigger`, async (req, res) => {
+    const { provider } = req.params;
+    const record = await manager.getCurrentIngestionRecord(provider);
+    if (record) {
+      await manager.triggerNextProviderAction(provider);
+      res.json({
+        success: true,
+        message: `${provider}: Next action triggered.`
+      });
+    } else {
+      const providers = await manager.listProviders();
+      if (providers.includes(provider)) {
+        logger.debug(`${provider} - Ingestion record found`);
+        res.json({
+          success: true,
+          message: "Unable to trigger next action (provider is restarting)"
+        });
+      } else {
+        res.status(404).json({
+          success: false,
+          message: `Provider '${provider}' not found`
+        });
+      }
+    }
+  });
+  router.post(`${PROVIDER_BASE_PATH}/start`, async (req, res) => {
+    const { provider } = req.params;
+    const record = await manager.getCurrentIngestionRecord(provider);
+    if (record) {
+      const ingestionId = record.id;
+      if (record.status === "resting") {
+        await manager.setProviderComplete(ingestionId);
+      } else {
+        await manager.setProviderCanceling(ingestionId);
+      }
+      res.json({
+        success: true,
+        message: `${provider}: Next cycle triggered.`
+      });
+    } else {
+      const providers = await manager.listProviders();
+      if (providers.includes(provider)) {
+        logger.debug(`${provider} - Ingestion record found`);
+        res.json({
+          success: true,
+          message: "Provider is already restarting"
+        });
+      } else {
+        res.status(404).json({
+          success: false,
+          message: `Provider '${provider}' not found`
+        });
+      }
+    }
+  });
+  router.post(`${PROVIDER_BASE_PATH}/cancel`, async (req, res) => {
+    const { provider } = req.params;
+    const record = await manager.getCurrentIngestionRecord(provider);
+    if (record) {
+      const next_action_at = new Date();
+      next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1e3);
+      await manager.updateByName(provider, {
+        next_action: "nothing (done)",
+        ingestion_completed_at: new Date(),
+        next_action_at,
+        status: "resting"
+      });
+      res.json({
+        success: true,
+        message: `${provider}: Current ingestion canceled.`
+      });
+    } else {
+      const providers = await manager.listProviders();
+      if (providers.includes(provider)) {
+        logger.debug(`${provider} - Ingestion record found`);
+        res.json({
+          success: true,
+          message: "Provider is currently restarting, please wait."
+        });
+      } else {
+        res.status(404).json({
+          success: false,
+          message: `Provider '${provider}' not found`
+        });
+      }
+    }
+  });
+  router.delete(PROVIDER_BASE_PATH, async (req, res) => {
+    const { provider } = req.params;
+    const result = await manager.purgeAndResetProvider(provider);
+    res.json(result);
+  });
+  router.get(`${PROVIDER_BASE_PATH}/marks`, async (req, res) => {
+    const { provider } = req.params;
+    const record = await manager.getCurrentIngestionRecord(provider);
+    if (record) {
+      const id = record.id;
+      const records = await manager.getAllMarks(id);
+      res.json({ success: true, records });
+    } else {
+      const providers = await manager.listProviders();
+      if (providers.includes(provider)) {
+        logger.debug(`${provider} - Ingestion record found`);
+        res.json({
+          success: true,
+          message: "No records yet (provider is restarting)"
+        });
+      } else {
+        logger.error(
+          `${provider} - No ingestion record found in the database!`
+        );
+        res.status(404).json({
+          success: false,
+          status: {},
+          last_error: `Provider '${provider}' not found`
+        });
+      }
+    }
+  });
+  router.delete(`${PROVIDER_BASE_PATH}/marks`, async (req, res) => {
+    const { provider } = req.params;
+    const deletions = await manager.clearFinishedIngestions(provider);
+    res.json({
+      success: true,
+      message: `Expired marks for provider '${provider}' removed.`,
+      deletions
+    });
+  });
+  router.use(backendCommon.errorHandler());
+  return router;
+};
+
+var __accessCheck = (obj, member, msg) => {
+  if (!member.has(obj))
+    throw TypeError("Cannot " + msg);
+};
+var __privateGet = (obj, member, getter) => {
+  __accessCheck(obj, member, "read from private field");
+  return getter ? getter.call(obj) : member.get(obj);
+};
+var __privateAdd = (obj, member, value) => {
+  if (member.has(obj))
+    throw TypeError("Cannot add the same private member more than once");
+  member instanceof WeakSet ? member.add(obj) : member.set(obj, value);
+};
+var __privateSet = (obj, member, value, setter) => {
+  __accessCheck(obj, member, "write to private field");
+  setter ? setter.call(obj, value) : member.set(obj, value);
+  return value;
+};
+var _resolve, _reject, _a;
+class Deferred {
+  constructor() {
+    __privateAdd(this, _resolve, void 0);
+    __privateAdd(this, _reject, void 0);
+    this[_a] = "Deferred";
+    const promise = new Promise((resolve, reject) => {
+      __privateSet(this, _resolve, resolve);
+      __privateSet(this, _reject, reject);
+    });
+    this.then = promise.then.bind(promise);
+    this.catch = promise.catch.bind(promise);
+    this.finally = promise.finally.bind(promise);
+  }
+  get resolve() {
+    return __privateGet(this, _resolve);
+  }
+  get reject() {
+    return __privateGet(this, _reject);
+  }
+}
+_a = Symbol.toStringTag;
+_resolve = new WeakMap();
+_reject = new WeakMap();
+
+class WrapperProviders {
+  constructor(options) {
+    this.options = options;
+    this.numberOfProvidersToConnect = 0;
+    this.readySignal = new Deferred();
+  }
+  wrap(provider, options) {
+    this.numberOfProvidersToConnect += 1;
+    return {
+      getProviderName: () => provider.getProviderName(),
+      connect: async (connection) => {
+        await this.startProvider(provider, options, connection);
+        this.numberOfProvidersToConnect -= 1;
+        if (this.numberOfProvidersToConnect === 0) {
+          this.readySignal.resolve();
+        }
+      }
+    };
+  }
+  async adminRouter() {
+    return createIncrementalProviderRouter(
+      new IncrementalIngestionDatabaseManager({ client: this.options.client }),
+      backendPluginApi.loggerToWinstonLogger(this.options.logger)
+    );
+  }
+  async startProvider(provider, providerOptions, connection) {
+    const logger = backendPluginApi.loggerToWinstonLogger(
+      this.options.logger.child({
+        entityProvider: provider.getProviderName()
+      })
+    );
+    try {
+      if (!this.migrate) {
+        this.migrate = Promise.resolve().then(async () => {
+          var _a;
+          const apply = (_a = this.options.applyDatabaseMigrations) != null ? _a : applyDatabaseMigrations;
+          await apply(this.options.client);
+        });
+      }
+      await this.migrate;
+      const { burstInterval, burstLength, restLength } = providerOptions;
+      logger.info(`Connecting`);
+      const manager = new IncrementalIngestionDatabaseManager({
+        client: this.options.client
+      });
+      const engine = new IncrementalIngestionEngine({
+        ...providerOptions,
+        ready: this.readySignal,
+        manager,
+        logger,
+        provider,
+        restLength,
+        connection
+      });
+      const frequency = luxon.Duration.isDuration(burstInterval) ? burstInterval : luxon.Duration.fromObject(burstInterval);
+      const length = luxon.Duration.isDuration(burstLength) ? burstLength : luxon.Duration.fromObject(burstLength);
+      await this.options.scheduler.scheduleTask({
+        id: provider.getProviderName(),
+        fn: engine.taskFn.bind(engine),
+        frequency,
+        timeout: length
+      });
+    } catch (error) {
+      logger.warn(
+        `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${errors.stringifyError(
+          error
+        )}`
+      );
+      throw error;
+    }
+  }
+}
+
+const incrementalIngestionEntityProviderCatalogModule = backendPluginApi.createBackendModule({
+  pluginId: "catalog",
+  moduleId: "incrementalIngestionEntityProvider",
+  register(env, options) {
+    env.registerInit({
+      deps: {
+        catalog: pluginCatalogNode.catalogProcessingExtensionPoint,
+        config: backendPluginApi.configServiceRef,
+        database: backendPluginApi.databaseServiceRef,
+        httpRouter: backendPluginApi.httpRouterServiceRef,
+        logger: backendPluginApi.loggerServiceRef,
+        scheduler: backendPluginApi.schedulerServiceRef
+      },
+      async init({
+        catalog,
+        config,
+        database,
+        httpRouter,
+        logger,
+        scheduler
+      }) {
+        const client = await database.getClient();
+        const providers = new WrapperProviders({
+          config,
+          logger,
+          client,
+          scheduler
+        });
+        for (const entry of options.providers) {
+          const wrapped = providers.wrap(entry.provider, entry.options);
+          catalog.addEntityProvider(wrapped);
+        }
+        httpRouter.use(await providers.adminRouter());
+      }
+    });
+  }
+});
+
+class IncrementalCatalogBuilder {
+  constructor(env, builder, client, manager) {
+    this.env = env;
+    this.builder = builder;
+    this.client = client;
+    this.manager = manager;
+    this.ready = new Deferred();
+  }
+  static async create(env, builder) {
+    const client = await env.database.getClient();
+    const manager = new IncrementalIngestionDatabaseManager({ client });
+    return new IncrementalCatalogBuilder(env, builder, client, manager);
+  }
+  async build() {
+    await applyDatabaseMigrations(this.client);
+    this.ready.resolve();
+    const routerLogger = this.env.logger.child({
+      router: "IncrementalProviderAdmin"
+    });
+    const incrementalAdminRouter = await createIncrementalProviderRouter(
+      this.manager,
+      routerLogger
+    );
+    return { incrementalAdminRouter };
+  }
+  addIncrementalEntityProvider(provider, options) {
+    const { burstInterval, burstLength, restLength } = options;
+    const { logger: catalogLogger, scheduler } = this.env;
+    const ready = this.ready;
+    const manager = this.manager;
+    this.builder.addEntityProvider({
+      getProviderName: provider.getProviderName.bind(provider),
+      async connect(connection) {
+        const logger = catalogLogger.child({
+          entityProvider: provider.getProviderName()
+        });
+        logger.info(`Connecting`);
+        const engine = new IncrementalIngestionEngine({
+          ...options,
+          ready,
+          manager,
+          logger,
+          provider,
+          restLength,
+          connection
+        });
+        const frequency = luxon.Duration.isDuration(burstInterval) ? burstInterval : luxon.Duration.fromObject(burstInterval);
+        const length = luxon.Duration.isDuration(burstLength) ? burstLength : luxon.Duration.fromObject(burstLength);
+        await scheduler.scheduleTask({
+          id: provider.getProviderName(),
+          fn: engine.taskFn.bind(engine),
+          frequency,
+          timeout: length
+        });
+      }
+    });
+  }
+}
+
+exports.INCREMENTAL_ENTITY_PROVIDER_ANNOTATION = INCREMENTAL_ENTITY_PROVIDER_ANNOTATION;
+exports.IncrementalCatalogBuilder = IncrementalCatalogBuilder;
+exports.incrementalIngestionEntityProviderCatalogModule = incrementalIngestionEntityProviderCatalogModule;
+//# sourceMappingURL=index.cjs.js.map