@backstage/plugin-catalog-backend-module-incremental-ingestion 0.8.0-next.1 → 0.9.0-next.0

package/CHANGELOG.md

@@ -1,5 +1,48 @@
 # @backstage/plugin-catalog-backend-module-incremental-ingestion
 
+## 0.9.0-next.0
+
+### Minor Changes
+
+- e003123: Added a `source_key` column and a `UNIQUE(source_key, entity_ref)` constraint to the `ingestion_mark_entities` table, enabling a single native upsert instead of a select-then-update-or-insert sequence per ingestion mark.
+  This significantly reduces the number of database round trips during ingestion.
+
+  As part of this change, the table's `ref` column is renamed to `entity_ref` to standardize its naming with the rest of the table's columns.
+  This rename means the migration cannot be applied as part of a rolling, zero-downtime upgrade, which is fine since incremental ingestion providers are designed to run sequentially on a single Backstage backend instance.
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/plugin-catalog-backend@4.0.1-next.0
+  - @backstage/config@1.3.10-next.0
+  - @backstage/errors@1.3.2-next.0
+  - @backstage/backend-defaults@0.18.1-next.0
+  - @backstage/backend-plugin-api@1.10.2-next.0
+  - @backstage/plugin-permission-common@0.9.12-next.0
+  - @backstage/catalog-model@1.10.2-next.0
+  - @backstage/plugin-catalog-node@2.2.6-next.0
+  - @backstage/plugin-events-node@0.4.27-next.0
+  - @backstage/plugin-catalog-common@1.2.1-next.0
+
+## 0.8.0
+
+### Minor Changes
+
+- 305e2f7: **BREAKING**: The incremental ingestion administrative routes now enforce separate read and manage permissions. Installations with custom permission policies must add decisions for the new permissions.
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/backend-defaults@0.18.0
+  - @backstage/plugin-catalog-common@1.2.0
+  - @backstage/catalog-model@1.10.1
+  - @backstage/plugin-catalog-backend@4.0.0
+  - @backstage/config@1.3.9
+  - @backstage/backend-plugin-api@1.10.1
+  - @backstage/plugin-catalog-node@2.2.5
+  - @backstage/plugin-events-node@0.4.26
+  - @backstage/plugin-permission-common@0.9.11
+
 ## 0.8.0-next.1
 
 ### Minor Changes

package/dist/database/IncrementalIngestionDatabaseManager.cjs.js

@@ -112,26 +112,6 @@ class IncrementalIngestionDatabaseManager {
       };
     });
   }
-  /**
-   * Automatically cleans up duplicate ingestion records if they were accidentally created.
-   * Any ingestion record where the `rest_completed_at` is null (meaning it is active) AND
-   * the ingestionId is incorrect is a duplicate ingestion record.
-   * @param ingestionId - string
-   * @param provider - string
-   */
-  async clearDuplicateIngestions(ingestionId, provider) {
-    await this.client.transaction(async (tx) => {
-      const invalid = await tx("ingestions").where("provider_name", provider).andWhere("rest_completed_at", null).andWhereNot("id", ingestionId);
-      if (invalid.length > 0) {
-        await tx("ingestions").delete().whereIn("id", invalid);
-        await tx("ingestion_mark_entities").delete().whereIn(
-          "ingestion_mark_id",
-          tx("ingestion_marks").select("id").whereIn("ingestion_id", invalid)
-        );
-        await tx("ingestion_marks").delete().whereIn("ingestion_id", invalid);
-      }
-    });
-  }
   /**
    * This method fully purges and resets all ingestion records for the named provider, and
    * leaves it in a paused state.
@@ -174,12 +154,15 @@ class IncrementalIngestionDatabaseManager {
   }
   /**
    * This method is used to remove entity records from the ingestion_mark_entities
-   * table by their entity reference.
+   * table by their entity reference, scoped to a single source_key.
    */
-  async deleteEntityRecordsByRef(entities) {
+  async deleteEntityRecordsByRef(sourceKey, entities) {
+    if (entities.length === 0) {
+      return;
+    }
     const refs = entities.map((e) => e.entityRef);
     await this.client.transaction(async (tx) => {
-      await tx("ingestion_mark_entities").delete().modify(this.whereInArray("ref", refs));
+      await tx("ingestion_mark_entities").delete().where("source_key", sourceKey).modify(this.whereInArray("entity_ref", refs));
     });
   }
   /**
@@ -212,7 +195,7 @@ class IncrementalIngestionDatabaseManager {
   async computeRemoved(provider, ingestionId) {
     const previousIngestion = await this.getPreviousIngestionRecord(provider);
     return await this.client.transaction(async (tx) => {
-      const count = await tx("ingestion_mark_entities").count({ total: "ingestion_mark_entities.ref" }).join(
+      const count = await tx("ingestion_mark_entities").count({ total: "ingestion_mark_entities.entity_ref" }).join(
         "ingestion_marks",
         "ingestion_marks.id",
         "ingestion_mark_entities.ingestion_mark_id"
@@ -220,13 +203,15 @@ class IncrementalIngestionDatabaseManager {
       const total = count.reduce((acc, cur) => acc + Number(cur.total), 0);
       const removed = [];
       if (previousIngestion) {
-        const stale = await tx("ingestion_mark_entities").select("ingestion_mark_entities.ref").join(
+        const stale = await tx(
+          "ingestion_mark_entities"
+        ).select("ingestion_mark_entities.entity_ref").join(
           "ingestion_marks",
           "ingestion_marks.id",
           "ingestion_mark_entities.ingestion_mark_id"
         ).join("ingestions", "ingestions.id", "ingestion_marks.ingestion_id").where("ingestions.id", previousIngestion.id);
-        for (const entityRef of stale) {
-          removed.push({ entityRef: entityRef.ref });
+        for (const row of stale) {
+          removed.push({ entityRef: row.entity_ref });
         }
       }
       return { total, removed };
@@ -444,27 +429,24 @@ class IncrementalIngestionDatabaseManager {
   }
   /**
    * Performs an upsert to the `ingestion_mark_entities` table for all deferred entities.
-   * @param markId - string
+   * @param sourceKey - string
    * @param entities - DeferredEntity[]
+   * @param markId - string
    */
-  async createMarkEntities(markId, entities) {
+  async createMarkEntities(sourceKey, entities, markId) {
+    if (entities.length === 0) {
+      return;
+    }
     const refs = entities.map((e) => catalogModel.stringifyEntityRef(e.entity));
     await this.client.transaction(async (tx) => {
-      const existingRefsArray = (await tx("ingestion_mark_entities").select("ref").modify(this.whereInArray("ref", refs))).map((e) => e.ref);
-      const existingRefsSet = new Set(existingRefsArray);
-      const newRefs = refs.filter((e) => !existingRefsSet.has(e));
-      if (existingRefsArray.length > 0) {
-        await tx("ingestion_mark_entities").update("ingestion_mark_id", markId).modify(this.whereInArray("ref", existingRefsArray));
-      }
-      if (newRefs.length > 0) {
-        await tx("ingestion_mark_entities").insert(
-          newRefs.map((ref) => ({
-            id: node_crypto.randomUUID(),
-            ingestion_mark_id: markId,
-            ref
-          }))
-        );
-      }
+      await tx("ingestion_mark_entities").insert(
+        refs.map((entityRef) => ({
+          id: node_crypto.randomUUID(),
+          ingestion_mark_id: markId,
+          source_key: sourceKey,
+          entity_ref: entityRef
+        }))
+      ).onConflict(["source_key", "entity_ref"]).merge(["ingestion_mark_id"]);
     });
   }
   /**

package/dist/database/IncrementalIngestionDatabaseManager.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"IncrementalIngestionDatabaseManager.cjs.js","sources":["../../src/database/IncrementalIngestionDatabaseManager.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport type { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { stringifyEntityRef } from '@backstage/catalog-model';\nimport { Duration } from 'luxon';\nimport { randomUUID as v4 } from 'node:crypto';\nimport {\n  IngestionRecord,\n  IngestionRecordUpdate,\n  IngestionUpsert,\n  MarkRecord,\n  MarkRecordInsert,\n} from './tables';\n\nexport class IncrementalIngestionDatabaseManager {\n  private client: Knex;\n\n  constructor(options: { client: Knex }) {\n    this.client = options.client;\n  }\n\n  private whereInArray(column: string, values: string[]) {\n    const isPg = this.client.client.config.client === 'pg';\n    return (qb: Knex.QueryBuilder) => {\n      if (isPg) {\n        qb.whereRaw('?? = ANY(?)', [column, values]);\n      } else {\n        qb.whereIn(column, values);\n      }\n    };\n  }\n\n  /**\n   * Performs an update to the ingestion record with matching `id`.\n   * @param options - IngestionRecordUpdate\n   */\n  async updateIngestionRecordById(options: IngestionRecordUpdate) {\n    await this.client.transaction(async tx => {\n      const { ingestionId, update } = options;\n      await tx('ingestions').where('id', ingestionId).update(update);\n    });\n  }\n\n  /**\n   * Performs an update to the ingestion record with matching provider name. Will only update active records.\n   * @param provider - string\n   * @param update - Partial<IngestionUpsertIFace>\n   */\n  async updateIngestionRecordByProvider(\n    provider: string,\n    update: Partial<IngestionUpsert>,\n  ) {\n    await this.client.transaction(async tx => {\n      await tx('ingestions')\n        .where('provider_name', provider)\n        .andWhere('completion_ticket', 'open')\n        .update(update);\n    });\n  }\n\n  /**\n   * Performs an insert into the `ingestions` table with the supplied values.\n   * @param record - IngestionUpsertIFace\n   */\n  async insertIngestionRecord(record: IngestionUpsert) {\n    await this.client.transaction(async tx => {\n      await tx('ingestions').insert(record);\n    });\n  }\n\n  private async deleteMarkEntities(\n    tx: Knex.Transaction,\n    ids: { id: string }[],\n  ) {\n    if (ids.length === 0) {\n      return 0;\n    }\n\n    const allIds = ids.map(entry => entry.id);\n\n    if (this.client.client.config.client === 'pg') {\n      return await tx('ingestion_mark_entities')\n        .delete()\n        .modify(this.whereInArray('id', allIds));\n    }\n\n    let deleted = 0;\n    for (let i = 0; i < allIds.length; i += 100) {\n      const chunk = allIds.slice(i, i + 100);\n      deleted += await tx('ingestion_mark_entities')\n        .delete()\n        .whereIn('id', chunk);\n    }\n    return deleted;\n  }\n\n  /**\n   * Finds the current ingestion record for the named provider.\n   * @param provider - string\n   * @returns IngestionRecord | undefined\n   */\n  async getCurrentIngestionRecord(provider: string) {\n    return await this.client.transaction(async tx => {\n      const record = await tx<IngestionRecord>('ingestions')\n        .where('provider_name', provider)\n        .andWhere('completion_ticket', 'open')\n        .first();\n      return record;\n    });\n  }\n\n  /**\n   * Finds the last ingestion record for the named provider.\n   * @param provider - string\n   * @returns IngestionRecord | undefined\n   */\n  async getPreviousIngestionRecord(provider: string) {\n    return await this.client.transaction(async tx => {\n      return await tx<IngestionRecord>('ingestions')\n        .where('provider_name', provider)\n        .andWhereNot('completion_ticket', 'open')\n        .first();\n    });\n  }\n\n  /**\n   * Removes all entries from `ingestion_marks_entities`, `ingestion_marks`, and `ingestions`\n   * for prior ingestions that completed (i.e., have a `completion_ticket` value other than 'open').\n   * @param provider - string\n   * @returns A count of deletions for each record type.\n   */\n  async clearFinishedIngestions(provider: string) {\n    return await this.client.transaction(async tx => {\n      const markEntitiesDeleted = await tx('ingestion_mark_entities')\n        .delete()\n        .whereIn(\n          'ingestion_mark_id',\n          tx('ingestion_marks')\n            .select('id')\n            .whereIn(\n              'ingestion_id',\n              tx('ingestions')\n                .select('id')\n                .where('provider_name', provider)\n                .andWhereNot('completion_ticket', 'open'),\n            ),\n        );\n\n      const marksDeleted = await tx('ingestion_marks')\n        .delete()\n        .whereIn(\n          'ingestion_id',\n          tx('ingestions')\n            .select('id')\n            .where('provider_name', provider)\n            .andWhereNot('completion_ticket', 'open'),\n        );\n\n      const ingestionsDeleted = await tx('ingestions')\n        .delete()\n        .where('provider_name', provider)\n        .andWhereNot('completion_ticket', 'open');\n\n      return {\n        deletions: {\n          markEntitiesDeleted,\n          marksDeleted,\n          ingestionsDeleted,\n        },\n      };\n    });\n  }\n\n  /**\n   * Automatically cleans up duplicate ingestion records if they were accidentally created.\n   * Any ingestion record where the `rest_completed_at` is null (meaning it is active) AND\n   * the ingestionId is incorrect is a duplicate ingestion record.\n   * @param ingestionId - string\n   * @param provider - string\n   */\n  async clearDuplicateIngestions(ingestionId: string, provider: string) {\n    await this.client.transaction(async tx => {\n      const invalid = await tx<IngestionRecord>('ingestions')\n        .where('provider_name', provider)\n        .andWhere('rest_completed_at', null)\n        .andWhereNot('id', ingestionId);\n\n      if (invalid.length > 0) {\n        await tx('ingestions').delete().whereIn('id', invalid);\n        await tx('ingestion_mark_entities')\n          .delete()\n          .whereIn(\n            'ingestion_mark_id',\n            tx('ingestion_marks').select('id').whereIn('ingestion_id', invalid),\n          );\n        await tx('ingestion_marks').delete().whereIn('ingestion_id', invalid);\n      }\n    });\n  }\n\n  /**\n   * This method fully purges and resets all ingestion records for the named provider, and\n   * leaves it in a paused state.\n   * @param provider - string\n   * @returns Counts of all deleted ingestion records\n   */\n  async purgeAndResetProvider(provider: string) {\n    return await this.client.transaction(async tx => {\n      const ingestionIDs: { id: string }[] = await tx('ingestions')\n        .select('id')\n        .where('provider_name', provider);\n\n      const markIDs: { id: string }[] =\n        ingestionIDs.length > 0\n          ? await tx('ingestion_marks')\n              .select('id')\n              .whereIn(\n                'ingestion_id',\n                ingestionIDs.map(entry => entry.id),\n              )\n          : [];\n\n      const markEntityIDs: { id: string }[] =\n        markIDs.length > 0\n          ? await tx('ingestion_mark_entities')\n              .select('id')\n              .whereIn(\n                'ingestion_mark_id',\n                markIDs.map(entry => entry.id),\n              )\n          : [];\n\n      const markEntitiesDeleted = await this.deleteMarkEntities(\n        tx,\n        markEntityIDs,\n      );\n\n      const marksDeleted =\n        markIDs.length > 0\n          ? await tx('ingestion_marks')\n              .delete()\n              .whereIn(\n                'ingestion_id',\n                ingestionIDs.map(entry => entry.id),\n              )\n          : 0;\n\n      const ingestionsDeleted = await tx('ingestions')\n        .delete()\n        .where('provider_name', provider);\n\n      const next_action_at = new Date();\n      next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1000);\n\n      await this.insertIngestionRecord({\n        id: v4(),\n        next_action: 'rest',\n        provider_name: provider,\n        next_action_at,\n        ingestion_completed_at: new Date(),\n        status: 'resting',\n        completion_ticket: 'open',\n      });\n\n      return { provider, ingestionsDeleted, marksDeleted, markEntitiesDeleted };\n    });\n  }\n\n  /**\n   * This method is used to remove entity records from the ingestion_mark_entities\n   * table by their entity reference.\n   */\n  async deleteEntityRecordsByRef(entities: { entityRef: string }[]) {\n    const refs = entities.map(e => e.entityRef);\n    await this.client.transaction(async tx => {\n      await tx('ingestion_mark_entities')\n        .delete()\n        .modify(this.whereInArray('ref', refs));\n    });\n  }\n\n  /**\n   * Creates a new ingestion record.\n   * @param provider - string\n   * @returns A new ingestion record\n   */\n  async createProviderIngestionRecord(provider: string) {\n    const ingestionId = v4();\n    const nextAction = 'ingest';\n    try {\n      await this.insertIngestionRecord({\n        id: ingestionId,\n        next_action: nextAction,\n        provider_name: provider,\n        status: 'bursting',\n        completion_ticket: 'open',\n      });\n      return { ingestionId, nextAction, attempts: 0, nextActionAt: Date.now() };\n    } catch (_e) {\n      // Creating the ingestion record failed. Return undefined.\n      return undefined;\n    }\n  }\n\n  /**\n   * Computes which entities to remove, if any, at the end of a burst.\n   * @param provider - string\n   * @param ingestionId - string\n   * @returns All entities to remove for this burst.\n   */\n  async computeRemoved(provider: string, ingestionId: string) {\n    const previousIngestion = await this.getPreviousIngestionRecord(provider);\n    return await this.client.transaction(async tx => {\n      const count = await tx('ingestion_mark_entities')\n        .count({ total: 'ingestion_mark_entities.ref' })\n        .join(\n          'ingestion_marks',\n          'ingestion_marks.id',\n          'ingestion_mark_entities.ingestion_mark_id',\n        )\n        .join('ingestions', 'ingestions.id', 'ingestion_marks.ingestion_id')\n        .where('ingestions.id', ingestionId);\n\n      const total = count.reduce((acc, cur) => acc + Number(cur.total), 0);\n\n      const removed: { entityRef: string }[] = [];\n      if (previousIngestion) {\n        const stale: { ref: string }[] = await tx('ingestion_mark_entities')\n          .select('ingestion_mark_entities.ref')\n          .join(\n            'ingestion_marks',\n            'ingestion_marks.id',\n            'ingestion_mark_entities.ingestion_mark_id',\n          )\n          .join('ingestions', 'ingestions.id', 'ingestion_marks.ingestion_id')\n          .where('ingestions.id', previousIngestion.id);\n\n        for (const entityRef of stale) {\n          removed.push({ entityRef: entityRef.ref });\n        }\n      }\n\n      return { total, removed };\n    });\n  }\n\n  /**\n   * Performs a lookup of all providers that have duplicate active ingestion records.\n   * @returns An array of all duplicate active ingestions\n   */\n  async healthcheck() {\n    return await this.client.transaction(async tx => {\n      const records = await tx<{ id: string; provider_name: string }>(\n        'ingestions',\n      )\n        .distinct('id', 'provider_name')\n        .where('rest_completed_at', null);\n      return records;\n    });\n  }\n\n  /**\n   * Skips any wait time for the next action to run.\n   * @param provider - string\n   */\n  async triggerNextProviderAction(provider: string) {\n    await this.updateIngestionRecordByProvider(provider, {\n      next_action_at: new Date(),\n    });\n  }\n\n  /**\n   * Purges the following tables:\n   * * `ingestions`\n   * * `ingestion_marks`\n   * * `ingestion_mark_entities`\n   *\n   * This function leaves the ingestions table with all providers in a paused state.\n   * @returns Results from cleaning up all ingestion tables.\n   */\n  async cleanupProviders() {\n    const providers = await this.listProviders();\n\n    const ingestionsDeleted = await this.purgeTable('ingestions');\n\n    const next_action_at = new Date();\n    next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1000);\n\n    for (const provider of providers) {\n      await this.insertIngestionRecord({\n        id: v4(),\n        next_action: 'rest',\n        provider_name: provider,\n        next_action_at,\n        ingestion_completed_at: new Date(),\n        status: 'resting',\n        completion_ticket: 'open',\n      });\n    }\n\n    const ingestionMarksDeleted = await this.purgeTable('ingestion_marks');\n    const markEntitiesDeleted = await this.purgeTable(\n      'ingestion_mark_entities',\n    );\n\n    return { ingestionsDeleted, ingestionMarksDeleted, markEntitiesDeleted };\n  }\n\n  /**\n   * Configures the current ingestion record to ingest a burst.\n   * @param ingestionId - string\n   */\n  async setProviderIngesting(ingestionId: string) {\n    await this.updateIngestionRecordById({\n      ingestionId,\n      update: { next_action: 'ingest' },\n    });\n  }\n\n  /**\n   * Indicates the provider is currently ingesting a burst.\n   * @param ingestionId - string\n   */\n  async setProviderBursting(ingestionId: string) {\n    await this.updateIngestionRecordById({\n      ingestionId,\n      update: { status: 'bursting' },\n    });\n  }\n\n  /**\n   * Finalizes the current ingestion record to indicate that the post-ingestion rest period is complete.\n   * @param ingestionId - string\n   */\n  async setProviderComplete(ingestionId: string) {\n    await this.updateIngestionRecordById({\n      ingestionId,\n      update: {\n        next_action: 'nothing (done)',\n        rest_completed_at: new Date(),\n        status: 'complete',\n        completion_ticket: v4(),\n      },\n    });\n  }\n\n  /**\n   * Marks ingestion as complete and starts the post-ingestion rest cycle.\n   * @param ingestionId - string\n   * @param restLength - Duration\n   */\n  async setProviderResting(ingestionId: string, restLength: Duration) {\n    await this.updateIngestionRecordById({\n      ingestionId,\n      update: {\n        next_action: 'rest',\n        next_action_at: new Date(Date.now() + restLength.as('milliseconds')),\n        ingestion_completed_at: new Date(),\n        status: 'resting',\n      },\n    });\n  }\n\n  /**\n   * Marks ingestion as paused after a burst completes.\n   * @param ingestionId - string\n   */\n  async setProviderInterstitial(ingestionId: string) {\n    await this.updateIngestionRecordById({\n      ingestionId,\n      update: { attempts: 0, status: 'interstitial' },\n    });\n  }\n\n  /**\n   * Starts the cancel process for the current ingestion.\n   * @param ingestionId - string\n   * @param message - string (optional)\n   */\n  async setProviderCanceling(ingestionId: string, message?: string) {\n    const update: Partial<IngestionUpsert> = {\n      next_action: 'cancel',\n      last_error: message ? message : undefined,\n      next_action_at: new Date(),\n      status: 'canceling',\n    };\n    await this.updateIngestionRecordById({ ingestionId, update });\n  }\n\n  /**\n   * Completes the cancel process and triggers a new ingestion.\n   * @param ingestionId - string\n   */\n  async setProviderCanceled(ingestionId: string) {\n    await this.updateIngestionRecordById({\n      ingestionId,\n      update: {\n        next_action: 'nothing (canceled)',\n        rest_completed_at: new Date(),\n        status: 'complete',\n        completion_ticket: v4(),\n      },\n    });\n  }\n\n  /**\n   * Configures the current ingestion to wait and retry, due to a data source error.\n   * @param ingestionId - string\n   * @param attempts - number\n   * @param error - Error\n   * @param backoffLength - number\n   */\n  async setProviderBackoff(\n    ingestionId: string,\n    attempts: number,\n    error: Error,\n    backoffLength: number,\n  ) {\n    await this.updateIngestionRecordById({\n      ingestionId,\n      update: {\n        next_action: 'backoff',\n        attempts: attempts + 1,\n        last_error: String(error),\n        next_action_at: new Date(Date.now() + backoffLength),\n        status: 'backing off',\n      },\n    });\n  }\n\n  /**\n   * Returns the last record from `ingestion_marks` for the supplied ingestionId.\n   * @param ingestionId - string\n   * @returns MarkRecord | undefined\n   */\n  async getLastMark(ingestionId: string) {\n    return await this.client.transaction(async tx => {\n      const mark = await tx<MarkRecord>('ingestion_marks')\n        .where('ingestion_id', ingestionId)\n        .orderBy('sequence', 'desc')\n        .first();\n      return this.#decodeMark(this.client, mark);\n    });\n  }\n\n  /**\n   * Returns the first record from `ingestion_marks` for the supplied ingestionId.\n   * @param ingestionId - string\n   * @returns MarkRecord | undefined\n   */\n  async getFirstMark(ingestionId: string) {\n    return await this.client.transaction(async tx => {\n      const mark = await tx<MarkRecord>('ingestion_marks')\n        .where('ingestion_id', ingestionId)\n        .orderBy('sequence', 'asc')\n        .first();\n      return this.#decodeMark(this.client, mark);\n    });\n  }\n\n  async getAllMarks(ingestionId: string) {\n    return await this.client.transaction(async tx => {\n      const marks = await tx<MarkRecord>('ingestion_marks')\n        .where('ingestion_id', ingestionId)\n        .orderBy('sequence', 'desc');\n      return marks.map(m => this.#decodeMark(this.client, m));\n    });\n  }\n\n  /**\n   * Performs an insert into the `ingestion_marks` table with the supplied values.\n   * @param options - MarkRecordInsert\n   */\n  async createMark(options: MarkRecordInsert) {\n    const { record } = options;\n    await this.client.transaction(async tx => {\n      await tx('ingestion_marks').insert(record);\n    });\n  }\n\n  // Handles the fact that sqlite does not support json columns; they just\n  // persist the stringified data instead\n  #decodeMark<T extends MarkRecord | undefined>(knex: Knex, record: T): T {\n    if (record && knex.client.config.client.includes('sqlite3')) {\n      return {\n        ...record,\n        cursor: JSON.parse(record.cursor as string),\n      };\n    }\n    return record;\n  }\n\n  /**\n   * Performs an upsert to the `ingestion_mark_entities` table for all deferred entities.\n   * @param markId - string\n   * @param entities - DeferredEntity[]\n   */\n  async createMarkEntities(markId: string, entities: DeferredEntity[]) {\n    const refs = entities.map(e => stringifyEntityRef(e.entity));\n\n    await this.client.transaction(async tx => {\n      const existingRefsArray = (\n        await tx<{ ref: string }>('ingestion_mark_entities')\n          .select('ref')\n          .modify(this.whereInArray('ref', refs))\n      ).map((e: { ref: string }) => e.ref);\n\n      const existingRefsSet = new Set(existingRefsArray);\n\n      const newRefs = refs.filter(e => !existingRefsSet.has(e));\n\n      if (existingRefsArray.length > 0) {\n        await tx('ingestion_mark_entities')\n          .update('ingestion_mark_id', markId)\n          .modify(this.whereInArray('ref', existingRefsArray));\n      }\n\n      if (newRefs.length > 0) {\n        await tx('ingestion_mark_entities').insert(\n          newRefs.map(ref => ({\n            id: v4(),\n            ingestion_mark_id: markId,\n            ref,\n          })),\n        );\n      }\n    });\n  }\n\n  /**\n   * Deletes the entire content of a table, and returns the number of records deleted.\n   * @param table - string\n   * @returns number\n   */\n  async purgeTable(table: string) {\n    return await this.client.transaction(async tx => {\n      return await tx(table).delete();\n    });\n  }\n\n  /**\n   * Returns a list of all providers.\n   * @returns string[]\n   */\n  async listProviders() {\n    return await this.client.transaction(async tx => {\n      const providers = await tx<{ provider_name: string }>(\n        'ingestions',\n      ).distinct('provider_name');\n      return providers.map(entry => entry.provider_name);\n    });\n  }\n\n  async updateByName(provider: string, update: Partial<IngestionUpsert>) {\n    await this.updateIngestionRecordByProvider(provider, update);\n  }\n}\n"],"names":["v4","stringifyEntityRef"],"mappings":";;;;;AA6BO,MAAM,mCAAA,CAAoC;AAAA,EACvC,MAAA;AAAA,EAER,YAAY,OAAA,EAA2B;AACrC,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AAAA,EACxB;AAAA,EAEQ,YAAA,CAAa,QAAgB,MAAA,EAAkB;AACrD,IAAA,MAAM,IAAA,GAAO,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,OAAO,MAAA,KAAW,IAAA;AAClD,IAAA,OAAO,CAAC,EAAA,KAA0B;AAChC,MAAA,IAAI,IAAA,EAAM;AACR,QAAA,EAAA,CAAG,QAAA,CAAS,aAAA,EAAe,CAAC,MAAA,EAAQ,MAAM,CAAC,CAAA;AAAA,MAC7C,CAAA,MAAO;AACL,QAAA,EAAA,CAAG,OAAA,CAAQ,QAAQ,MAAM,CAAA;AAAA,MAC3B;AAAA,IACF,CAAA;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,0BAA0B,OAAA,EAAgC;AAC9D,IAAA,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AACxC,MAAA,MAAM,EAAE,WAAA,EAAa,MAAA,EAAO,GAAI,OAAA;AAChC,MAAA,MAAM,EAAA,CAAG,YAAY,CAAA,CAAE,KAAA,CAAM,MAAM,WAAW,CAAA,CAAE,OAAO,MAAM,CAAA;AAAA,IAC/D,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,+BAAA,CACJ,QAAA,EACA,MAAA,EACA;AACA,IAAA,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AACxC,MAAA,MAAM,EAAA,CAAG,YAAY,CAAA,CAClB,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA,CAC/B,QAAA,CAAS,mBAAA,EAAqB,MAAM,CAAA,CACpC,MAAA,CAAO,MAAM,CAAA;AAAA,IAClB,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,sBAAsB,MAAA,EAAyB;AACnD,IAAA,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AACxC,MAAA,MAAM,EAAA,CAAG,YAAY,CAAA,CAAE,MAAA,CAAO,MAAM,CAAA;AAAA,IACtC,CAAC,CAAA;AAAA,EACH;AAAA,EAEA,MAAc,kBAAA,CACZ,EAAA,EACA,GAAA,EACA;AACA,IAAA,IAAI,GAAA,CAAI,WAAW,CAAA,EAAG;AACpB,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,GAAA,CAAI,CAAA,KAAA,KAAS,MAAM,EAAE,CAAA;AAExC,IAAA,IAAI,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,WAAW,IAAA,EAAM;AAC7C,MAAA,OAAO,MAAM,EAAA,CAAG,yBAAyB,CAAA,CACtC,MAAA,EAAO,CACP,MAAA,CAAO,IAAA,CAAK,YAAA,CAAa,IAAA,EAAM,MAAM,CAAC,CAAA;AAAA,IAC3C;AAEA,IAAA,IAAI,OAAA,GAAU,CAAA;AACd,IAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,MAAA,CAAO,MAAA,EAAQ,KAAK,GAAA,EAAK;AAC3C,MAAA,MAAM,KAAA,GAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,IAAI,GAAG,CAAA;AACrC,MAAA,OAAA,IAAW,MAAM,GAAG,yBAAyB,CAAA,CAC1C,QAAO,CACP,OAAA,CAAQ,MAAM,KAAK,CAAA;AAAA,IACxB;AACA,IAAA,OAAO,OAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,0BAA0B,QAAA,EAAkB;AAChD,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,MAAA,GAAS,MAAM,EAAA,CAAoB,YAAY,CAAA,CAClD,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA,CAC/B,QAAA,CAAS,mBAAA,EAAqB,MAAM,EACpC,KAAA,EAAM;AACT,MAAA,OAAO,MAAA;AAAA,IACT,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,2BAA2B,QAAA,EAAkB;AACjD,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,OAAO,MAAM,EAAA,CAAoB,YAAY,CAAA,CAC1C,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA,CAC/B,WAAA,CAAY,mBAAA,EAAqB,MAAM,CAAA,CACvC,KAAA,EAAM;AAAA,IACX,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,wBAAwB,QAAA,EAAkB;AAC9C,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,sBAAsB,MAAM,EAAA,CAAG,yBAAyB,CAAA,CAC3D,QAAO,CACP,OAAA;AAAA,QACC,mBAAA;AAAA,QACA,EAAA,CAAG,iBAAiB,CAAA,CACjB,MAAA,CAAO,IAAI,CAAA,CACX,OAAA;AAAA,UACC,cAAA;AAAA,UACA,EAAA,CAAG,YAAY,CAAA,CACZ,MAAA,CAAO,IAAI,CAAA,CACX,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA,CAC/B,WAAA,CAAY,mBAAA,EAAqB,MAAM;AAAA;AAC5C,OACJ;AAEF,MAAA,MAAM,eAAe,MAAM,EAAA,CAAG,iBAAiB,CAAA,CAC5C,QAAO,CACP,OAAA;AAAA,QACC,cAAA;AAAA,QACA,EAAA,CAAG,YAAY,CAAA,CACZ,MAAA,CAAO,IAAI,CAAA,CACX,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA,CAC/B,WAAA,CAAY,mBAAA,EAAqB,MAAM;AAAA,OAC5C;AAEF,MAAA,MAAM,iBAAA,GAAoB,MAAM,EAAA,CAAG,YAAY,CAAA,CAC5C,MAAA,EAAO,CACP,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA,CAC/B,WAAA,CAAY,qBAAqB,MAAM,CAAA;AAE1C,MAAA,OAAO;AAAA,QACL,SAAA,EAAW;AAAA,UACT,mBAAA;AAAA,UACA,YAAA;AAAA,UACA;AAAA;AACF,OACF;AAAA,IACF,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAM,wBAAA,CAAyB,WAAA,EAAqB,QAAA,EAAkB;AACpE,IAAA,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AACxC,MAAA,MAAM,OAAA,GAAU,MAAM,EAAA,CAAoB,YAAY,EACnD,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA,CAC/B,SAAS,mBAAA,EAAqB,IAAI,CAAA,CAClC,WAAA,CAAY,MAAM,WAAW,CAAA;AAEhC,MAAA,IAAI,OAAA,CAAQ,SAAS,CAAA,EAAG;AACtB,QAAA,MAAM,GAAG,YAAY,CAAA,CAAE,QAAO,CAAE,OAAA,CAAQ,MAAM,OAAO,CAAA;AACrD,QAAA,MAAM,EAAA,CAAG,yBAAyB,CAAA,CAC/B,MAAA,EAAO,CACP,OAAA;AAAA,UACC,mBAAA;AAAA,UACA,EAAA,CAAG,iBAAiB,CAAA,CAAE,MAAA,CAAO,IAAI,CAAA,CAAE,OAAA,CAAQ,gBAAgB,OAAO;AAAA,SACpE;AACF,QAAA,MAAM,GAAG,iBAAiB,CAAA,CAAE,QAAO,CAAE,OAAA,CAAQ,gBAAgB,OAAO,CAAA;AAAA,MACtE;AAAA,IACF,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,sBAAsB,QAAA,EAAkB;AAC5C,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,YAAA,GAAiC,MAAM,EAAA,CAAG,YAAY,CAAA,CACzD,OAAO,IAAI,CAAA,CACX,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA;AAElC,MAAA,MAAM,OAAA,GACJ,YAAA,CAAa,MAAA,GAAS,CAAA,GAClB,MAAM,GAAG,iBAAiB,CAAA,CACvB,MAAA,CAAO,IAAI,CAAA,CACX,OAAA;AAAA,QACC,cAAA;AAAA,QACA,YAAA,CAAa,GAAA,CAAI,CAAA,KAAA,KAAS,KAAA,CAAM,EAAE;AAAA,UAEtC,EAAC;AAEP,MAAA,MAAM,aAAA,GACJ,OAAA,CAAQ,MAAA,GAAS,CAAA,GACb,MAAM,GAAG,yBAAyB,CAAA,CAC/B,MAAA,CAAO,IAAI,CAAA,CACX,OAAA;AAAA,QACC,mBAAA;AAAA,QACA,OAAA,CAAQ,GAAA,CAAI,CAAA,KAAA,KAAS,KAAA,CAAM,EAAE;AAAA,UAEjC,EAAC;AAEP,MAAA,MAAM,mBAAA,GAAsB,MAAM,IAAA,CAAK,kBAAA;AAAA,QACrC,EAAA;AAAA,QACA;AAAA,OACF;AAEA,MAAA,MAAM,YAAA,GACJ,QAAQ,MAAA,GAAS,CAAA,GACb,MAAM,EAAA,CAAG,iBAAiB,CAAA,CACvB,MAAA,EAAO,CACP,OAAA;AAAA,QACC,cAAA;AAAA,QACA,YAAA,CAAa,GAAA,CAAI,CAAA,KAAA,KAAS,KAAA,CAAM,EAAE;AAAA,OACpC,GACF,CAAA;AAEN,MAAA,MAAM,iBAAA,GAAoB,MAAM,EAAA,CAAG,YAAY,EAC5C,MAAA,EAAO,CACP,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA;AAElC,MAAA,MAAM,cAAA,uBAAqB,IAAA,EAAK;AAChC,MAAA,cAAA,CAAe,QAAQ,cAAA,CAAe,OAAA,KAAY,EAAA,GAAK,EAAA,GAAK,KAAK,GAAI,CAAA;AAErE,MAAA,MAAM,KAAK,qBAAA,CAAsB;AAAA,QAC/B,IAAIA,sBAAA,EAAG;AAAA,QACP,WAAA,EAAa,MAAA;AAAA,QACb,aAAA,EAAe,QAAA;AAAA,QACf,cAAA;AAAA,QACA,sBAAA,sBAA4B,IAAA,EAAK;AAAA,QACjC,MAAA,EAAQ,SAAA;AAAA,QACR,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAED,MAAA,OAAO,EAAE,QAAA,EAAU,iBAAA,EAAmB,YAAA,EAAc,mBAAA,EAAoB;AAAA,IAC1E,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,yBAAyB,QAAA,EAAmC;AAChE,IAAA,MAAM,IAAA,GAAO,QAAA,CAAS,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,SAAS,CAAA;AAC1C,IAAA,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AACxC,MAAA,MAAM,EAAA,CAAG,yBAAyB,CAAA,CAC/B,MAAA,EAAO,CACP,OAAO,IAAA,CAAK,YAAA,CAAa,KAAA,EAAO,IAAI,CAAC,CAAA;AAAA,IAC1C,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,8BAA8B,QAAA,EAAkB;AACpD,IAAA,MAAM,cAAcA,sBAAA,EAAG;AACvB,IAAA,MAAM,UAAA,GAAa,QAAA;AACnB,IAAA,IAAI;AACF,MAAA,MAAM,KAAK,qBAAA,CAAsB;AAAA,QAC/B,EAAA,EAAI,WAAA;AAAA,QACJ,WAAA,EAAa,UAAA;AAAA,QACb,aAAA,EAAe,QAAA;AAAA,QACf,MAAA,EAAQ,UAAA;AAAA,QACR,iBAAA,EAAmB;AAAA,OACpB,CAAA;AACD,MAAA,OAAO,EAAE,aAAa,UAAA,EAAY,QAAA,EAAU,GAAG,YAAA,EAAc,IAAA,CAAK,KAAI,EAAE;AAAA,IAC1E,SAAS,EAAA,EAAI;AAEX,MAAA,OAAO,MAAA;AAAA,IACT;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,cAAA,CAAe,QAAA,EAAkB,WAAA,EAAqB;AAC1D,IAAA,MAAM,iBAAA,GAAoB,MAAM,IAAA,CAAK,0BAAA,CAA2B,QAAQ,CAAA;AACxE,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,KAAA,GAAQ,MAAM,EAAA,CAAG,yBAAyB,CAAA,CAC7C,MAAM,EAAE,KAAA,EAAO,6BAAA,EAA+B,CAAA,CAC9C,IAAA;AAAA,QACC,iBAAA;AAAA,QACA,oBAAA;AAAA,QACA;AAAA,OACF,CACC,KAAK,YAAA,EAAc,eAAA,EAAiB,8BAA8B,CAAA,CAClE,KAAA,CAAM,iBAAiB,WAAW,CAAA;AAErC,MAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,MAAA,CAAO,CAAC,GAAA,EAAK,GAAA,KAAQ,GAAA,GAAM,MAAA,CAAO,GAAA,CAAI,KAAK,CAAA,EAAG,CAAC,CAAA;AAEnE,MAAA,MAAM,UAAmC,EAAC;AAC1C,MAAA,IAAI,iBAAA,EAAmB;AACrB,QAAA,MAAM,QAA2B,MAAM,EAAA,CAAG,yBAAyB,CAAA,CAChE,MAAA,CAAO,6BAA6B,CAAA,CACpC,IAAA;AAAA,UACC,iBAAA;AAAA,UACA,oBAAA;AAAA,UACA;AAAA,SACF,CACC,KAAK,YAAA,EAAc,eAAA,EAAiB,8BAA8B,CAAA,CAClE,KAAA,CAAM,eAAA,EAAiB,iBAAA,CAAkB,EAAE,CAAA;AAE9C,QAAA,KAAA,MAAW,aAAa,KAAA,EAAO;AAC7B,UAAA,OAAA,CAAQ,IAAA,CAAK,EAAE,SAAA,EAAW,SAAA,CAAU,KAAK,CAAA;AAAA,QAC3C;AAAA,MACF;AAEA,MAAA,OAAO,EAAE,OAAO,OAAA,EAAQ;AAAA,IAC1B,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,WAAA,GAAc;AAClB,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,UAAU,MAAM,EAAA;AAAA,QACpB;AAAA,QAEC,QAAA,CAAS,IAAA,EAAM,eAAe,CAAA,CAC9B,KAAA,CAAM,qBAAqB,IAAI,CAAA;AAClC,MAAA,OAAO,OAAA;AAAA,IACT,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,0BAA0B,QAAA,EAAkB;AAChD,IAAA,MAAM,IAAA,CAAK,gCAAgC,QAAA,EAAU;AAAA,MACnD,cAAA,sBAAoB,IAAA;AAAK,KAC1B,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,MAAM,gBAAA,GAAmB;AACvB,IAAA,MAAM,SAAA,GAAY,MAAM,IAAA,CAAK,aAAA,EAAc;AAE3C,IAAA,MAAM,iBAAA,GAAoB,MAAM,IAAA,CAAK,UAAA,CAAW,YAAY,CAAA;AAE5D,IAAA,MAAM,cAAA,uBAAqB,IAAA,EAAK;AAChC,IAAA,cAAA,CAAe,QAAQ,cAAA,CAAe,OAAA,KAAY,EAAA,GAAK,EAAA,GAAK,KAAK,GAAI,CAAA;AAErE,IAAA,KAAA,MAAW,YAAY,SAAA,EAAW;AAChC,MAAA,MAAM,KAAK,qBAAA,CAAsB;AAAA,QAC/B,IAAIA,sBAAA,EAAG;AAAA,QACP,WAAA,EAAa,MAAA;AAAA,QACb,aAAA,EAAe,QAAA;AAAA,QACf,cAAA;AAAA,QACA,sBAAA,sBAA4B,IAAA,EAAK;AAAA,QACjC,MAAA,EAAQ,SAAA;AAAA,QACR,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH;AAEA,IAAA,MAAM,qBAAA,GAAwB,MAAM,IAAA,CAAK,UAAA,CAAW,iBAAiB,CAAA;AACrE,IAAA,MAAM,mBAAA,GAAsB,MAAM,IAAA,CAAK,UAAA;AAAA,MACrC;AAAA,KACF;AAEA,IAAA,OAAO,EAAE,iBAAA,EAAmB,qBAAA,EAAuB,mBAAA,EAAoB;AAAA,EACzE;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,qBAAqB,WAAA,EAAqB;AAC9C,IAAA,MAAM,KAAK,yBAAA,CAA0B;AAAA,MACnC,WAAA;AAAA,MACA,MAAA,EAAQ,EAAE,WAAA,EAAa,QAAA;AAAS,KACjC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,oBAAoB,WAAA,EAAqB;AAC7C,IAAA,MAAM,KAAK,yBAAA,CAA0B;AAAA,MACnC,WAAA;AAAA,MACA,MAAA,EAAQ,EAAE,MAAA,EAAQ,UAAA;AAAW,KAC9B,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,oBAAoB,WAAA,EAAqB;AAC7C,IAAA,MAAM,KAAK,yBAAA,CAA0B;AAAA,MACnC,WAAA;AAAA,MACA,MAAA,EAAQ;AAAA,QACN,WAAA,EAAa,gBAAA;AAAA,QACb,iBAAA,sBAAuB,IAAA,EAAK;AAAA,QAC5B,MAAA,EAAQ,UAAA;AAAA,QACR,mBAAmBA,sBAAA;AAAG;AACxB,KACD,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,kBAAA,CAAmB,WAAA,EAAqB,UAAA,EAAsB;AAClE,IAAA,MAAM,KAAK,yBAAA,CAA0B;AAAA,MACnC,WAAA;AAAA,MACA,MAAA,EAAQ;AAAA,QACN,WAAA,EAAa,MAAA;AAAA,QACb,cAAA,EAAgB,IAAI,IAAA,CAAK,IAAA,CAAK,KAAI,GAAI,UAAA,CAAW,EAAA,CAAG,cAAc,CAAC,CAAA;AAAA,QACnE,sBAAA,sBAA4B,IAAA,EAAK;AAAA,QACjC,MAAA,EAAQ;AAAA;AACV,KACD,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,wBAAwB,WAAA,EAAqB;AACjD,IAAA,MAAM,KAAK,yBAAA,CAA0B;AAAA,MACnC,WAAA;AAAA,MACA,MAAA,EAAQ,EAAE,QAAA,EAAU,CAAA,EAAG,QAAQ,cAAA;AAAe,KAC/C,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,oBAAA,CAAqB,WAAA,EAAqB,OAAA,EAAkB;AAChE,IAAA,MAAM,MAAA,GAAmC;AAAA,MACvC,WAAA,EAAa,QAAA;AAAA,MACb,UAAA,EAAY,UAAU,OAAA,GAAU,MAAA;AAAA,MAChC,cAAA,sBAAoB,IAAA,EAAK;AAAA,MACzB,MAAA,EAAQ;AAAA,KACV;AACA,IAAA,MAAM,IAAA,CAAK,yBAAA,CAA0B,EAAE,WAAA,EAAa,QAAQ,CAAA;AAAA,EAC9D;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,oBAAoB,WAAA,EAAqB;AAC7C,IAAA,MAAM,KAAK,yBAAA,CAA0B;AAAA,MACnC,WAAA;AAAA,MACA,MAAA,EAAQ;AAAA,QACN,WAAA,EAAa,oBAAA;AAAA,QACb,iBAAA,sBAAuB,IAAA,EAAK;AAAA,QAC5B,MAAA,EAAQ,UAAA;AAAA,QACR,mBAAmBA,sBAAA;AAAG;AACxB,KACD,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAM,kBAAA,CACJ,WAAA,EACA,QAAA,EACA,OACA,aAAA,EACA;AACA,IAAA,MAAM,KAAK,yBAAA,CAA0B;AAAA,MACnC,WAAA;AAAA,MACA,MAAA,EAAQ;AAAA,QACN,WAAA,EAAa,SAAA;AAAA,QACb,UAAU,QAAA,GAAW,CAAA;AAAA,QACrB,UAAA,EAAY,OAAO,KAAK,CAAA;AAAA,QACxB,gBAAgB,IAAI,IAAA,CAAK,IAAA,CAAK,GAAA,KAAQ,aAAa,CAAA;AAAA,QACnD,MAAA,EAAQ;AAAA;AACV,KACD,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,YAAY,WAAA,EAAqB;AACrC,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,IAAA,GAAO,MAAM,EAAA,CAAe,iBAAiB,CAAA,CAChD,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,OAAA,CAAQ,UAAA,EAAY,MAAM,EAC1B,KAAA,EAAM;AACT,MAAA,OAAO,IAAA,CAAK,WAAA,CAAY,IAAA,CAAK,MAAA,EAAQ,IAAI,CAAA;AAAA,IAC3C,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,aAAa,WAAA,EAAqB;AACtC,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,IAAA,GAAO,MAAM,EAAA,CAAe,iBAAiB,CAAA,CAChD,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,OAAA,CAAQ,UAAA,EAAY,KAAK,EACzB,KAAA,EAAM;AACT,MAAA,OAAO,IAAA,CAAK,WAAA,CAAY,IAAA,CAAK,MAAA,EAAQ,IAAI,CAAA;AAAA,IAC3C,CAAC,CAAA;AAAA,EACH;AAAA,EAEA,MAAM,YAAY,WAAA,EAAqB;AACrC,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,KAAA,GAAQ,MAAM,EAAA,CAAe,iBAAiB,CAAA,CACjD,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,OAAA,CAAQ,UAAA,EAAY,MAAM,CAAA;AAC7B,MAAA,OAAO,KAAA,CAAM,IAAI,CAAA,CAAA,KAAK,IAAA,CAAK,YAAY,IAAA,CAAK,MAAA,EAAQ,CAAC,CAAC,CAAA;AAAA,IACxD,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,WAAW,OAAA,EAA2B;AAC1C,IAAA,MAAM,EAAE,QAAO,GAAI,OAAA;AACnB,IAAA,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AACxC,MAAA,MAAM,EAAA,CAAG,iBAAiB,CAAA,CAAE,MAAA,CAAO,MAAM,CAAA;AAAA,IAC3C,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA,EAIA,WAAA,CAA8C,MAAY,MAAA,EAAc;AACtE,IAAA,IAAI,UAAU,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AAC3D,MAAA,OAAO;AAAA,QACL,GAAG,MAAA;AAAA,QACH,MAAA,EAAQ,IAAA,CAAK,KAAA,CAAM,MAAA,CAAO,MAAgB;AAAA,OAC5C;AAAA,IACF;AACA,IAAA,OAAO,MAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,kBAAA,CAAmB,MAAA,EAAgB,QAAA,EAA4B;AACnE,IAAA,MAAM,OAAO,QAAA,CAAS,GAAA,CAAI,OAAKC,+BAAA,CAAmB,CAAA,CAAE,MAAM,CAAC,CAAA;AAE3D,IAAA,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AACxC,MAAA,MAAM,qBACJ,MAAM,EAAA,CAAoB,yBAAyB,CAAA,CAChD,MAAA,CAAO,KAAK,CAAA,CACZ,MAAA,CAAO,KAAK,YAAA,CAAa,KAAA,EAAO,IAAI,CAAC,CAAA,EACxC,IAAI,CAAC,CAAA,KAAuB,EAAE,GAAG,CAAA;AAEnC,MAAA,MAAM,eAAA,GAAkB,IAAI,GAAA,CAAI,iBAAiB,CAAA;AAEjD,MAAA,MAAM,OAAA,GAAU,KAAK,MAAA,CAAO,CAAA,CAAA,KAAK,CAAC,eAAA,CAAgB,GAAA,CAAI,CAAC,CAAC,CAAA;AAExD,MAAA,IAAI,iBAAA,CAAkB,SAAS,CAAA,EAAG;AAChC,QAAA,MAAM,EAAA,CAAG,yBAAyB,CAAA,CAC/B,MAAA,CAAO,mBAAA,EAAqB,MAAM,CAAA,CAClC,MAAA,CAAO,IAAA,CAAK,YAAA,CAAa,KAAA,EAAO,iBAAiB,CAAC,CAAA;AAAA,MACvD;AAEA,MAAA,IAAI,OAAA,CAAQ,SAAS,CAAA,EAAG;AACtB,QAAA,MAAM,EAAA,CAAG,yBAAyB,CAAA,CAAE,MAAA;AAAA,UAClC,OAAA,CAAQ,IAAI,CAAA,GAAA,MAAQ;AAAA,YAClB,IAAID,sBAAA,EAAG;AAAA,YACP,iBAAA,EAAmB,MAAA;AAAA,YACnB;AAAA,WACF,CAAE;AAAA,SACJ;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,WAAW,KAAA,EAAe;AAC9B,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,OAAO,MAAM,EAAA,CAAG,KAAK,CAAA,CAAE,MAAA,EAAO;AAAA,IAChC,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,aAAA,GAAgB;AACpB,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,YAAY,MAAM,EAAA;AAAA,QACtB;AAAA,OACF,CAAE,SAAS,eAAe,CAAA;AAC1B,MAAA,OAAO,SAAA,CAAU,GAAA,CAAI,CAAA,KAAA,KAAS,KAAA,CAAM,aAAa,CAAA;AAAA,IACnD,CAAC,CAAA;AAAA,EACH;AAAA,EAEA,MAAM,YAAA,CAAa,QAAA,EAAkB,MAAA,EAAkC;AACrE,IAAA,MAAM,IAAA,CAAK,+BAAA,CAAgC,QAAA,EAAU,MAAM,CAAA;AAAA,EAC7D;AACF;;;;"}
+{"version":3,"file":"IncrementalIngestionDatabaseManager.cjs.js","sources":["../../src/database/IncrementalIngestionDatabaseManager.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Knex } from 'knex';\nimport type { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { stringifyEntityRef } from '@backstage/catalog-model';\nimport { Duration } from 'luxon';\nimport { randomUUID as v4 } from 'node:crypto';\nimport {\n  IngestionRecord,\n  IngestionRecordUpdate,\n  IngestionUpsert,\n  MarkRecord,\n  MarkRecordInsert,\n} from './tables';\n\nexport class IncrementalIngestionDatabaseManager {\n  private client: Knex;\n\n  constructor(options: { client: Knex }) {\n    this.client = options.client;\n  }\n\n  private whereInArray(column: string, values: string[]) {\n    const isPg = this.client.client.config.client === 'pg';\n    return (qb: Knex.QueryBuilder) => {\n      if (isPg) {\n        qb.whereRaw('?? = ANY(?)', [column, values]);\n      } else {\n        qb.whereIn(column, values);\n      }\n    };\n  }\n\n  /**\n   * Performs an update to the ingestion record with matching `id`.\n   * @param options - IngestionRecordUpdate\n   */\n  async updateIngestionRecordById(options: IngestionRecordUpdate) {\n    await this.client.transaction(async tx => {\n      const { ingestionId, update } = options;\n      await tx('ingestions').where('id', ingestionId).update(update);\n    });\n  }\n\n  /**\n   * Performs an update to the ingestion record with matching provider name. Will only update active records.\n   * @param provider - string\n   * @param update - Partial<IngestionUpsertIFace>\n   */\n  async updateIngestionRecordByProvider(\n    provider: string,\n    update: Partial<IngestionUpsert>,\n  ) {\n    await this.client.transaction(async tx => {\n      await tx('ingestions')\n        .where('provider_name', provider)\n        .andWhere('completion_ticket', 'open')\n        .update(update);\n    });\n  }\n\n  /**\n   * Performs an insert into the `ingestions` table with the supplied values.\n   * @param record - IngestionUpsertIFace\n   */\n  async insertIngestionRecord(record: IngestionUpsert) {\n    await this.client.transaction(async tx => {\n      await tx('ingestions').insert(record);\n    });\n  }\n\n  private async deleteMarkEntities(\n    tx: Knex.Transaction,\n    ids: { id: string }[],\n  ) {\n    if (ids.length === 0) {\n      return 0;\n    }\n\n    const allIds = ids.map(entry => entry.id);\n\n    if (this.client.client.config.client === 'pg') {\n      return await tx('ingestion_mark_entities')\n        .delete()\n        .modify(this.whereInArray('id', allIds));\n    }\n\n    let deleted = 0;\n    for (let i = 0; i < allIds.length; i += 100) {\n      const chunk = allIds.slice(i, i + 100);\n      deleted += await tx('ingestion_mark_entities')\n        .delete()\n        .whereIn('id', chunk);\n    }\n    return deleted;\n  }\n\n  /**\n   * Finds the current ingestion record for the named provider.\n   * @param provider - string\n   * @returns IngestionRecord | undefined\n   */\n  async getCurrentIngestionRecord(provider: string) {\n    return await this.client.transaction(async tx => {\n      const record = await tx<IngestionRecord>('ingestions')\n        .where('provider_name', provider)\n        .andWhere('completion_ticket', 'open')\n        .first();\n      return record;\n    });\n  }\n\n  /**\n   * Finds the last ingestion record for the named provider.\n   * @param provider - string\n   * @returns IngestionRecord | undefined\n   */\n  async getPreviousIngestionRecord(provider: string) {\n    return await this.client.transaction(async tx => {\n      return await tx<IngestionRecord>('ingestions')\n        .where('provider_name', provider)\n        .andWhereNot('completion_ticket', 'open')\n        .first();\n    });\n  }\n\n  /**\n   * Removes all entries from `ingestion_marks_entities`, `ingestion_marks`, and `ingestions`\n   * for prior ingestions that completed (i.e., have a `completion_ticket` value other than 'open').\n   * @param provider - string\n   * @returns A count of deletions for each record type.\n   */\n  async clearFinishedIngestions(provider: string) {\n    return await this.client.transaction(async tx => {\n      const markEntitiesDeleted = await tx('ingestion_mark_entities')\n        .delete()\n        .whereIn(\n          'ingestion_mark_id',\n          tx('ingestion_marks')\n            .select('id')\n            .whereIn(\n              'ingestion_id',\n              tx('ingestions')\n                .select('id')\n                .where('provider_name', provider)\n                .andWhereNot('completion_ticket', 'open'),\n            ),\n        );\n\n      const marksDeleted = await tx('ingestion_marks')\n        .delete()\n        .whereIn(\n          'ingestion_id',\n          tx('ingestions')\n            .select('id')\n            .where('provider_name', provider)\n            .andWhereNot('completion_ticket', 'open'),\n        );\n\n      const ingestionsDeleted = await tx('ingestions')\n        .delete()\n        .where('provider_name', provider)\n        .andWhereNot('completion_ticket', 'open');\n\n      return {\n        deletions: {\n          markEntitiesDeleted,\n          marksDeleted,\n          ingestionsDeleted,\n        },\n      };\n    });\n  }\n\n  /**\n   * This method fully purges and resets all ingestion records for the named provider, and\n   * leaves it in a paused state.\n   * @param provider - string\n   * @returns Counts of all deleted ingestion records\n   */\n  async purgeAndResetProvider(provider: string) {\n    return await this.client.transaction(async tx => {\n      const ingestionIDs: { id: string }[] = await tx('ingestions')\n        .select('id')\n        .where('provider_name', provider);\n\n      const markIDs: { id: string }[] =\n        ingestionIDs.length > 0\n          ? await tx('ingestion_marks')\n              .select('id')\n              .whereIn(\n                'ingestion_id',\n                ingestionIDs.map(entry => entry.id),\n              )\n          : [];\n\n      const markEntityIDs: { id: string }[] =\n        markIDs.length > 0\n          ? await tx('ingestion_mark_entities')\n              .select('id')\n              .whereIn(\n                'ingestion_mark_id',\n                markIDs.map(entry => entry.id),\n              )\n          : [];\n\n      const markEntitiesDeleted = await this.deleteMarkEntities(\n        tx,\n        markEntityIDs,\n      );\n\n      const marksDeleted =\n        markIDs.length > 0\n          ? await tx('ingestion_marks')\n              .delete()\n              .whereIn(\n                'ingestion_id',\n                ingestionIDs.map(entry => entry.id),\n              )\n          : 0;\n\n      const ingestionsDeleted = await tx('ingestions')\n        .delete()\n        .where('provider_name', provider);\n\n      const next_action_at = new Date();\n      next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1000);\n\n      await this.insertIngestionRecord({\n        id: v4(),\n        next_action: 'rest',\n        provider_name: provider,\n        next_action_at,\n        ingestion_completed_at: new Date(),\n        status: 'resting',\n        completion_ticket: 'open',\n      });\n\n      return { provider, ingestionsDeleted, marksDeleted, markEntitiesDeleted };\n    });\n  }\n\n  /**\n   * This method is used to remove entity records from the ingestion_mark_entities\n   * table by their entity reference, scoped to a single source_key.\n   */\n  async deleteEntityRecordsByRef(\n    sourceKey: string,\n    entities: { entityRef: string }[],\n  ) {\n    if (entities.length === 0) {\n      return;\n    }\n\n    const refs = entities.map(e => e.entityRef);\n    await this.client.transaction(async tx => {\n      await tx('ingestion_mark_entities')\n        .delete()\n        .where('source_key', sourceKey)\n        .modify(this.whereInArray('entity_ref', refs));\n    });\n  }\n\n  /**\n   * Creates a new ingestion record.\n   * @param provider - string\n   * @returns A new ingestion record\n   */\n  async createProviderIngestionRecord(provider: string) {\n    const ingestionId = v4();\n    const nextAction = 'ingest';\n    try {\n      await this.insertIngestionRecord({\n        id: ingestionId,\n        next_action: nextAction,\n        provider_name: provider,\n        status: 'bursting',\n        completion_ticket: 'open',\n      });\n      return { ingestionId, nextAction, attempts: 0, nextActionAt: Date.now() };\n    } catch (_e) {\n      // Creating the ingestion record failed. Return undefined.\n      return undefined;\n    }\n  }\n\n  /**\n   * Computes which entities to remove, if any, at the end of a burst.\n   * @param provider - string\n   * @param ingestionId - string\n   * @returns All entities to remove for this burst.\n   */\n  async computeRemoved(provider: string, ingestionId: string) {\n    const previousIngestion = await this.getPreviousIngestionRecord(provider);\n    return await this.client.transaction(async tx => {\n      const count = await tx('ingestion_mark_entities')\n        .count({ total: 'ingestion_mark_entities.entity_ref' })\n        .join(\n          'ingestion_marks',\n          'ingestion_marks.id',\n          'ingestion_mark_entities.ingestion_mark_id',\n        )\n        .join('ingestions', 'ingestions.id', 'ingestion_marks.ingestion_id')\n        .where('ingestions.id', ingestionId);\n\n      const total = count.reduce((acc, cur) => acc + Number(cur.total), 0);\n\n      const removed: { entityRef: string }[] = [];\n      if (previousIngestion) {\n        const stale: { entity_ref: string }[] = await tx(\n          'ingestion_mark_entities',\n        )\n          .select('ingestion_mark_entities.entity_ref')\n          .join(\n            'ingestion_marks',\n            'ingestion_marks.id',\n            'ingestion_mark_entities.ingestion_mark_id',\n          )\n          .join('ingestions', 'ingestions.id', 'ingestion_marks.ingestion_id')\n          .where('ingestions.id', previousIngestion.id);\n\n        for (const row of stale) {\n          removed.push({ entityRef: row.entity_ref });\n        }\n      }\n\n      return { total, removed };\n    });\n  }\n\n  /**\n   * Performs a lookup of all providers that have duplicate active ingestion records.\n   * @returns An array of all duplicate active ingestions\n   */\n  async healthcheck() {\n    return await this.client.transaction(async tx => {\n      const records = await tx<{ id: string; provider_name: string }>(\n        'ingestions',\n      )\n        .distinct('id', 'provider_name')\n        .where('rest_completed_at', null);\n      return records;\n    });\n  }\n\n  /**\n   * Skips any wait time for the next action to run.\n   * @param provider - string\n   */\n  async triggerNextProviderAction(provider: string) {\n    await this.updateIngestionRecordByProvider(provider, {\n      next_action_at: new Date(),\n    });\n  }\n\n  /**\n   * Purges the following tables:\n   * * `ingestions`\n   * * `ingestion_marks`\n   * * `ingestion_mark_entities`\n   *\n   * This function leaves the ingestions table with all providers in a paused state.\n   * @returns Results from cleaning up all ingestion tables.\n   */\n  async cleanupProviders() {\n    const providers = await this.listProviders();\n\n    const ingestionsDeleted = await this.purgeTable('ingestions');\n\n    const next_action_at = new Date();\n    next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1000);\n\n    for (const provider of providers) {\n      await this.insertIngestionRecord({\n        id: v4(),\n        next_action: 'rest',\n        provider_name: provider,\n        next_action_at,\n        ingestion_completed_at: new Date(),\n        status: 'resting',\n        completion_ticket: 'open',\n      });\n    }\n\n    const ingestionMarksDeleted = await this.purgeTable('ingestion_marks');\n    const markEntitiesDeleted = await this.purgeTable(\n      'ingestion_mark_entities',\n    );\n\n    return { ingestionsDeleted, ingestionMarksDeleted, markEntitiesDeleted };\n  }\n\n  /**\n   * Configures the current ingestion record to ingest a burst.\n   * @param ingestionId - string\n   */\n  async setProviderIngesting(ingestionId: string) {\n    await this.updateIngestionRecordById({\n      ingestionId,\n      update: { next_action: 'ingest' },\n    });\n  }\n\n  /**\n   * Indicates the provider is currently ingesting a burst.\n   * @param ingestionId - string\n   */\n  async setProviderBursting(ingestionId: string) {\n    await this.updateIngestionRecordById({\n      ingestionId,\n      update: { status: 'bursting' },\n    });\n  }\n\n  /**\n   * Finalizes the current ingestion record to indicate that the post-ingestion rest period is complete.\n   * @param ingestionId - string\n   */\n  async setProviderComplete(ingestionId: string) {\n    await this.updateIngestionRecordById({\n      ingestionId,\n      update: {\n        next_action: 'nothing (done)',\n        rest_completed_at: new Date(),\n        status: 'complete',\n        completion_ticket: v4(),\n      },\n    });\n  }\n\n  /**\n   * Marks ingestion as complete and starts the post-ingestion rest cycle.\n   * @param ingestionId - string\n   * @param restLength - Duration\n   */\n  async setProviderResting(ingestionId: string, restLength: Duration) {\n    await this.updateIngestionRecordById({\n      ingestionId,\n      update: {\n        next_action: 'rest',\n        next_action_at: new Date(Date.now() + restLength.as('milliseconds')),\n        ingestion_completed_at: new Date(),\n        status: 'resting',\n      },\n    });\n  }\n\n  /**\n   * Marks ingestion as paused after a burst completes.\n   * @param ingestionId - string\n   */\n  async setProviderInterstitial(ingestionId: string) {\n    await this.updateIngestionRecordById({\n      ingestionId,\n      update: { attempts: 0, status: 'interstitial' },\n    });\n  }\n\n  /**\n   * Starts the cancel process for the current ingestion.\n   * @param ingestionId - string\n   * @param message - string (optional)\n   */\n  async setProviderCanceling(ingestionId: string, message?: string) {\n    const update: Partial<IngestionUpsert> = {\n      next_action: 'cancel',\n      last_error: message ? message : undefined,\n      next_action_at: new Date(),\n      status: 'canceling',\n    };\n    await this.updateIngestionRecordById({ ingestionId, update });\n  }\n\n  /**\n   * Completes the cancel process and triggers a new ingestion.\n   * @param ingestionId - string\n   */\n  async setProviderCanceled(ingestionId: string) {\n    await this.updateIngestionRecordById({\n      ingestionId,\n      update: {\n        next_action: 'nothing (canceled)',\n        rest_completed_at: new Date(),\n        status: 'complete',\n        completion_ticket: v4(),\n      },\n    });\n  }\n\n  /**\n   * Configures the current ingestion to wait and retry, due to a data source error.\n   * @param ingestionId - string\n   * @param attempts - number\n   * @param error - Error\n   * @param backoffLength - number\n   */\n  async setProviderBackoff(\n    ingestionId: string,\n    attempts: number,\n    error: Error,\n    backoffLength: number,\n  ) {\n    await this.updateIngestionRecordById({\n      ingestionId,\n      update: {\n        next_action: 'backoff',\n        attempts: attempts + 1,\n        last_error: String(error),\n        next_action_at: new Date(Date.now() + backoffLength),\n        status: 'backing off',\n      },\n    });\n  }\n\n  /**\n   * Returns the last record from `ingestion_marks` for the supplied ingestionId.\n   * @param ingestionId - string\n   * @returns MarkRecord | undefined\n   */\n  async getLastMark(ingestionId: string) {\n    return await this.client.transaction(async tx => {\n      const mark = await tx<MarkRecord>('ingestion_marks')\n        .where('ingestion_id', ingestionId)\n        .orderBy('sequence', 'desc')\n        .first();\n      return this.#decodeMark(this.client, mark);\n    });\n  }\n\n  /**\n   * Returns the first record from `ingestion_marks` for the supplied ingestionId.\n   * @param ingestionId - string\n   * @returns MarkRecord | undefined\n   */\n  async getFirstMark(ingestionId: string) {\n    return await this.client.transaction(async tx => {\n      const mark = await tx<MarkRecord>('ingestion_marks')\n        .where('ingestion_id', ingestionId)\n        .orderBy('sequence', 'asc')\n        .first();\n      return this.#decodeMark(this.client, mark);\n    });\n  }\n\n  async getAllMarks(ingestionId: string) {\n    return await this.client.transaction(async tx => {\n      const marks = await tx<MarkRecord>('ingestion_marks')\n        .where('ingestion_id', ingestionId)\n        .orderBy('sequence', 'desc');\n      return marks.map(m => this.#decodeMark(this.client, m));\n    });\n  }\n\n  /**\n   * Performs an insert into the `ingestion_marks` table with the supplied values.\n   * @param options - MarkRecordInsert\n   */\n  async createMark(options: MarkRecordInsert) {\n    const { record } = options;\n    await this.client.transaction(async tx => {\n      await tx('ingestion_marks').insert(record);\n    });\n  }\n\n  // Handles the fact that sqlite does not support json columns; they just\n  // persist the stringified data instead\n  #decodeMark<T extends MarkRecord | undefined>(knex: Knex, record: T): T {\n    if (record && knex.client.config.client.includes('sqlite3')) {\n      return {\n        ...record,\n        cursor: JSON.parse(record.cursor as string),\n      };\n    }\n    return record;\n  }\n\n  /**\n   * Performs an upsert to the `ingestion_mark_entities` table for all deferred entities.\n   * @param sourceKey - string\n   * @param entities - DeferredEntity[]\n   * @param markId - string\n   */\n  async createMarkEntities(\n    sourceKey: string,\n    entities: DeferredEntity[],\n    markId: string,\n  ) {\n    if (entities.length === 0) {\n      return;\n    }\n\n    const refs = entities.map(e => stringifyEntityRef(e.entity));\n\n    await this.client.transaction(async tx => {\n      await tx('ingestion_mark_entities')\n        .insert(\n          refs.map(entityRef => ({\n            id: v4(),\n            ingestion_mark_id: markId,\n            source_key: sourceKey,\n            entity_ref: entityRef,\n          })),\n        )\n        .onConflict(['source_key', 'entity_ref'])\n        .merge(['ingestion_mark_id']);\n    });\n  }\n\n  /**\n   * Deletes the entire content of a table, and returns the number of records deleted.\n   * @param table - string\n   * @returns number\n   */\n  async purgeTable(table: string) {\n    return await this.client.transaction(async tx => {\n      return await tx(table).delete();\n    });\n  }\n\n  /**\n   * Returns a list of all providers.\n   * @returns string[]\n   */\n  async listProviders() {\n    return await this.client.transaction(async tx => {\n      const providers = await tx<{ provider_name: string }>(\n        'ingestions',\n      ).distinct('provider_name');\n      return providers.map(entry => entry.provider_name);\n    });\n  }\n\n  async updateByName(provider: string, update: Partial<IngestionUpsert>) {\n    await this.updateIngestionRecordByProvider(provider, update);\n  }\n}\n"],"names":["v4","stringifyEntityRef"],"mappings":";;;;;AA6BO,MAAM,mCAAA,CAAoC;AAAA,EACvC,MAAA;AAAA,EAER,YAAY,OAAA,EAA2B;AACrC,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AAAA,EACxB;AAAA,EAEQ,YAAA,CAAa,QAAgB,MAAA,EAAkB;AACrD,IAAA,MAAM,IAAA,GAAO,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,OAAO,MAAA,KAAW,IAAA;AAClD,IAAA,OAAO,CAAC,EAAA,KAA0B;AAChC,MAAA,IAAI,IAAA,EAAM;AACR,QAAA,EAAA,CAAG,QAAA,CAAS,aAAA,EAAe,CAAC,MAAA,EAAQ,MAAM,CAAC,CAAA;AAAA,MAC7C,CAAA,MAAO;AACL,QAAA,EAAA,CAAG,OAAA,CAAQ,QAAQ,MAAM,CAAA;AAAA,MAC3B;AAAA,IACF,CAAA;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,0BAA0B,OAAA,EAAgC;AAC9D,IAAA,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AACxC,MAAA,MAAM,EAAE,WAAA,EAAa,MAAA,EAAO,GAAI,OAAA;AAChC,MAAA,MAAM,EAAA,CAAG,YAAY,CAAA,CAAE,KAAA,CAAM,MAAM,WAAW,CAAA,CAAE,OAAO,MAAM,CAAA;AAAA,IAC/D,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,+BAAA,CACJ,QAAA,EACA,MAAA,EACA;AACA,IAAA,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AACxC,MAAA,MAAM,EAAA,CAAG,YAAY,CAAA,CAClB,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA,CAC/B,QAAA,CAAS,mBAAA,EAAqB,MAAM,CAAA,CACpC,MAAA,CAAO,MAAM,CAAA;AAAA,IAClB,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,sBAAsB,MAAA,EAAyB;AACnD,IAAA,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AACxC,MAAA,MAAM,EAAA,CAAG,YAAY,CAAA,CAAE,MAAA,CAAO,MAAM,CAAA;AAAA,IACtC,CAAC,CAAA;AAAA,EACH;AAAA,EAEA,MAAc,kBAAA,CACZ,EAAA,EACA,GAAA,EACA;AACA,IAAA,IAAI,GAAA,CAAI,WAAW,CAAA,EAAG;AACpB,MAAA,OAAO,CAAA;AAAA,IACT;AAEA,IAAA,MAAM,MAAA,GAAS,GAAA,CAAI,GAAA,CAAI,CAAA,KAAA,KAAS,MAAM,EAAE,CAAA;AAExC,IAAA,IAAI,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,MAAA,CAAO,WAAW,IAAA,EAAM;AAC7C,MAAA,OAAO,MAAM,EAAA,CAAG,yBAAyB,CAAA,CACtC,MAAA,EAAO,CACP,MAAA,CAAO,IAAA,CAAK,YAAA,CAAa,IAAA,EAAM,MAAM,CAAC,CAAA;AAAA,IAC3C;AAEA,IAAA,IAAI,OAAA,GAAU,CAAA;AACd,IAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,MAAA,CAAO,MAAA,EAAQ,KAAK,GAAA,EAAK;AAC3C,MAAA,MAAM,KAAA,GAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,IAAI,GAAG,CAAA;AACrC,MAAA,OAAA,IAAW,MAAM,GAAG,yBAAyB,CAAA,CAC1C,QAAO,CACP,OAAA,CAAQ,MAAM,KAAK,CAAA;AAAA,IACxB;AACA,IAAA,OAAO,OAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,0BAA0B,QAAA,EAAkB;AAChD,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,MAAA,GAAS,MAAM,EAAA,CAAoB,YAAY,CAAA,CAClD,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA,CAC/B,QAAA,CAAS,mBAAA,EAAqB,MAAM,EACpC,KAAA,EAAM;AACT,MAAA,OAAO,MAAA;AAAA,IACT,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,2BAA2B,QAAA,EAAkB;AACjD,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,OAAO,MAAM,EAAA,CAAoB,YAAY,CAAA,CAC1C,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA,CAC/B,WAAA,CAAY,mBAAA,EAAqB,MAAM,CAAA,CACvC,KAAA,EAAM;AAAA,IACX,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,wBAAwB,QAAA,EAAkB;AAC9C,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,sBAAsB,MAAM,EAAA,CAAG,yBAAyB,CAAA,CAC3D,QAAO,CACP,OAAA;AAAA,QACC,mBAAA;AAAA,QACA,EAAA,CAAG,iBAAiB,CAAA,CACjB,MAAA,CAAO,IAAI,CAAA,CACX,OAAA;AAAA,UACC,cAAA;AAAA,UACA,EAAA,CAAG,YAAY,CAAA,CACZ,MAAA,CAAO,IAAI,CAAA,CACX,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA,CAC/B,WAAA,CAAY,mBAAA,EAAqB,MAAM;AAAA;AAC5C,OACJ;AAEF,MAAA,MAAM,eAAe,MAAM,EAAA,CAAG,iBAAiB,CAAA,CAC5C,QAAO,CACP,OAAA;AAAA,QACC,cAAA;AAAA,QACA,EAAA,CAAG,YAAY,CAAA,CACZ,MAAA,CAAO,IAAI,CAAA,CACX,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA,CAC/B,WAAA,CAAY,mBAAA,EAAqB,MAAM;AAAA,OAC5C;AAEF,MAAA,MAAM,iBAAA,GAAoB,MAAM,EAAA,CAAG,YAAY,CAAA,CAC5C,MAAA,EAAO,CACP,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA,CAC/B,WAAA,CAAY,qBAAqB,MAAM,CAAA;AAE1C,MAAA,OAAO;AAAA,QACL,SAAA,EAAW;AAAA,UACT,mBAAA;AAAA,UACA,YAAA;AAAA,UACA;AAAA;AACF,OACF;AAAA,IACF,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,sBAAsB,QAAA,EAAkB;AAC5C,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,YAAA,GAAiC,MAAM,EAAA,CAAG,YAAY,CAAA,CACzD,OAAO,IAAI,CAAA,CACX,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA;AAElC,MAAA,MAAM,OAAA,GACJ,YAAA,CAAa,MAAA,GAAS,CAAA,GAClB,MAAM,GAAG,iBAAiB,CAAA,CACvB,MAAA,CAAO,IAAI,CAAA,CACX,OAAA;AAAA,QACC,cAAA;AAAA,QACA,YAAA,CAAa,GAAA,CAAI,CAAA,KAAA,KAAS,KAAA,CAAM,EAAE;AAAA,UAEtC,EAAC;AAEP,MAAA,MAAM,aAAA,GACJ,OAAA,CAAQ,MAAA,GAAS,CAAA,GACb,MAAM,GAAG,yBAAyB,CAAA,CAC/B,MAAA,CAAO,IAAI,CAAA,CACX,OAAA;AAAA,QACC,mBAAA;AAAA,QACA,OAAA,CAAQ,GAAA,CAAI,CAAA,KAAA,KAAS,KAAA,CAAM,EAAE;AAAA,UAEjC,EAAC;AAEP,MAAA,MAAM,mBAAA,GAAsB,MAAM,IAAA,CAAK,kBAAA;AAAA,QACrC,EAAA;AAAA,QACA;AAAA,OACF;AAEA,MAAA,MAAM,YAAA,GACJ,QAAQ,MAAA,GAAS,CAAA,GACb,MAAM,EAAA,CAAG,iBAAiB,CAAA,CACvB,MAAA,EAAO,CACP,OAAA;AAAA,QACC,cAAA;AAAA,QACA,YAAA,CAAa,GAAA,CAAI,CAAA,KAAA,KAAS,KAAA,CAAM,EAAE;AAAA,OACpC,GACF,CAAA;AAEN,MAAA,MAAM,iBAAA,GAAoB,MAAM,EAAA,CAAG,YAAY,EAC5C,MAAA,EAAO,CACP,KAAA,CAAM,eAAA,EAAiB,QAAQ,CAAA;AAElC,MAAA,MAAM,cAAA,uBAAqB,IAAA,EAAK;AAChC,MAAA,cAAA,CAAe,QAAQ,cAAA,CAAe,OAAA,KAAY,EAAA,GAAK,EAAA,GAAK,KAAK,GAAI,CAAA;AAErE,MAAA,MAAM,KAAK,qBAAA,CAAsB;AAAA,QAC/B,IAAIA,sBAAA,EAAG;AAAA,QACP,WAAA,EAAa,MAAA;AAAA,QACb,aAAA,EAAe,QAAA;AAAA,QACf,cAAA;AAAA,QACA,sBAAA,sBAA4B,IAAA,EAAK;AAAA,QACjC,MAAA,EAAQ,SAAA;AAAA,QACR,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAED,MAAA,OAAO,EAAE,QAAA,EAAU,iBAAA,EAAmB,YAAA,EAAc,mBAAA,EAAoB;AAAA,IAC1E,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,wBAAA,CACJ,SAAA,EACA,QAAA,EACA;AACA,IAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,IAAA,GAAO,QAAA,CAAS,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,SAAS,CAAA;AAC1C,IAAA,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AACxC,MAAA,MAAM,EAAA,CAAG,yBAAyB,CAAA,CAC/B,MAAA,GACA,KAAA,CAAM,YAAA,EAAc,SAAS,CAAA,CAC7B,MAAA,CAAO,IAAA,CAAK,YAAA,CAAa,YAAA,EAAc,IAAI,CAAC,CAAA;AAAA,IACjD,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,8BAA8B,QAAA,EAAkB;AACpD,IAAA,MAAM,cAAcA,sBAAA,EAAG;AACvB,IAAA,MAAM,UAAA,GAAa,QAAA;AACnB,IAAA,IAAI;AACF,MAAA,MAAM,KAAK,qBAAA,CAAsB;AAAA,QAC/B,EAAA,EAAI,WAAA;AAAA,QACJ,WAAA,EAAa,UAAA;AAAA,QACb,aAAA,EAAe,QAAA;AAAA,QACf,MAAA,EAAQ,UAAA;AAAA,QACR,iBAAA,EAAmB;AAAA,OACpB,CAAA;AACD,MAAA,OAAO,EAAE,aAAa,UAAA,EAAY,QAAA,EAAU,GAAG,YAAA,EAAc,IAAA,CAAK,KAAI,EAAE;AAAA,IAC1E,SAAS,EAAA,EAAI;AAEX,MAAA,OAAO,MAAA;AAAA,IACT;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,cAAA,CAAe,QAAA,EAAkB,WAAA,EAAqB;AAC1D,IAAA,MAAM,iBAAA,GAAoB,MAAM,IAAA,CAAK,0BAAA,CAA2B,QAAQ,CAAA;AACxE,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,KAAA,GAAQ,MAAM,EAAA,CAAG,yBAAyB,CAAA,CAC7C,MAAM,EAAE,KAAA,EAAO,oCAAA,EAAsC,CAAA,CACrD,IAAA;AAAA,QACC,iBAAA;AAAA,QACA,oBAAA;AAAA,QACA;AAAA,OACF,CACC,KAAK,YAAA,EAAc,eAAA,EAAiB,8BAA8B,CAAA,CAClE,KAAA,CAAM,iBAAiB,WAAW,CAAA;AAErC,MAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,MAAA,CAAO,CAAC,GAAA,EAAK,GAAA,KAAQ,GAAA,GAAM,MAAA,CAAO,GAAA,CAAI,KAAK,CAAA,EAAG,CAAC,CAAA;AAEnE,MAAA,MAAM,UAAmC,EAAC;AAC1C,MAAA,IAAI,iBAAA,EAAmB;AACrB,QAAA,MAAM,QAAkC,MAAM,EAAA;AAAA,UAC5C;AAAA,SACF,CACG,MAAA,CAAO,oCAAoC,CAAA,CAC3C,IAAA;AAAA,UACC,iBAAA;AAAA,UACA,oBAAA;AAAA,UACA;AAAA,SACF,CACC,KAAK,YAAA,EAAc,eAAA,EAAiB,8BAA8B,CAAA,CAClE,KAAA,CAAM,eAAA,EAAiB,iBAAA,CAAkB,EAAE,CAAA;AAE9C,QAAA,KAAA,MAAW,OAAO,KAAA,EAAO;AACvB,UAAA,OAAA,CAAQ,IAAA,CAAK,EAAE,SAAA,EAAW,GAAA,CAAI,YAAY,CAAA;AAAA,QAC5C;AAAA,MACF;AAEA,MAAA,OAAO,EAAE,OAAO,OAAA,EAAQ;AAAA,IAC1B,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,WAAA,GAAc;AAClB,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,UAAU,MAAM,EAAA;AAAA,QACpB;AAAA,QAEC,QAAA,CAAS,IAAA,EAAM,eAAe,CAAA,CAC9B,KAAA,CAAM,qBAAqB,IAAI,CAAA;AAClC,MAAA,OAAO,OAAA;AAAA,IACT,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,0BAA0B,QAAA,EAAkB;AAChD,IAAA,MAAM,IAAA,CAAK,gCAAgC,QAAA,EAAU;AAAA,MACnD,cAAA,sBAAoB,IAAA;AAAK,KAC1B,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,MAAM,gBAAA,GAAmB;AACvB,IAAA,MAAM,SAAA,GAAY,MAAM,IAAA,CAAK,aAAA,EAAc;AAE3C,IAAA,MAAM,iBAAA,GAAoB,MAAM,IAAA,CAAK,UAAA,CAAW,YAAY,CAAA;AAE5D,IAAA,MAAM,cAAA,uBAAqB,IAAA,EAAK;AAChC,IAAA,cAAA,CAAe,QAAQ,cAAA,CAAe,OAAA,KAAY,EAAA,GAAK,EAAA,GAAK,KAAK,GAAI,CAAA;AAErE,IAAA,KAAA,MAAW,YAAY,SAAA,EAAW;AAChC,MAAA,MAAM,KAAK,qBAAA,CAAsB;AAAA,QAC/B,IAAIA,sBAAA,EAAG;AAAA,QACP,WAAA,EAAa,MAAA;AAAA,QACb,aAAA,EAAe,QAAA;AAAA,QACf,cAAA;AAAA,QACA,sBAAA,sBAA4B,IAAA,EAAK;AAAA,QACjC,MAAA,EAAQ,SAAA;AAAA,QACR,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAAA,IACH;AAEA,IAAA,MAAM,qBAAA,GAAwB,MAAM,IAAA,CAAK,UAAA,CAAW,iBAAiB,CAAA;AACrE,IAAA,MAAM,mBAAA,GAAsB,MAAM,IAAA,CAAK,UAAA;AAAA,MACrC;AAAA,KACF;AAEA,IAAA,OAAO,EAAE,iBAAA,EAAmB,qBAAA,EAAuB,mBAAA,EAAoB;AAAA,EACzE;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,qBAAqB,WAAA,EAAqB;AAC9C,IAAA,MAAM,KAAK,yBAAA,CAA0B;AAAA,MACnC,WAAA;AAAA,MACA,MAAA,EAAQ,EAAE,WAAA,EAAa,QAAA;AAAS,KACjC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,oBAAoB,WAAA,EAAqB;AAC7C,IAAA,MAAM,KAAK,yBAAA,CAA0B;AAAA,MACnC,WAAA;AAAA,MACA,MAAA,EAAQ,EAAE,MAAA,EAAQ,UAAA;AAAW,KAC9B,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,oBAAoB,WAAA,EAAqB;AAC7C,IAAA,MAAM,KAAK,yBAAA,CAA0B;AAAA,MACnC,WAAA;AAAA,MACA,MAAA,EAAQ;AAAA,QACN,WAAA,EAAa,gBAAA;AAAA,QACb,iBAAA,sBAAuB,IAAA,EAAK;AAAA,QAC5B,MAAA,EAAQ,UAAA;AAAA,QACR,mBAAmBA,sBAAA;AAAG;AACxB,KACD,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,kBAAA,CAAmB,WAAA,EAAqB,UAAA,EAAsB;AAClE,IAAA,MAAM,KAAK,yBAAA,CAA0B;AAAA,MACnC,WAAA;AAAA,MACA,MAAA,EAAQ;AAAA,QACN,WAAA,EAAa,MAAA;AAAA,QACb,cAAA,EAAgB,IAAI,IAAA,CAAK,IAAA,CAAK,KAAI,GAAI,UAAA,CAAW,EAAA,CAAG,cAAc,CAAC,CAAA;AAAA,QACnE,sBAAA,sBAA4B,IAAA,EAAK;AAAA,QACjC,MAAA,EAAQ;AAAA;AACV,KACD,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,wBAAwB,WAAA,EAAqB;AACjD,IAAA,MAAM,KAAK,yBAAA,CAA0B;AAAA,MACnC,WAAA;AAAA,MACA,MAAA,EAAQ,EAAE,QAAA,EAAU,CAAA,EAAG,QAAQ,cAAA;AAAe,KAC/C,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,oBAAA,CAAqB,WAAA,EAAqB,OAAA,EAAkB;AAChE,IAAA,MAAM,MAAA,GAAmC;AAAA,MACvC,WAAA,EAAa,QAAA;AAAA,MACb,UAAA,EAAY,UAAU,OAAA,GAAU,MAAA;AAAA,MAChC,cAAA,sBAAoB,IAAA,EAAK;AAAA,MACzB,MAAA,EAAQ;AAAA,KACV;AACA,IAAA,MAAM,IAAA,CAAK,yBAAA,CAA0B,EAAE,WAAA,EAAa,QAAQ,CAAA;AAAA,EAC9D;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,oBAAoB,WAAA,EAAqB;AAC7C,IAAA,MAAM,KAAK,yBAAA,CAA0B;AAAA,MACnC,WAAA;AAAA,MACA,MAAA,EAAQ;AAAA,QACN,WAAA,EAAa,oBAAA;AAAA,QACb,iBAAA,sBAAuB,IAAA,EAAK;AAAA,QAC5B,MAAA,EAAQ,UAAA;AAAA,QACR,mBAAmBA,sBAAA;AAAG;AACxB,KACD,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAM,kBAAA,CACJ,WAAA,EACA,QAAA,EACA,OACA,aAAA,EACA;AACA,IAAA,MAAM,KAAK,yBAAA,CAA0B;AAAA,MACnC,WAAA;AAAA,MACA,MAAA,EAAQ;AAAA,QACN,WAAA,EAAa,SAAA;AAAA,QACb,UAAU,QAAA,GAAW,CAAA;AAAA,QACrB,UAAA,EAAY,OAAO,KAAK,CAAA;AAAA,QACxB,gBAAgB,IAAI,IAAA,CAAK,IAAA,CAAK,GAAA,KAAQ,aAAa,CAAA;AAAA,QACnD,MAAA,EAAQ;AAAA;AACV,KACD,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,YAAY,WAAA,EAAqB;AACrC,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,IAAA,GAAO,MAAM,EAAA,CAAe,iBAAiB,CAAA,CAChD,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,OAAA,CAAQ,UAAA,EAAY,MAAM,EAC1B,KAAA,EAAM;AACT,MAAA,OAAO,IAAA,CAAK,WAAA,CAAY,IAAA,CAAK,MAAA,EAAQ,IAAI,CAAA;AAAA,IAC3C,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,aAAa,WAAA,EAAqB;AACtC,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,IAAA,GAAO,MAAM,EAAA,CAAe,iBAAiB,CAAA,CAChD,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,OAAA,CAAQ,UAAA,EAAY,KAAK,EACzB,KAAA,EAAM;AACT,MAAA,OAAO,IAAA,CAAK,WAAA,CAAY,IAAA,CAAK,MAAA,EAAQ,IAAI,CAAA;AAAA,IAC3C,CAAC,CAAA;AAAA,EACH;AAAA,EAEA,MAAM,YAAY,WAAA,EAAqB;AACrC,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,KAAA,GAAQ,MAAM,EAAA,CAAe,iBAAiB,CAAA,CACjD,KAAA,CAAM,cAAA,EAAgB,WAAW,CAAA,CACjC,OAAA,CAAQ,UAAA,EAAY,MAAM,CAAA;AAC7B,MAAA,OAAO,KAAA,CAAM,IAAI,CAAA,CAAA,KAAK,IAAA,CAAK,YAAY,IAAA,CAAK,MAAA,EAAQ,CAAC,CAAC,CAAA;AAAA,IACxD,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,WAAW,OAAA,EAA2B;AAC1C,IAAA,MAAM,EAAE,QAAO,GAAI,OAAA;AACnB,IAAA,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AACxC,MAAA,MAAM,EAAA,CAAG,iBAAiB,CAAA,CAAE,MAAA,CAAO,MAAM,CAAA;AAAA,IAC3C,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA,EAIA,WAAA,CAA8C,MAAY,MAAA,EAAc;AACtE,IAAA,IAAI,UAAU,IAAA,CAAK,MAAA,CAAO,OAAO,MAAA,CAAO,QAAA,CAAS,SAAS,CAAA,EAAG;AAC3D,MAAA,OAAO;AAAA,QACL,GAAG,MAAA;AAAA,QACH,MAAA,EAAQ,IAAA,CAAK,KAAA,CAAM,MAAA,CAAO,MAAgB;AAAA,OAC5C;AAAA,IACF;AACA,IAAA,OAAO,MAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAM,kBAAA,CACJ,SAAA,EACA,QAAA,EACA,MAAA,EACA;AACA,IAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,OAAO,QAAA,CAAS,GAAA,CAAI,OAAKC,+BAAA,CAAmB,CAAA,CAAE,MAAM,CAAC,CAAA;AAE3D,IAAA,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AACxC,MAAA,MAAM,EAAA,CAAG,yBAAyB,CAAA,CAC/B,MAAA;AAAA,QACC,IAAA,CAAK,IAAI,CAAA,SAAA,MAAc;AAAA,UACrB,IAAID,sBAAA,EAAG;AAAA,UACP,iBAAA,EAAmB,MAAA;AAAA,UACnB,UAAA,EAAY,SAAA;AAAA,UACZ,UAAA,EAAY;AAAA,SACd,CAAE;AAAA,OACJ,CACC,UAAA,CAAW,CAAC,YAAA,EAAc,YAAY,CAAC,CAAA,CACvC,KAAA,CAAM,CAAC,mBAAmB,CAAC,CAAA;AAAA,IAChC,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,WAAW,KAAA,EAAe;AAC9B,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,OAAO,MAAM,EAAA,CAAG,KAAK,CAAA,CAAE,MAAA,EAAO;AAAA,IAChC,CAAC,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,aAAA,GAAgB;AACpB,IAAA,OAAO,MAAM,IAAA,CAAK,MAAA,CAAO,WAAA,CAAY,OAAM,EAAA,KAAM;AAC/C,MAAA,MAAM,YAAY,MAAM,EAAA;AAAA,QACtB;AAAA,OACF,CAAE,SAAS,eAAe,CAAA;AAC1B,MAAA,OAAO,SAAA,CAAU,GAAA,CAAI,CAAA,KAAA,KAAS,KAAA,CAAM,aAAa,CAAA;AAAA,IACnD,CAAC,CAAA;AAAA,EACH;AAAA,EAEA,MAAM,YAAA,CAAa,QAAA,EAAkB,MAAA,EAAkC;AACrE,IAAA,MAAM,IAAA,CAAK,+BAAA,CAAgC,QAAA,EAAU,MAAM,CAAA;AAAA,EAC7D;AACF;;"}

package/dist/database/migrations.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"migrations.cjs.js","sources":["../../src/database/migrations.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { resolvePackagePath } from '@backstage/backend-plugin-api';\nimport { Knex } from 'knex';\nimport { DB_MIGRATIONS_TABLE } from './tables';\n\nexport async function applyDatabaseMigrations(knex: Knex): Promise<void> {\n  const migrationsDir = resolvePackagePath(\n    '@backstage/plugin-catalog-backend-module-incremental-ingestion',\n    'migrations',\n  );\n\n  await knex.migrate.latest({\n    directory: migrationsDir,\n    tableName: DB_MIGRATIONS_TABLE,\n  });\n}\n"],"names":["resolvePackagePath","DB_MIGRATIONS_TABLE"],"mappings":";;;;;AAoBA,eAAsB,wBAAwB,IAAA,EAA2B;AACvE,EAAA,MAAM,aAAA,GAAgBA,mCAAA;AAAA,IACpB,gEAAA;AAAA,IACA;AAAA,GACF;AAEA,EAAA,MAAM,IAAA,CAAK,QAAQ,MAAA,CAAO;AAAA,IACxB,SAAA,EAAW,aAAA;AAAA,IACX,SAAA,EAAWC;AAAA,GACZ,CAAA;AACH;;;;"}
+{"version":3,"file":"migrations.cjs.js","sources":["../../src/database/migrations.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { resolvePackagePath } from '@backstage/backend-plugin-api';\nimport { Knex } from 'knex';\nimport { DB_MIGRATIONS_TABLE } from './tables';\n\nexport async function applyDatabaseMigrations(knex: Knex): Promise<void> {\n  const migrationsDir = resolvePackagePath(\n    '@backstage/plugin-catalog-backend-module-incremental-ingestion',\n    'migrations',\n  );\n\n  await knex.migrate.latest({\n    directory: migrationsDir,\n    tableName: DB_MIGRATIONS_TABLE,\n  });\n}\n"],"names":["resolvePackagePath","DB_MIGRATIONS_TABLE"],"mappings":";;;;;AAoBA,eAAsB,wBAAwB,IAAA,EAA2B;AACvE,EAAA,MAAM,aAAA,GAAgBA,mCAAA;AAAA,IACpB,gEAAA;AAAA,IACA;AAAA,GACF;AAEA,EAAA,MAAM,IAAA,CAAK,QAAQ,MAAA,CAAO;AAAA,IACxB,SAAA,EAAW,aAAA;AAAA,IACX,SAAA,EAAWC;AAAA,GACZ,CAAA;AACH;;"}

package/dist/database/tables.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"tables.cjs.js","sources":["../../src/database/tables.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nexport const DB_MIGRATIONS_TABLE = 'incremental_ingestion__knex_migrations';\n\n/**\n * The shape of data inserted into or updated in the `ingestions` table.\n */\nexport interface IngestionUpsert {\n  /**\n   * The ingestion record id.\n   */\n  id?: string;\n  /**\n   * The next action the incremental entity provider will take.\n   */\n  next_action:\n    | 'rest'\n    | 'ingest'\n    | 'backoff'\n    | 'cancel'\n    | 'nothing (done)'\n    | 'nothing (canceled)';\n  /**\n   * Current status of the incremental entity provider.\n   */\n  status:\n    | 'complete'\n    | 'bursting'\n    | 'resting'\n    | 'canceling'\n    | 'interstitial'\n    | 'backing off';\n  /**\n   * The name of the incremental entity provider being updated.\n   */\n  provider_name: string;\n  /**\n   * Date/time stamp for when the next action will trigger.\n   */\n  next_action_at?: Date;\n  /**\n   * A record of the last error generated by the incremental entity provider.\n   */\n  last_error?: string | null;\n  /**\n   * The number of attempts the provider has attempted during the current cycle.\n   */\n  attempts?: number;\n  /**\n   * Date/time stamp for the completion of ingestion.\n   */\n  ingestion_completed_at?: Date | string | null;\n  /**\n   * Date/time stamp for the end of the rest cycle before the next ingestion.\n   */\n  rest_completed_at?: Date | string | null;\n  /**\n   * A record of the finalized status of the ingestion record. Values are either 'open' or a uuid.\n   */\n  completion_ticket: string;\n}\n\n/**\n * This interface is for updating an existing ingestion record.\n */\nexport interface IngestionRecordUpdate {\n  ingestionId: string;\n  update: Partial<IngestionUpsert>;\n}\n\n/**\n * The expected response from the `ingestion_marks` table.\n */\nexport interface MarkRecord {\n  id: string;\n  sequence: number;\n  ingestion_id: string;\n  cursor: unknown;\n  created_at: string;\n}\n\n/**\n * The expected response from the `ingestions` table.\n */\nexport interface IngestionRecord extends IngestionUpsert {\n  id: string;\n  next_action_at: Date;\n  /**\n   * The date/time the ingestion record was created.\n   */\n  created_at: string;\n}\n\n/**\n * This interface supplies all the values for adding an ingestion mark.\n */\nexport interface MarkRecordInsert {\n  record: {\n    id: string;\n    ingestion_id: string;\n    cursor: unknown;\n    sequence: number;\n  };\n}\n"],"names":[],"mappings":";;AAgBO,MAAM,mBAAA,GAAsB;;;;"}
+{"version":3,"file":"tables.cjs.js","sources":["../../src/database/tables.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nexport const DB_MIGRATIONS_TABLE = 'incremental_ingestion__knex_migrations';\n\n/**\n * The shape of data inserted into or updated in the `ingestions` table.\n */\nexport interface IngestionUpsert {\n  /**\n   * The ingestion record id.\n   */\n  id?: string;\n  /**\n   * The next action the incremental entity provider will take.\n   */\n  next_action:\n    | 'rest'\n    | 'ingest'\n    | 'backoff'\n    | 'cancel'\n    | 'nothing (done)'\n    | 'nothing (canceled)';\n  /**\n   * Current status of the incremental entity provider.\n   */\n  status:\n    | 'complete'\n    | 'bursting'\n    | 'resting'\n    | 'canceling'\n    | 'interstitial'\n    | 'backing off';\n  /**\n   * The name of the incremental entity provider being updated.\n   */\n  provider_name: string;\n  /**\n   * Date/time stamp for when the next action will trigger.\n   */\n  next_action_at?: Date;\n  /**\n   * A record of the last error generated by the incremental entity provider.\n   */\n  last_error?: string | null;\n  /**\n   * The number of attempts the provider has attempted during the current cycle.\n   */\n  attempts?: number;\n  /**\n   * Date/time stamp for the completion of ingestion.\n   */\n  ingestion_completed_at?: Date | string | null;\n  /**\n   * Date/time stamp for the end of the rest cycle before the next ingestion.\n   */\n  rest_completed_at?: Date | string | null;\n  /**\n   * A record of the finalized status of the ingestion record. Values are either 'open' or a uuid.\n   */\n  completion_ticket: string;\n}\n\n/**\n * This interface is for updating an existing ingestion record.\n */\nexport interface IngestionRecordUpdate {\n  ingestionId: string;\n  update: Partial<IngestionUpsert>;\n}\n\n/**\n * The expected response from the `ingestion_marks` table.\n */\nexport interface MarkRecord {\n  id: string;\n  sequence: number;\n  ingestion_id: string;\n  cursor: unknown;\n  created_at: string;\n}\n\n/**\n * The expected response from the `ingestions` table.\n */\nexport interface IngestionRecord extends IngestionUpsert {\n  id: string;\n  next_action_at: Date;\n  /**\n   * The date/time the ingestion record was created.\n   */\n  created_at: string;\n}\n\n/**\n * This interface supplies all the values for adding an ingestion mark.\n */\nexport interface MarkRecordInsert {\n  record: {\n    id: string;\n    ingestion_id: string;\n    cursor: unknown;\n    sequence: number;\n  };\n}\n"],"names":[],"mappings":";;AAgBO,MAAM,mBAAA,GAAsB;;"}

package/dist/engine/IncrementalIngestionEngine.cjs.js

@@ -32,6 +32,7 @@ class IncrementalIngestionEngine {
       }
     );
   }
+  options;
   restLength;
   burstLength;
   backoff;
@@ -224,10 +225,15 @@ class IncrementalIngestionEngine {
     );
     return done;
   }
-  async mark(options) {
-    const { id, sequence, entities, done, cursor } = options;
+  async mark({
+    id,
+    sequence,
+    entities = [],
+    done,
+    cursor
+  }) {
     this.options.logger.debug(
-      `incremental-engine: Ingestion '${id}': MARK ${entities ? entities.length : 0} entities, cursor: ${cursor ? JSON.stringify(cursor) : "none"}, done: ${done}`
+      `incremental-engine: Ingestion '${id}': MARK ${entities.length} entities, cursor: ${cursor ? JSON.stringify(cursor) : "none"}, done: ${done}`
     );
     const markId = node_crypto.randomUUID();
     await this.manager.createMark({
@@ -238,10 +244,12 @@ class IncrementalIngestionEngine {
         sequence
       }
     });
-    if (entities && entities.length > 0) {
-      await this.manager.createMarkEntities(markId, entities);
-    }
-    const added = entities?.map((deferred) => ({
+    await this.manager.createMarkEntities(
+      this.options.provider.getProviderName(),
+      entities,
+      markId
+    );
+    const added = entities.map((deferred) => ({
       ...deferred,
       entity: {
         ...deferred.entity,
@@ -252,7 +260,7 @@ class IncrementalIngestionEngine {
           }
         }
       }
-    })) ?? [];
+    }));
     const removed = [];
     if (done) {
       this.options.logger.info(
@@ -316,37 +324,39 @@ class IncrementalIngestionEngine {
       return;
     }
     const result = await provider.eventHandler.onEvent(params);
-    if (result.type === "delta") {
-      if (result.added.length > 0) {
-        const ingestionRecord = await this.manager.getCurrentIngestionRecord(
-          providerName
+    if (result.type === "ignored") {
+      logger.debug(
+        `incremental-engine: ${providerName} ignored event from topic '${topic}'`
+      );
+      return;
+    }
+    if (result.added.length > 0) {
+      const ingestionRecord = await this.manager.getCurrentIngestionRecord(
+        providerName
+      );
+      if (!ingestionRecord) {
+        logger.debug(
+          `incremental-engine: ${providerName} skipping delta addition because incremental ingestion is restarting.`
         );
-        if (!ingestionRecord) {
-          logger.debug(
-            `incremental-engine: ${providerName} skipping delta addition because incremental ingestion is restarting.`
+      } else {
+        const mark = ingestionRecord.status === "resting" ? await this.manager.getLastMark(ingestionRecord.id) : await this.manager.getFirstMark(ingestionRecord.id);
+        if (!mark) {
+          throw new Error(
+            `Cannot apply delta, page records are missing! Please re-run incremental ingestion for ${providerName}.`
           );
-        } else {
-          const mark = ingestionRecord.status === "resting" ? await this.manager.getLastMark(ingestionRecord.id) : await this.manager.getFirstMark(ingestionRecord.id);
-          if (!mark) {
-            throw new Error(
-              `Cannot apply delta, page records are missing! Please re-run incremental ingestion for ${providerName}.`
-            );
-          }
-          await this.manager.createMarkEntities(mark.id, result.added);
         }
+        await this.manager.createMarkEntities(
+          providerName,
+          result.added,
+          mark.id
+        );
       }
-      if (result.removed.length > 0) {
-        await this.manager.deleteEntityRecordsByRef(result.removed);
-      }
-      await connection.applyMutation(result);
-      logger.debug(
-        `incremental-engine: ${providerName} processed delta from '${topic}' event`
-      );
-    } else {
-      logger.debug(
-        `incremental-engine: ${providerName} ignored event from topic '${topic}'`
-      );
     }
+    await this.manager.deleteEntityRecordsByRef(providerName, result.removed);
+    await connection.applyMutation(result);
+    logger.debug(
+      `incremental-engine: ${providerName} processed delta from '${topic}' event`
+    );
   }
   supportsEventTopics() {
     const { provider } = this.options;

package/dist/engine/IncrementalIngestionEngine.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"IncrementalIngestionEngine.cjs.js","sources":["../../src/engine/IncrementalIngestionEngine.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport type { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { MetricsServiceGauge } from '@backstage/backend-plugin-api/alpha';\nimport { IterationEngine, IterationEngineOptions } from '../types';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { performance } from 'node:perf_hooks';\nimport { Duration } from 'luxon';\nimport { randomUUID as v4 } from 'node:crypto';\nimport { stringifyError, toError } from '@backstage/errors';\nimport { EventParams } from '@backstage/plugin-events-node';\nimport { HumanDuration } from '@backstage/types';\n\nexport class IncrementalIngestionEngine implements IterationEngine {\n  private readonly restLength: Duration;\n  private readonly burstLength: Duration;\n  private readonly backoff: HumanDuration[];\n  private readonly lastStarted: MetricsServiceGauge;\n  private readonly lastCompleted: MetricsServiceGauge;\n\n  private manager: IncrementalIngestionDatabaseManager;\n\n  constructor(private options: IterationEngineOptions) {\n    this.manager = options.manager;\n    this.restLength = Duration.fromObject(options.restLength);\n    this.burstLength = Duration.fromObject(options.burstLength);\n    this.backoff = options.backoff ?? [\n      { minutes: 1 },\n      { minutes: 5 },\n      { minutes: 30 },\n      { hours: 3 },\n    ];\n\n    this.lastStarted = options.metrics.createGauge(\n      'catalog_incremental.ingestions.started',\n      {\n        description: 'Epoch timestamp when the ingestion was last started',\n        unit: 's',\n      },\n    );\n    this.lastCompleted = options.metrics.createGauge(\n      'catalog_incremental.ingestions.completed',\n      {\n        description: 'Epoch timestamp when the ingestion was last completed',\n        unit: 's',\n      },\n    );\n  }\n\n  async taskFn(signal: AbortSignal) {\n    try {\n      this.options.logger.debug('Begin tick');\n      await this.handleNextAction(signal);\n    } catch (error) {\n      this.options.logger.error(`${error}`);\n      throw error;\n    } finally {\n      this.options.logger.debug('End tick');\n    }\n  }\n\n  async handleNextAction(signal: AbortSignal) {\n    await this.options.ready;\n\n    const result = await this.getCurrentAction();\n    if (result) {\n      const { ingestionId, nextActionAt, nextAction, attempts } = result;\n\n      switch (nextAction) {\n        case 'rest':\n          if (Date.now() > nextActionAt) {\n            await this.manager.clearFinishedIngestions(\n              this.options.provider.getProviderName(),\n            );\n            this.options.logger.debug(\n              `incremental-engine: Ingestion ${ingestionId} rest period complete. Ingestion will start again`,\n            );\n\n            this.lastStarted.record(Date.now() / 1000, {\n              providerName: this.options.provider.getProviderName(),\n            });\n            await this.manager.setProviderComplete(ingestionId);\n          } else {\n            this.options.logger.debug(\n              `incremental-engine: Ingestion '${ingestionId}' rest period continuing`,\n            );\n          }\n          break;\n        case 'ingest':\n          try {\n            await this.manager.setProviderBursting(ingestionId);\n            const done = await this.ingestOneBurst(ingestionId, signal);\n            if (done) {\n              this.options.logger.info(\n                `incremental-engine: Ingestion '${ingestionId}' complete, transitioning to rest period of ${this.restLength.toHuman()}`,\n              );\n              this.lastCompleted.record(Date.now() / 1000, {\n                providerName: this.options.provider.getProviderName(),\n                status: 'completed',\n              });\n              await this.manager.setProviderResting(\n                ingestionId,\n                this.restLength,\n              );\n            } else {\n              await this.manager.setProviderInterstitial(ingestionId);\n              this.options.logger.debug(\n                `incremental-engine: Ingestion '${ingestionId}' continuing`,\n              );\n            }\n          } catch (error) {\n            const err = toError(error);\n            if (err.message === 'CANCEL') {\n              this.options.logger.info(\n                `incremental-engine: Ingestion '${ingestionId}' canceled`,\n              );\n              await this.manager.setProviderCanceling(ingestionId, err.message);\n            } else {\n              const currentBackoff = Duration.fromObject(\n                this.backoff[Math.min(this.backoff.length - 1, attempts)],\n              );\n\n              const backoffLength = currentBackoff.as('milliseconds');\n              this.options.logger.error(\n                `incremental-engine: Ingestion '${ingestionId}' failed`,\n                error,\n              );\n\n              const truncatedError = stringifyError(error).substring(0, 700);\n              this.options.logger.error(\n                `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`,\n              );\n              this.lastCompleted.record(Date.now() / 1000, {\n                providerName: this.options.provider.getProviderName(),\n                status: 'failed',\n              });\n\n              await this.manager.setProviderBackoff(\n                ingestionId,\n                attempts,\n                error as Error,\n                backoffLength,\n              );\n            }\n          }\n          break;\n        case 'backoff':\n          if (Date.now() > nextActionAt) {\n            this.options.logger.info(\n              `incremental-engine: Ingestion '${ingestionId}' backoff complete, will attempt to resume`,\n            );\n            await this.manager.setProviderIngesting(ingestionId);\n          } else {\n            this.options.logger.debug(\n              `incremental-engine: Ingestion '${ingestionId}' backoff continuing`,\n            );\n          }\n          break;\n        case 'cancel':\n          this.options.logger.info(\n            `incremental-engine: Ingestion '${ingestionId}' canceling, will restart`,\n          );\n          await this.manager.setProviderCanceled(ingestionId);\n          break;\n        default:\n          this.options.logger.error(\n            `incremental-engine: Ingestion '${ingestionId}' received unknown action '${nextAction}'`,\n          );\n      }\n    } else {\n      this.options.logger.error(\n        `incremental-engine: Engine tried to create duplicate ingestion record for provider '${this.options.provider.getProviderName()}'.`,\n      );\n    }\n  }\n\n  async getCurrentAction() {\n    const providerName = this.options.provider.getProviderName();\n    const record = await this.manager.getCurrentIngestionRecord(providerName);\n    if (record) {\n      this.options.logger.debug(\n        `incremental-engine: Ingestion record found: '${record.id}'`,\n      );\n      return {\n        ingestionId: record.id,\n        nextAction: record.next_action as 'rest' | 'ingest' | 'backoff',\n        attempts: record.attempts as number,\n        nextActionAt: record.next_action_at.valueOf() as number,\n      };\n    }\n    const result = await this.manager.createProviderIngestionRecord(\n      providerName,\n    );\n    if (result) {\n      this.options.logger.info(\n        `incremental-engine: Ingestion record created: '${result.ingestionId}'`,\n      );\n    }\n    return result;\n  }\n\n  async ingestOneBurst(id: string, signal: AbortSignal) {\n    const lastMark = await this.manager.getLastMark(id);\n\n    const cursor = lastMark ? lastMark.cursor : undefined;\n    let sequence = lastMark ? lastMark.sequence + 1 : 0;\n\n    const start = performance.now();\n    let count = 0;\n    let done = false;\n    this.options.logger.info(\n      `incremental-engine: Ingestion '${id}' burst initiated`,\n    );\n\n    await this.options.provider.around(async (context: unknown) => {\n      let next = await this.options.provider.next(context, cursor);\n      count++;\n      for (;;) {\n        done = next.done;\n        await this.mark({\n          id,\n          sequence,\n          entities: next?.entities,\n          done: next.done,\n          cursor: next?.cursor,\n        });\n        if (signal.aborted || next.done) {\n          break;\n        } else if (\n          performance.now() - start >\n          this.burstLength.as('milliseconds')\n        ) {\n          this.options.logger.info(\n            `incremental-engine: Ingestion '${id}' burst ending after ${this.burstLength.toHuman()}.`,\n          );\n          break;\n        } else {\n          next = await this.options.provider.next(context, next.cursor);\n          count++;\n          sequence++;\n        }\n      }\n    });\n\n    this.options.logger.info(\n      `incremental-engine: Ingestion '${id}' burst complete. (${count} batches in ${Math.round(\n        performance.now() - start,\n      )}ms).`,\n    );\n    return done;\n  }\n\n  async mark(options: {\n    id: string;\n    sequence: number;\n    entities?: DeferredEntity[];\n    done: boolean;\n    cursor?: unknown;\n  }) {\n    const { id, sequence, entities, done, cursor } = options;\n    this.options.logger.debug(\n      `incremental-engine: Ingestion '${id}': MARK ${\n        entities ? entities.length : 0\n      } entities, cursor: ${\n        cursor ? JSON.stringify(cursor) : 'none'\n      }, done: ${done}`,\n    );\n    const markId = v4();\n\n    await this.manager.createMark({\n      record: {\n        id: markId,\n        ingestion_id: id,\n        cursor,\n        sequence,\n      },\n    });\n\n    if (entities && entities.length > 0) {\n      await this.manager.createMarkEntities(markId, entities);\n    }\n\n    const added =\n      entities?.map(deferred => ({\n        ...deferred,\n        entity: {\n          ...deferred.entity,\n          metadata: {\n            ...deferred.entity.metadata,\n            annotations: {\n              ...deferred.entity.metadata.annotations,\n            },\n          },\n        },\n      })) ?? [];\n\n    const removed: { entityRef: string }[] = [];\n\n    if (done) {\n      this.options.logger.info(\n        `incremental-engine: Ingestion '${id}': Final page reached, calculating removed entities`,\n      );\n      const result = await this.manager.computeRemoved(\n        this.options.provider.getProviderName(),\n        id,\n      );\n\n      const { total } = result;\n\n      let doRemoval = true;\n      if (this.options.rejectEmptySourceCollections) {\n        if (total === 0) {\n          this.options.logger.error(\n            `incremental-engine: Ingestion '${id}': Rejecting empty entity collection!`,\n          );\n          doRemoval = false;\n        }\n      }\n\n      if (this.options.rejectRemovalsAbovePercentage) {\n        // If the total entities upserted in this ingestion is 0, then\n        // 100% of entities are stale and marked for removal.\n        const percentRemoved =\n          total > 0 ? (result.removed.length / total) * 100 : 100;\n        if (percentRemoved <= this.options.rejectRemovalsAbovePercentage) {\n          this.options.logger.info(\n            `incremental-engine: Ingestion '${id}': Removing ${result.removed.length} entities that have no matching assets`,\n          );\n        } else {\n          const notice = `Attempted to remove ${percentRemoved}% of matching entities!`;\n          this.options.logger.error(\n            `incremental-engine: Ingestion '${id}': ${notice}`,\n          );\n          await this.manager.updateIngestionRecordById({\n            ingestionId: id,\n            update: {\n              last_error: `REMOVAL_THRESHOLD exceeded on ingestion mark ${markId}: ${notice}`,\n            },\n          });\n          doRemoval = false;\n        }\n      }\n      if (doRemoval) {\n        for (const entityRef of result.removed) {\n          removed.push(entityRef);\n        }\n      }\n    }\n\n    await this.options.connection.applyMutation({\n      type: 'delta',\n      added,\n      removed,\n    });\n  }\n\n  async onEvent(params: EventParams): Promise<void> {\n    const { topic } = params;\n    if (!this.supportsEventTopics().includes(topic)) {\n      return;\n    }\n\n    const { logger, provider, connection } = this.options;\n    const providerName = provider.getProviderName();\n    logger.debug(`incremental-engine: ${providerName} received ${topic} event`);\n\n    if (!provider.eventHandler) {\n      return;\n    }\n\n    const result = await provider.eventHandler.onEvent(params);\n\n    if (result.type === 'delta') {\n      if (result.added.length > 0) {\n        const ingestionRecord = await this.manager.getCurrentIngestionRecord(\n          providerName,\n        );\n\n        if (!ingestionRecord) {\n          logger.debug(\n            `incremental-engine: ${providerName} skipping delta addition because incremental ingestion is restarting.`,\n          );\n        } else {\n          const mark =\n            ingestionRecord.status === 'resting'\n              ? await this.manager.getLastMark(ingestionRecord.id)\n              : await this.manager.getFirstMark(ingestionRecord.id);\n\n          if (!mark) {\n            throw new Error(\n              `Cannot apply delta, page records are missing! Please re-run incremental ingestion for ${providerName}.`,\n            );\n          }\n          await this.manager.createMarkEntities(mark.id, result.added);\n        }\n      }\n\n      if (result.removed.length > 0) {\n        await this.manager.deleteEntityRecordsByRef(result.removed);\n      }\n\n      await connection.applyMutation(result);\n      logger.debug(\n        `incremental-engine: ${providerName} processed delta from '${topic}' event`,\n      );\n    } else {\n      logger.debug(\n        `incremental-engine: ${providerName} ignored event from topic '${topic}'`,\n      );\n    }\n  }\n\n  supportsEventTopics(): string[] {\n    const { provider } = this.options;\n    const topics = provider.eventHandler\n      ? provider.eventHandler.supportsEventTopics()\n      : [];\n    return topics;\n  }\n}\n"],"names":["Duration","toError","stringifyError","performance","v4"],"mappings":";;;;;;;AA2BO,MAAM,0BAAA,CAAsD;AAAA,EASjE,YAAoB,OAAA,EAAiC;AAAjC,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAClB,IAAA,IAAA,CAAK,UAAU,OAAA,CAAQ,OAAA;AACvB,IAAA,IAAA,CAAK,UAAA,GAAaA,cAAA,CAAS,UAAA,CAAW,OAAA,CAAQ,UAAU,CAAA;AACxD,IAAA,IAAA,CAAK,WAAA,GAAcA,cAAA,CAAS,UAAA,CAAW,OAAA,CAAQ,WAAW,CAAA;AAC1D,IAAA,IAAA,CAAK,OAAA,GAAU,QAAQ,OAAA,IAAW;AAAA,MAChC,EAAE,SAAS,CAAA,EAAE;AAAA,MACb,EAAE,SAAS,CAAA,EAAE;AAAA,MACb,EAAE,SAAS,EAAA,EAAG;AAAA,MACd,EAAE,OAAO,CAAA;AAAE,KACb;AAEA,IAAA,IAAA,CAAK,WAAA,GAAc,QAAQ,OAAA,CAAQ,WAAA;AAAA,MACjC,wCAAA;AAAA,MACA;AAAA,QACE,WAAA,EAAa,qDAAA;AAAA,QACb,IAAA,EAAM;AAAA;AACR,KACF;AACA,IAAA,IAAA,CAAK,aAAA,GAAgB,QAAQ,OAAA,CAAQ,WAAA;AAAA,MACnC,0CAAA;AAAA,MACA;AAAA,QACE,WAAA,EAAa,uDAAA;AAAA,QACb,IAAA,EAAM;AAAA;AACR,KACF;AAAA,EACF;AAAA,EAjCiB,UAAA;AAAA,EACA,WAAA;AAAA,EACA,OAAA;AAAA,EACA,WAAA;AAAA,EACA,aAAA;AAAA,EAET,OAAA;AAAA,EA6BR,MAAM,OAAO,MAAA,EAAqB;AAChC,IAAA,IAAI;AACF,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,YAAY,CAAA;AACtC,MAAA,MAAM,IAAA,CAAK,iBAAiB,MAAM,CAAA;AAAA,IACpC,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,KAAK,CAAA,CAAE,CAAA;AACpC,MAAA,MAAM,KAAA;AAAA,IACR,CAAA,SAAE;AACA,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,UAAU,CAAA;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,MAAM,iBAAiB,MAAA,EAAqB;AAC1C,IAAA,MAAM,KAAK,OAAA,CAAQ,KAAA;AAEnB,IAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,gBAAA,EAAiB;AAC3C,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,MAAM,EAAE,WAAA,EAAa,YAAA,EAAc,UAAA,EAAY,UAAS,GAAI,MAAA;AAE5D,MAAA,QAAQ,UAAA;AAAY,QAClB,KAAK,MAAA;AACH,UAAA,IAAI,IAAA,CAAK,GAAA,EAAI,GAAI,YAAA,EAAc;AAC7B,YAAA,MAAM,KAAK,OAAA,CAAQ,uBAAA;AAAA,cACjB,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA;AAAgB,aACxC;AACA,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,cAClB,iCAAiC,WAAW,CAAA,iDAAA;AAAA,aAC9C;AAEA,YAAA,IAAA,CAAK,WAAA,CAAY,MAAA,CAAO,IAAA,CAAK,GAAA,KAAQ,GAAA,EAAM;AAAA,cACzC,YAAA,EAAc,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA;AAAgB,aACrD,CAAA;AACD,YAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,mBAAA,CAAoB,WAAW,CAAA;AAAA,UACpD,CAAA,MAAO;AACL,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,wBAAA;AAAA,aAC/C;AAAA,UACF;AACA,UAAA;AAAA,QACF,KAAK,QAAA;AACH,UAAA,IAAI;AACF,YAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,mBAAA,CAAoB,WAAW,CAAA;AAClD,YAAA,MAAM,IAAA,GAAO,MAAM,IAAA,CAAK,cAAA,CAAe,aAAa,MAAM,CAAA;AAC1D,YAAA,IAAI,IAAA,EAAM;AACR,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,4CAAA,EAA+C,IAAA,CAAK,UAAA,CAAW,SAAS,CAAA;AAAA,eACvH;AACA,cAAA,IAAA,CAAK,aAAA,CAAc,MAAA,CAAO,IAAA,CAAK,GAAA,KAAQ,GAAA,EAAM;AAAA,gBAC3C,YAAA,EAAc,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAAA,gBACpD,MAAA,EAAQ;AAAA,eACT,CAAA;AACD,cAAA,MAAM,KAAK,OAAA,CAAQ,kBAAA;AAAA,gBACjB,WAAA;AAAA,gBACA,IAAA,CAAK;AAAA,eACP;AAAA,YACF,CAAA,MAAO;AACL,cAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,uBAAA,CAAwB,WAAW,CAAA;AACtD,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,YAAA;AAAA,eAC/C;AAAA,YACF;AAAA,UACF,SAAS,KAAA,EAAO;AACd,YAAA,MAAM,GAAA,GAAMC,eAAQ,KAAK,CAAA;AACzB,YAAA,IAAI,GAAA,CAAI,YAAY,QAAA,EAAU;AAC5B,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,UAAA;AAAA,eAC/C;AACA,cAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,oBAAA,CAAqB,WAAA,EAAa,IAAI,OAAO,CAAA;AAAA,YAClE,CAAA,MAAO;AACL,cAAA,MAAM,iBAAiBD,cAAA,CAAS,UAAA;AAAA,gBAC9B,IAAA,CAAK,QAAQ,IAAA,CAAK,GAAA,CAAI,KAAK,OAAA,CAAQ,MAAA,GAAS,CAAA,EAAG,QAAQ,CAAC;AAAA,eAC1D;AAEA,cAAA,MAAM,aAAA,GAAgB,cAAA,CAAe,EAAA,CAAG,cAAc,CAAA;AACtD,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,QAAA,CAAA;AAAA,gBAC7C;AAAA,eACF;AAEA,cAAA,MAAM,iBAAiBE,qBAAA,CAAe,KAAK,CAAA,CAAE,SAAA,CAAU,GAAG,GAAG,CAAA;AAC7D,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,oEAAA,EAAuE,eAAe,OAAA,EAAS,KAAK,cAAc,CAAA,CAAA;AAAA,eACjK;AACA,cAAA,IAAA,CAAK,aAAA,CAAc,MAAA,CAAO,IAAA,CAAK,GAAA,KAAQ,GAAA,EAAM;AAAA,gBAC3C,YAAA,EAAc,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAAA,gBACpD,MAAA,EAAQ;AAAA,eACT,CAAA;AAED,cAAA,MAAM,KAAK,OAAA,CAAQ,kBAAA;AAAA,gBACjB,WAAA;AAAA,gBACA,QAAA;AAAA,gBACA,KAAA;AAAA,gBACA;AAAA,eACF;AAAA,YACF;AAAA,UACF;AACA,UAAA;AAAA,QACF,KAAK,SAAA;AACH,UAAA,IAAI,IAAA,CAAK,GAAA,EAAI,GAAI,YAAA,EAAc;AAC7B,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,0CAAA;AAAA,aAC/C;AACA,YAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,oBAAA,CAAqB,WAAW,CAAA;AAAA,UACrD,CAAA,MAAO;AACL,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,oBAAA;AAAA,aAC/C;AAAA,UACF;AACA,UAAA;AAAA,QACF,KAAK,QAAA;AACH,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,YAClB,kCAAkC,WAAW,CAAA,yBAAA;AAAA,WAC/C;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,mBAAA,CAAoB,WAAW,CAAA;AAClD,UAAA;AAAA,QACF;AACE,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,WAAW,CAAA,2BAAA,EAA8B,UAAU,CAAA,CAAA;AAAA,WACvF;AAAA;AACJ,IACF,CAAA,MAAO;AACL,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,QAClB,CAAA,oFAAA,EAAuF,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,iBAAiB,CAAA,EAAA;AAAA,OAChI;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,gBAAA,GAAmB;AACvB,IAAA,MAAM,YAAA,GAAe,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAC3D,IAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,YAAY,CAAA;AACxE,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,QAClB,CAAA,6CAAA,EAAgD,OAAO,EAAE,CAAA,CAAA;AAAA,OAC3D;AACA,MAAA,OAAO;AAAA,QACL,aAAa,MAAA,CAAO,EAAA;AAAA,QACpB,YAAY,MAAA,CAAO,WAAA;AAAA,QACnB,UAAU,MAAA,CAAO,QAAA;AAAA,QACjB,YAAA,EAAc,MAAA,CAAO,cAAA,CAAe,OAAA;AAAQ,OAC9C;AAAA,IACF;AACA,IAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,6BAAA;AAAA,MAChC;AAAA,KACF;AACA,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,QAClB,CAAA,+CAAA,EAAkD,OAAO,WAAW,CAAA,CAAA;AAAA,OACtE;AAAA,IACF;AACA,IAAA,OAAO,MAAA;AAAA,EACT;AAAA,EAEA,MAAM,cAAA,CAAe,EAAA,EAAY,MAAA,EAAqB;AACpD,IAAA,MAAM,QAAA,GAAW,MAAM,IAAA,CAAK,OAAA,CAAQ,YAAY,EAAE,CAAA;AAElD,IAAA,MAAM,MAAA,GAAS,QAAA,GAAW,QAAA,CAAS,MAAA,GAAS,MAAA;AAC5C,IAAA,IAAI,QAAA,GAAW,QAAA,GAAW,QAAA,CAAS,QAAA,GAAW,CAAA,GAAI,CAAA;AAElD,IAAA,MAAM,KAAA,GAAQC,4BAAY,GAAA,EAAI;AAC9B,IAAA,IAAI,KAAA,GAAQ,CAAA;AACZ,IAAA,IAAI,IAAA,GAAO,KAAA;AACX,IAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,MAClB,kCAAkC,EAAE,CAAA,iBAAA;AAAA,KACtC;AAEA,IAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,MAAA,CAAO,OAAO,OAAA,KAAqB;AAC7D,MAAA,IAAI,OAAO,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,IAAA,CAAK,SAAS,MAAM,CAAA;AAC3D,MAAA,KAAA,EAAA;AACA,MAAA,WAAS;AACP,QAAA,IAAA,GAAO,IAAA,CAAK,IAAA;AACZ,QAAA,MAAM,KAAK,IAAA,CAAK;AAAA,UACd,EAAA;AAAA,UACA,QAAA;AAAA,UACA,UAAU,IAAA,EAAM,QAAA;AAAA,UAChB,MAAM,IAAA,CAAK,IAAA;AAAA,UACX,QAAQ,IAAA,EAAM;AAAA,SACf,CAAA;AACD,QAAA,IAAI,MAAA,CAAO,OAAA,IAAW,IAAA,CAAK,IAAA,EAAM;AAC/B,UAAA;AAAA,QACF,CAAA,MAAA,IACEA,4BAAY,GAAA,EAAI,GAAI,QACpB,IAAA,CAAK,WAAA,CAAY,EAAA,CAAG,cAAc,CAAA,EAClC;AACA,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,YAClB,kCAAkC,EAAE,CAAA,qBAAA,EAAwB,IAAA,CAAK,WAAA,CAAY,SAAS,CAAA,CAAA;AAAA,WACxF;AACA,UAAA;AAAA,QACF,CAAA,MAAO;AACL,UAAA,IAAA,GAAO,MAAM,IAAA,CAAK,OAAA,CAAQ,SAAS,IAAA,CAAK,OAAA,EAAS,KAAK,MAAM,CAAA;AAC5D,UAAA,KAAA,EAAA;AACA,UAAA,QAAA,EAAA;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAED,IAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,MAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,mBAAA,EAAsB,KAAK,eAAe,IAAA,CAAK,KAAA;AAAA,QACjFA,2BAAA,CAAY,KAAI,GAAI;AAAA,OACrB,CAAA,IAAA;AAAA,KACH;AACA,IAAA,OAAO,IAAA;AAAA,EACT;AAAA,EAEA,MAAM,KAAK,OAAA,EAMR;AACD,IAAA,MAAM,EAAE,EAAA,EAAI,QAAA,EAAU,QAAA,EAAU,IAAA,EAAM,QAAO,GAAI,OAAA;AACjD,IAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,MAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,QAAA,EAClC,QAAA,GAAW,SAAS,MAAA,GAAS,CAC/B,CAAA,mBAAA,EACE,MAAA,GAAS,KAAK,SAAA,CAAU,MAAM,CAAA,GAAI,MACpC,WAAW,IAAI,CAAA;AAAA,KACjB;AACA,IAAA,MAAM,SAASC,sBAAA,EAAG;AAElB,IAAA,MAAM,IAAA,CAAK,QAAQ,UAAA,CAAW;AAAA,MAC5B,MAAA,EAAQ;AAAA,QACN,EAAA,EAAI,MAAA;AAAA,QACJ,YAAA,EAAc,EAAA;AAAA,QACd,MAAA;AAAA,QACA;AAAA;AACF,KACD,CAAA;AAED,IAAA,IAAI,QAAA,IAAY,QAAA,CAAS,MAAA,GAAS,CAAA,EAAG;AACnC,MAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,kBAAA,CAAmB,MAAA,EAAQ,QAAQ,CAAA;AAAA,IACxD;AAEA,IAAA,MAAM,KAAA,GACJ,QAAA,EAAU,GAAA,CAAI,CAAA,QAAA,MAAa;AAAA,MACzB,GAAG,QAAA;AAAA,MACH,MAAA,EAAQ;AAAA,QACN,GAAG,QAAA,CAAS,MAAA;AAAA,QACZ,QAAA,EAAU;AAAA,UACR,GAAG,SAAS,MAAA,CAAO,QAAA;AAAA,UACnB,WAAA,EAAa;AAAA,YACX,GAAG,QAAA,CAAS,MAAA,CAAO,QAAA,CAAS;AAAA;AAC9B;AACF;AACF,KACF,CAAE,KAAK,EAAC;AAEV,IAAA,MAAM,UAAmC,EAAC;AAE1C,IAAA,IAAI,IAAA,EAAM;AACR,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,QAClB,kCAAkC,EAAE,CAAA,mDAAA;AAAA,OACtC;AACA,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,cAAA;AAAA,QAChC,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAAA,QACtC;AAAA,OACF;AAEA,MAAA,MAAM,EAAE,OAAM,GAAI,MAAA;AAElB,MAAA,IAAI,SAAA,GAAY,IAAA;AAChB,MAAA,IAAI,IAAA,CAAK,QAAQ,4BAAA,EAA8B;AAC7C,QAAA,IAAI,UAAU,CAAA,EAAG;AACf,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,YAClB,kCAAkC,EAAE,CAAA,qCAAA;AAAA,WACtC;AACA,UAAA,SAAA,GAAY,KAAA;AAAA,QACd;AAAA,MACF;AAEA,MAAA,IAAI,IAAA,CAAK,QAAQ,6BAAA,EAA+B;AAG9C,QAAA,MAAM,iBACJ,KAAA,GAAQ,CAAA,GAAK,OAAO,OAAA,CAAQ,MAAA,GAAS,QAAS,GAAA,GAAM,GAAA;AACtD,QAAA,IAAI,cAAA,IAAkB,IAAA,CAAK,OAAA,CAAQ,6BAAA,EAA+B;AAChE,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,YAAA,EAAe,MAAA,CAAO,QAAQ,MAAM,CAAA,sCAAA;AAAA,WAC1E;AAAA,QACF,CAAA,MAAO;AACL,UAAA,MAAM,MAAA,GAAS,uBAAuB,cAAc,CAAA,uBAAA,CAAA;AACpD,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,GAAA,EAAM,MAAM,CAAA;AAAA,WAClD;AACA,UAAA,MAAM,IAAA,CAAK,QAAQ,yBAAA,CAA0B;AAAA,YAC3C,WAAA,EAAa,EAAA;AAAA,YACb,MAAA,EAAQ;AAAA,cACN,UAAA,EAAY,CAAA,6CAAA,EAAgD,MAAM,CAAA,EAAA,EAAK,MAAM,CAAA;AAAA;AAC/E,WACD,CAAA;AACD,UAAA,SAAA,GAAY,KAAA;AAAA,QACd;AAAA,MACF;AACA,MAAA,IAAI,SAAA,EAAW;AACb,QAAA,KAAA,MAAW,SAAA,IAAa,OAAO,OAAA,EAAS;AACtC,UAAA,OAAA,CAAQ,KAAK,SAAS,CAAA;AAAA,QACxB;AAAA,MACF;AAAA,IACF;AAEA,IAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,UAAA,CAAW,aAAA,CAAc;AAAA,MAC1C,IAAA,EAAM,OAAA;AAAA,MACN,KAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ,MAAA,EAAoC;AAChD,IAAA,MAAM,EAAE,OAAM,GAAI,MAAA;AAClB,IAAA,IAAI,CAAC,IAAA,CAAK,mBAAA,EAAoB,CAAE,QAAA,CAAS,KAAK,CAAA,EAAG;AAC/C,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,EAAE,MAAA,EAAQ,QAAA,EAAU,UAAA,KAAe,IAAA,CAAK,OAAA;AAC9C,IAAA,MAAM,YAAA,GAAe,SAAS,eAAA,EAAgB;AAC9C,IAAA,MAAA,CAAO,KAAA,CAAM,CAAA,oBAAA,EAAuB,YAAY,CAAA,UAAA,EAAa,KAAK,CAAA,MAAA,CAAQ,CAAA;AAE1E,IAAA,IAAI,CAAC,SAAS,YAAA,EAAc;AAC1B,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,MAAA,GAAS,MAAM,QAAA,CAAS,YAAA,CAAa,QAAQ,MAAM,CAAA;AAEzD,IAAA,IAAI,MAAA,CAAO,SAAS,OAAA,EAAS;AAC3B,MAAA,IAAI,MAAA,CAAO,KAAA,CAAM,MAAA,GAAS,CAAA,EAAG;AAC3B,QAAA,MAAM,eAAA,GAAkB,MAAM,IAAA,CAAK,OAAA,CAAQ,yBAAA;AAAA,UACzC;AAAA,SACF;AAEA,QAAA,IAAI,CAAC,eAAA,EAAiB;AACpB,UAAA,MAAA,CAAO,KAAA;AAAA,YACL,uBAAuB,YAAY,CAAA,qEAAA;AAAA,WACrC;AAAA,QACF,CAAA,MAAO;AACL,UAAA,MAAM,OACJ,eAAA,CAAgB,MAAA,KAAW,SAAA,GACvB,MAAM,KAAK,OAAA,CAAQ,WAAA,CAAY,eAAA,CAAgB,EAAE,IACjD,MAAM,IAAA,CAAK,OAAA,CAAQ,YAAA,CAAa,gBAAgB,EAAE,CAAA;AAExD,UAAA,IAAI,CAAC,IAAA,EAAM;AACT,YAAA,MAAM,IAAI,KAAA;AAAA,cACR,yFAAyF,YAAY,CAAA,CAAA;AAAA,aACvG;AAAA,UACF;AACA,UAAA,MAAM,KAAK,OAAA,CAAQ,kBAAA,CAAmB,IAAA,CAAK,EAAA,EAAI,OAAO,KAAK,CAAA;AAAA,QAC7D;AAAA,MACF;AAEA,MAAA,IAAI,MAAA,CAAO,OAAA,CAAQ,MAAA,GAAS,CAAA,EAAG;AAC7B,QAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,wBAAA,CAAyB,MAAA,CAAO,OAAO,CAAA;AAAA,MAC5D;AAEA,MAAA,MAAM,UAAA,CAAW,cAAc,MAAM,CAAA;AACrC,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,CAAA,oBAAA,EAAuB,YAAY,CAAA,uBAAA,EAA0B,KAAK,CAAA,OAAA;AAAA,OACpE;AAAA,IACF,CAAA,MAAO;AACL,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,CAAA,oBAAA,EAAuB,YAAY,CAAA,2BAAA,EAA8B,KAAK,CAAA,CAAA;AAAA,OACxE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,mBAAA,GAAgC;AAC9B,IAAA,MAAM,EAAE,QAAA,EAAS,GAAI,IAAA,CAAK,OAAA;AAC1B,IAAA,MAAM,SAAS,QAAA,CAAS,YAAA,GACpB,SAAS,YAAA,CAAa,mBAAA,KACtB,EAAC;AACL,IAAA,OAAO,MAAA;AAAA,EACT;AACF;;;;"}
+{"version":3,"file":"IncrementalIngestionEngine.cjs.js","sources":["../../src/engine/IncrementalIngestionEngine.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport type { DeferredEntity } from '@backstage/plugin-catalog-node';\nimport { MetricsServiceGauge } from '@backstage/backend-plugin-api/alpha';\nimport { IterationEngine, IterationEngineOptions } from '../types';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { performance } from 'node:perf_hooks';\nimport { Duration } from 'luxon';\nimport { randomUUID as v4 } from 'node:crypto';\nimport { stringifyError, toError } from '@backstage/errors';\nimport { EventParams } from '@backstage/plugin-events-node';\nimport { HumanDuration } from '@backstage/types';\n\nexport class IncrementalIngestionEngine implements IterationEngine {\n  private readonly restLength: Duration;\n  private readonly burstLength: Duration;\n  private readonly backoff: HumanDuration[];\n  private readonly lastStarted: MetricsServiceGauge;\n  private readonly lastCompleted: MetricsServiceGauge;\n\n  private manager: IncrementalIngestionDatabaseManager;\n\n  constructor(private options: IterationEngineOptions) {\n    this.manager = options.manager;\n    this.restLength = Duration.fromObject(options.restLength);\n    this.burstLength = Duration.fromObject(options.burstLength);\n    this.backoff = options.backoff ?? [\n      { minutes: 1 },\n      { minutes: 5 },\n      { minutes: 30 },\n      { hours: 3 },\n    ];\n\n    this.lastStarted = options.metrics.createGauge(\n      'catalog_incremental.ingestions.started',\n      {\n        description: 'Epoch timestamp when the ingestion was last started',\n        unit: 's',\n      },\n    );\n    this.lastCompleted = options.metrics.createGauge(\n      'catalog_incremental.ingestions.completed',\n      {\n        description: 'Epoch timestamp when the ingestion was last completed',\n        unit: 's',\n      },\n    );\n  }\n\n  async taskFn(signal: AbortSignal) {\n    try {\n      this.options.logger.debug('Begin tick');\n      await this.handleNextAction(signal);\n    } catch (error) {\n      this.options.logger.error(`${error}`);\n      throw error;\n    } finally {\n      this.options.logger.debug('End tick');\n    }\n  }\n\n  async handleNextAction(signal: AbortSignal) {\n    await this.options.ready;\n\n    const result = await this.getCurrentAction();\n    if (result) {\n      const { ingestionId, nextActionAt, nextAction, attempts } = result;\n\n      switch (nextAction) {\n        case 'rest':\n          if (Date.now() > nextActionAt) {\n            await this.manager.clearFinishedIngestions(\n              this.options.provider.getProviderName(),\n            );\n            this.options.logger.debug(\n              `incremental-engine: Ingestion ${ingestionId} rest period complete. Ingestion will start again`,\n            );\n\n            this.lastStarted.record(Date.now() / 1000, {\n              providerName: this.options.provider.getProviderName(),\n            });\n            await this.manager.setProviderComplete(ingestionId);\n          } else {\n            this.options.logger.debug(\n              `incremental-engine: Ingestion '${ingestionId}' rest period continuing`,\n            );\n          }\n          break;\n        case 'ingest':\n          try {\n            await this.manager.setProviderBursting(ingestionId);\n            const done = await this.ingestOneBurst(ingestionId, signal);\n            if (done) {\n              this.options.logger.info(\n                `incremental-engine: Ingestion '${ingestionId}' complete, transitioning to rest period of ${this.restLength.toHuman()}`,\n              );\n              this.lastCompleted.record(Date.now() / 1000, {\n                providerName: this.options.provider.getProviderName(),\n                status: 'completed',\n              });\n              await this.manager.setProviderResting(\n                ingestionId,\n                this.restLength,\n              );\n            } else {\n              await this.manager.setProviderInterstitial(ingestionId);\n              this.options.logger.debug(\n                `incremental-engine: Ingestion '${ingestionId}' continuing`,\n              );\n            }\n          } catch (error) {\n            const err = toError(error);\n            if (err.message === 'CANCEL') {\n              this.options.logger.info(\n                `incremental-engine: Ingestion '${ingestionId}' canceled`,\n              );\n              await this.manager.setProviderCanceling(ingestionId, err.message);\n            } else {\n              const currentBackoff = Duration.fromObject(\n                this.backoff[Math.min(this.backoff.length - 1, attempts)],\n              );\n\n              const backoffLength = currentBackoff.as('milliseconds');\n              this.options.logger.error(\n                `incremental-engine: Ingestion '${ingestionId}' failed`,\n                error,\n              );\n\n              const truncatedError = stringifyError(error).substring(0, 700);\n              this.options.logger.error(\n                `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`,\n              );\n              this.lastCompleted.record(Date.now() / 1000, {\n                providerName: this.options.provider.getProviderName(),\n                status: 'failed',\n              });\n\n              await this.manager.setProviderBackoff(\n                ingestionId,\n                attempts,\n                error as Error,\n                backoffLength,\n              );\n            }\n          }\n          break;\n        case 'backoff':\n          if (Date.now() > nextActionAt) {\n            this.options.logger.info(\n              `incremental-engine: Ingestion '${ingestionId}' backoff complete, will attempt to resume`,\n            );\n            await this.manager.setProviderIngesting(ingestionId);\n          } else {\n            this.options.logger.debug(\n              `incremental-engine: Ingestion '${ingestionId}' backoff continuing`,\n            );\n          }\n          break;\n        case 'cancel':\n          this.options.logger.info(\n            `incremental-engine: Ingestion '${ingestionId}' canceling, will restart`,\n          );\n          await this.manager.setProviderCanceled(ingestionId);\n          break;\n        default:\n          this.options.logger.error(\n            `incremental-engine: Ingestion '${ingestionId}' received unknown action '${nextAction}'`,\n          );\n      }\n    } else {\n      this.options.logger.error(\n        `incremental-engine: Engine tried to create duplicate ingestion record for provider '${this.options.provider.getProviderName()}'.`,\n      );\n    }\n  }\n\n  async getCurrentAction() {\n    const providerName = this.options.provider.getProviderName();\n    const record = await this.manager.getCurrentIngestionRecord(providerName);\n    if (record) {\n      this.options.logger.debug(\n        `incremental-engine: Ingestion record found: '${record.id}'`,\n      );\n      return {\n        ingestionId: record.id,\n        nextAction: record.next_action as 'rest' | 'ingest' | 'backoff',\n        attempts: record.attempts as number,\n        nextActionAt: record.next_action_at.valueOf() as number,\n      };\n    }\n    const result = await this.manager.createProviderIngestionRecord(\n      providerName,\n    );\n    if (result) {\n      this.options.logger.info(\n        `incremental-engine: Ingestion record created: '${result.ingestionId}'`,\n      );\n    }\n    return result;\n  }\n\n  async ingestOneBurst(id: string, signal: AbortSignal) {\n    const lastMark = await this.manager.getLastMark(id);\n\n    const cursor = lastMark ? lastMark.cursor : undefined;\n    let sequence = lastMark ? lastMark.sequence + 1 : 0;\n\n    const start = performance.now();\n    let count = 0;\n    let done = false;\n    this.options.logger.info(\n      `incremental-engine: Ingestion '${id}' burst initiated`,\n    );\n\n    await this.options.provider.around(async (context: unknown) => {\n      let next = await this.options.provider.next(context, cursor);\n      count++;\n      for (;;) {\n        done = next.done;\n        await this.mark({\n          id,\n          sequence,\n          entities: next?.entities,\n          done: next.done,\n          cursor: next?.cursor,\n        });\n        if (signal.aborted || next.done) {\n          break;\n        } else if (\n          performance.now() - start >\n          this.burstLength.as('milliseconds')\n        ) {\n          this.options.logger.info(\n            `incremental-engine: Ingestion '${id}' burst ending after ${this.burstLength.toHuman()}.`,\n          );\n          break;\n        } else {\n          next = await this.options.provider.next(context, next.cursor);\n          count++;\n          sequence++;\n        }\n      }\n    });\n\n    this.options.logger.info(\n      `incremental-engine: Ingestion '${id}' burst complete. (${count} batches in ${Math.round(\n        performance.now() - start,\n      )}ms).`,\n    );\n    return done;\n  }\n\n  async mark({\n    id,\n    sequence,\n    entities = [],\n    done,\n    cursor,\n  }: {\n    id: string;\n    sequence: number;\n    entities?: DeferredEntity[];\n    done: boolean;\n    cursor?: unknown;\n  }) {\n    this.options.logger.debug(\n      `incremental-engine: Ingestion '${id}': MARK ${\n        entities.length\n      } entities, cursor: ${\n        cursor ? JSON.stringify(cursor) : 'none'\n      }, done: ${done}`,\n    );\n    const markId = v4();\n\n    await this.manager.createMark({\n      record: {\n        id: markId,\n        ingestion_id: id,\n        cursor,\n        sequence,\n      },\n    });\n\n    await this.manager.createMarkEntities(\n      this.options.provider.getProviderName(),\n      entities,\n      markId,\n    );\n\n    const added = entities.map(deferred => ({\n      ...deferred,\n      entity: {\n        ...deferred.entity,\n        metadata: {\n          ...deferred.entity.metadata,\n          annotations: {\n            ...deferred.entity.metadata.annotations,\n          },\n        },\n      },\n    }));\n\n    const removed: { entityRef: string }[] = [];\n\n    if (done) {\n      this.options.logger.info(\n        `incremental-engine: Ingestion '${id}': Final page reached, calculating removed entities`,\n      );\n      const result = await this.manager.computeRemoved(\n        this.options.provider.getProviderName(),\n        id,\n      );\n\n      const { total } = result;\n\n      let doRemoval = true;\n      if (this.options.rejectEmptySourceCollections) {\n        if (total === 0) {\n          this.options.logger.error(\n            `incremental-engine: Ingestion '${id}': Rejecting empty entity collection!`,\n          );\n          doRemoval = false;\n        }\n      }\n\n      if (this.options.rejectRemovalsAbovePercentage) {\n        // If the total entities upserted in this ingestion is 0, then\n        // 100% of entities are stale and marked for removal.\n        const percentRemoved =\n          total > 0 ? (result.removed.length / total) * 100 : 100;\n        if (percentRemoved <= this.options.rejectRemovalsAbovePercentage) {\n          this.options.logger.info(\n            `incremental-engine: Ingestion '${id}': Removing ${result.removed.length} entities that have no matching assets`,\n          );\n        } else {\n          const notice = `Attempted to remove ${percentRemoved}% of matching entities!`;\n          this.options.logger.error(\n            `incremental-engine: Ingestion '${id}': ${notice}`,\n          );\n          await this.manager.updateIngestionRecordById({\n            ingestionId: id,\n            update: {\n              last_error: `REMOVAL_THRESHOLD exceeded on ingestion mark ${markId}: ${notice}`,\n            },\n          });\n          doRemoval = false;\n        }\n      }\n      if (doRemoval) {\n        for (const entityRef of result.removed) {\n          removed.push(entityRef);\n        }\n      }\n    }\n\n    await this.options.connection.applyMutation({\n      type: 'delta',\n      added,\n      removed,\n    });\n  }\n\n  async onEvent(params: EventParams): Promise<void> {\n    const { topic } = params;\n    if (!this.supportsEventTopics().includes(topic)) {\n      return;\n    }\n\n    const { logger, provider, connection } = this.options;\n    const providerName = provider.getProviderName();\n    logger.debug(`incremental-engine: ${providerName} received ${topic} event`);\n\n    if (!provider.eventHandler) {\n      return;\n    }\n\n    const result = await provider.eventHandler.onEvent(params);\n\n    if (result.type === 'ignored') {\n      logger.debug(\n        `incremental-engine: ${providerName} ignored event from topic '${topic}'`,\n      );\n      return;\n    }\n\n    if (result.added.length > 0) {\n      const ingestionRecord = await this.manager.getCurrentIngestionRecord(\n        providerName,\n      );\n\n      if (!ingestionRecord) {\n        logger.debug(\n          `incremental-engine: ${providerName} skipping delta addition because incremental ingestion is restarting.`,\n        );\n      } else {\n        const mark =\n          ingestionRecord.status === 'resting'\n            ? await this.manager.getLastMark(ingestionRecord.id)\n            : await this.manager.getFirstMark(ingestionRecord.id);\n\n        if (!mark) {\n          throw new Error(\n            `Cannot apply delta, page records are missing! Please re-run incremental ingestion for ${providerName}.`,\n          );\n        }\n        await this.manager.createMarkEntities(\n          providerName,\n          result.added,\n          mark.id,\n        );\n      }\n    }\n\n    await this.manager.deleteEntityRecordsByRef(providerName, result.removed);\n\n    await connection.applyMutation(result);\n    logger.debug(\n      `incremental-engine: ${providerName} processed delta from '${topic}' event`,\n    );\n  }\n\n  supportsEventTopics(): string[] {\n    const { provider } = this.options;\n    const topics = provider.eventHandler\n      ? provider.eventHandler.supportsEventTopics()\n      : [];\n    return topics;\n  }\n}\n"],"names":["Duration","toError","stringifyError","performance","v4"],"mappings":";;;;;;;AA2BO,MAAM,0BAAA,CAAsD;AAAA,EASjE,YAAoB,OAAA,EAAiC;AAAjC,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAClB,IAAA,IAAA,CAAK,UAAU,OAAA,CAAQ,OAAA;AACvB,IAAA,IAAA,CAAK,UAAA,GAAaA,cAAA,CAAS,UAAA,CAAW,OAAA,CAAQ,UAAU,CAAA;AACxD,IAAA,IAAA,CAAK,WAAA,GAAcA,cAAA,CAAS,UAAA,CAAW,OAAA,CAAQ,WAAW,CAAA;AAC1D,IAAA,IAAA,CAAK,OAAA,GAAU,QAAQ,OAAA,IAAW;AAAA,MAChC,EAAE,SAAS,CAAA,EAAE;AAAA,MACb,EAAE,SAAS,CAAA,EAAE;AAAA,MACb,EAAE,SAAS,EAAA,EAAG;AAAA,MACd,EAAE,OAAO,CAAA;AAAE,KACb;AAEA,IAAA,IAAA,CAAK,WAAA,GAAc,QAAQ,OAAA,CAAQ,WAAA;AAAA,MACjC,wCAAA;AAAA,MACA;AAAA,QACE,WAAA,EAAa,qDAAA;AAAA,QACb,IAAA,EAAM;AAAA;AACR,KACF;AACA,IAAA,IAAA,CAAK,aAAA,GAAgB,QAAQ,OAAA,CAAQ,WAAA;AAAA,MACnC,0CAAA;AAAA,MACA;AAAA,QACE,WAAA,EAAa,uDAAA;AAAA,QACb,IAAA,EAAM;AAAA;AACR,KACF;AAAA,EACF;AAAA,EAzBoB,OAAA;AAAA,EARH,UAAA;AAAA,EACA,WAAA;AAAA,EACA,OAAA;AAAA,EACA,WAAA;AAAA,EACA,aAAA;AAAA,EAET,OAAA;AAAA,EA6BR,MAAM,OAAO,MAAA,EAAqB;AAChC,IAAA,IAAI;AACF,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,YAAY,CAAA;AACtC,MAAA,MAAM,IAAA,CAAK,iBAAiB,MAAM,CAAA;AAAA,IACpC,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,KAAK,CAAA,CAAE,CAAA;AACpC,MAAA,MAAM,KAAA;AAAA,IACR,CAAA,SAAE;AACA,MAAA,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,UAAU,CAAA;AAAA,IACtC;AAAA,EACF;AAAA,EAEA,MAAM,iBAAiB,MAAA,EAAqB;AAC1C,IAAA,MAAM,KAAK,OAAA,CAAQ,KAAA;AAEnB,IAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,gBAAA,EAAiB;AAC3C,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,MAAM,EAAE,WAAA,EAAa,YAAA,EAAc,UAAA,EAAY,UAAS,GAAI,MAAA;AAE5D,MAAA,QAAQ,UAAA;AAAY,QAClB,KAAK,MAAA;AACH,UAAA,IAAI,IAAA,CAAK,GAAA,EAAI,GAAI,YAAA,EAAc;AAC7B,YAAA,MAAM,KAAK,OAAA,CAAQ,uBAAA;AAAA,cACjB,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA;AAAgB,aACxC;AACA,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,cAClB,iCAAiC,WAAW,CAAA,iDAAA;AAAA,aAC9C;AAEA,YAAA,IAAA,CAAK,WAAA,CAAY,MAAA,CAAO,IAAA,CAAK,GAAA,KAAQ,GAAA,EAAM;AAAA,cACzC,YAAA,EAAc,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA;AAAgB,aACrD,CAAA;AACD,YAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,mBAAA,CAAoB,WAAW,CAAA;AAAA,UACpD,CAAA,MAAO;AACL,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,wBAAA;AAAA,aAC/C;AAAA,UACF;AACA,UAAA;AAAA,QACF,KAAK,QAAA;AACH,UAAA,IAAI;AACF,YAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,mBAAA,CAAoB,WAAW,CAAA;AAClD,YAAA,MAAM,IAAA,GAAO,MAAM,IAAA,CAAK,cAAA,CAAe,aAAa,MAAM,CAAA;AAC1D,YAAA,IAAI,IAAA,EAAM;AACR,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,4CAAA,EAA+C,IAAA,CAAK,UAAA,CAAW,SAAS,CAAA;AAAA,eACvH;AACA,cAAA,IAAA,CAAK,aAAA,CAAc,MAAA,CAAO,IAAA,CAAK,GAAA,KAAQ,GAAA,EAAM;AAAA,gBAC3C,YAAA,EAAc,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAAA,gBACpD,MAAA,EAAQ;AAAA,eACT,CAAA;AACD,cAAA,MAAM,KAAK,OAAA,CAAQ,kBAAA;AAAA,gBACjB,WAAA;AAAA,gBACA,IAAA,CAAK;AAAA,eACP;AAAA,YACF,CAAA,MAAO;AACL,cAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,uBAAA,CAAwB,WAAW,CAAA;AACtD,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,YAAA;AAAA,eAC/C;AAAA,YACF;AAAA,UACF,SAAS,KAAA,EAAO;AACd,YAAA,MAAM,GAAA,GAAMC,eAAQ,KAAK,CAAA;AACzB,YAAA,IAAI,GAAA,CAAI,YAAY,QAAA,EAAU;AAC5B,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,UAAA;AAAA,eAC/C;AACA,cAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,oBAAA,CAAqB,WAAA,EAAa,IAAI,OAAO,CAAA;AAAA,YAClE,CAAA,MAAO;AACL,cAAA,MAAM,iBAAiBD,cAAA,CAAS,UAAA;AAAA,gBAC9B,IAAA,CAAK,QAAQ,IAAA,CAAK,GAAA,CAAI,KAAK,OAAA,CAAQ,MAAA,GAAS,CAAA,EAAG,QAAQ,CAAC;AAAA,eAC1D;AAEA,cAAA,MAAM,aAAA,GAAgB,cAAA,CAAe,EAAA,CAAG,cAAc,CAAA;AACtD,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,QAAA,CAAA;AAAA,gBAC7C;AAAA,eACF;AAEA,cAAA,MAAM,iBAAiBE,qBAAA,CAAe,KAAK,CAAA,CAAE,SAAA,CAAU,GAAG,GAAG,CAAA;AAC7D,cAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,gBAClB,kCAAkC,WAAW,CAAA,oEAAA,EAAuE,eAAe,OAAA,EAAS,KAAK,cAAc,CAAA,CAAA;AAAA,eACjK;AACA,cAAA,IAAA,CAAK,aAAA,CAAc,MAAA,CAAO,IAAA,CAAK,GAAA,KAAQ,GAAA,EAAM;AAAA,gBAC3C,YAAA,EAAc,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAAA,gBACpD,MAAA,EAAQ;AAAA,eACT,CAAA;AAED,cAAA,MAAM,KAAK,OAAA,CAAQ,kBAAA;AAAA,gBACjB,WAAA;AAAA,gBACA,QAAA;AAAA,gBACA,KAAA;AAAA,gBACA;AAAA,eACF;AAAA,YACF;AAAA,UACF;AACA,UAAA;AAAA,QACF,KAAK,SAAA;AACH,UAAA,IAAI,IAAA,CAAK,GAAA,EAAI,GAAI,YAAA,EAAc;AAC7B,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,0CAAA;AAAA,aAC/C;AACA,YAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,oBAAA,CAAqB,WAAW,CAAA;AAAA,UACrD,CAAA,MAAO;AACL,YAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,cAClB,kCAAkC,WAAW,CAAA,oBAAA;AAAA,aAC/C;AAAA,UACF;AACA,UAAA;AAAA,QACF,KAAK,QAAA;AACH,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,YAClB,kCAAkC,WAAW,CAAA,yBAAA;AAAA,WAC/C;AACA,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,mBAAA,CAAoB,WAAW,CAAA;AAClD,UAAA;AAAA,QACF;AACE,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,WAAW,CAAA,2BAAA,EAA8B,UAAU,CAAA,CAAA;AAAA,WACvF;AAAA;AACJ,IACF,CAAA,MAAO;AACL,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,QAClB,CAAA,oFAAA,EAAuF,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,iBAAiB,CAAA,EAAA;AAAA,OAChI;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,gBAAA,GAAmB;AACvB,IAAA,MAAM,YAAA,GAAe,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAC3D,IAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,YAAY,CAAA;AACxE,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,QAClB,CAAA,6CAAA,EAAgD,OAAO,EAAE,CAAA,CAAA;AAAA,OAC3D;AACA,MAAA,OAAO;AAAA,QACL,aAAa,MAAA,CAAO,EAAA;AAAA,QACpB,YAAY,MAAA,CAAO,WAAA;AAAA,QACnB,UAAU,MAAA,CAAO,QAAA;AAAA,QACjB,YAAA,EAAc,MAAA,CAAO,cAAA,CAAe,OAAA;AAAQ,OAC9C;AAAA,IACF;AACA,IAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,6BAAA;AAAA,MAChC;AAAA,KACF;AACA,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,QAClB,CAAA,+CAAA,EAAkD,OAAO,WAAW,CAAA,CAAA;AAAA,OACtE;AAAA,IACF;AACA,IAAA,OAAO,MAAA;AAAA,EACT;AAAA,EAEA,MAAM,cAAA,CAAe,EAAA,EAAY,MAAA,EAAqB;AACpD,IAAA,MAAM,QAAA,GAAW,MAAM,IAAA,CAAK,OAAA,CAAQ,YAAY,EAAE,CAAA;AAElD,IAAA,MAAM,MAAA,GAAS,QAAA,GAAW,QAAA,CAAS,MAAA,GAAS,MAAA;AAC5C,IAAA,IAAI,QAAA,GAAW,QAAA,GAAW,QAAA,CAAS,QAAA,GAAW,CAAA,GAAI,CAAA;AAElD,IAAA,MAAM,KAAA,GAAQC,4BAAY,GAAA,EAAI;AAC9B,IAAA,IAAI,KAAA,GAAQ,CAAA;AACZ,IAAA,IAAI,IAAA,GAAO,KAAA;AACX,IAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,MAClB,kCAAkC,EAAE,CAAA,iBAAA;AAAA,KACtC;AAEA,IAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,MAAA,CAAO,OAAO,OAAA,KAAqB;AAC7D,MAAA,IAAI,OAAO,MAAM,IAAA,CAAK,QAAQ,QAAA,CAAS,IAAA,CAAK,SAAS,MAAM,CAAA;AAC3D,MAAA,KAAA,EAAA;AACA,MAAA,WAAS;AACP,QAAA,IAAA,GAAO,IAAA,CAAK,IAAA;AACZ,QAAA,MAAM,KAAK,IAAA,CAAK;AAAA,UACd,EAAA;AAAA,UACA,QAAA;AAAA,UACA,UAAU,IAAA,EAAM,QAAA;AAAA,UAChB,MAAM,IAAA,CAAK,IAAA;AAAA,UACX,QAAQ,IAAA,EAAM;AAAA,SACf,CAAA;AACD,QAAA,IAAI,MAAA,CAAO,OAAA,IAAW,IAAA,CAAK,IAAA,EAAM;AAC/B,UAAA;AAAA,QACF,CAAA,MAAA,IACEA,4BAAY,GAAA,EAAI,GAAI,QACpB,IAAA,CAAK,WAAA,CAAY,EAAA,CAAG,cAAc,CAAA,EAClC;AACA,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,YAClB,kCAAkC,EAAE,CAAA,qBAAA,EAAwB,IAAA,CAAK,WAAA,CAAY,SAAS,CAAA,CAAA;AAAA,WACxF;AACA,UAAA;AAAA,QACF,CAAA,MAAO;AACL,UAAA,IAAA,GAAO,MAAM,IAAA,CAAK,OAAA,CAAQ,SAAS,IAAA,CAAK,OAAA,EAAS,KAAK,MAAM,CAAA;AAC5D,UAAA,KAAA,EAAA;AACA,UAAA,QAAA,EAAA;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAED,IAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,MAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,mBAAA,EAAsB,KAAK,eAAe,IAAA,CAAK,KAAA;AAAA,QACjFA,2BAAA,CAAY,KAAI,GAAI;AAAA,OACrB,CAAA,IAAA;AAAA,KACH;AACA,IAAA,OAAO,IAAA;AAAA,EACT;AAAA,EAEA,MAAM,IAAA,CAAK;AAAA,IACT,EAAA;AAAA,IACA,QAAA;AAAA,IACA,WAAW,EAAC;AAAA,IACZ,IAAA;AAAA,IACA;AAAA,GACF,EAMG;AACD,IAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,MAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,QAAA,EAClC,QAAA,CAAS,MACX,CAAA,mBAAA,EACE,MAAA,GAAS,IAAA,CAAK,SAAA,CAAU,MAAM,CAAA,GAAI,MACpC,WAAW,IAAI,CAAA;AAAA,KACjB;AACA,IAAA,MAAM,SAASC,sBAAA,EAAG;AAElB,IAAA,MAAM,IAAA,CAAK,QAAQ,UAAA,CAAW;AAAA,MAC5B,MAAA,EAAQ;AAAA,QACN,EAAA,EAAI,MAAA;AAAA,QACJ,YAAA,EAAc,EAAA;AAAA,QACd,MAAA;AAAA,QACA;AAAA;AACF,KACD,CAAA;AAED,IAAA,MAAM,KAAK,OAAA,CAAQ,kBAAA;AAAA,MACjB,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAAA,MACtC,QAAA;AAAA,MACA;AAAA,KACF;AAEA,IAAA,MAAM,KAAA,GAAQ,QAAA,CAAS,GAAA,CAAI,CAAA,QAAA,MAAa;AAAA,MACtC,GAAG,QAAA;AAAA,MACH,MAAA,EAAQ;AAAA,QACN,GAAG,QAAA,CAAS,MAAA;AAAA,QACZ,QAAA,EAAU;AAAA,UACR,GAAG,SAAS,MAAA,CAAO,QAAA;AAAA,UACnB,WAAA,EAAa;AAAA,YACX,GAAG,QAAA,CAAS,MAAA,CAAO,QAAA,CAAS;AAAA;AAC9B;AACF;AACF,KACF,CAAE,CAAA;AAEF,IAAA,MAAM,UAAmC,EAAC;AAE1C,IAAA,IAAI,IAAA,EAAM;AACR,MAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,QAClB,kCAAkC,EAAE,CAAA,mDAAA;AAAA,OACtC;AACA,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,cAAA;AAAA,QAChC,IAAA,CAAK,OAAA,CAAQ,QAAA,CAAS,eAAA,EAAgB;AAAA,QACtC;AAAA,OACF;AAEA,MAAA,MAAM,EAAE,OAAM,GAAI,MAAA;AAElB,MAAA,IAAI,SAAA,GAAY,IAAA;AAChB,MAAA,IAAI,IAAA,CAAK,QAAQ,4BAAA,EAA8B;AAC7C,QAAA,IAAI,UAAU,CAAA,EAAG;AACf,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,YAClB,kCAAkC,EAAE,CAAA,qCAAA;AAAA,WACtC;AACA,UAAA,SAAA,GAAY,KAAA;AAAA,QACd;AAAA,MACF;AAEA,MAAA,IAAI,IAAA,CAAK,QAAQ,6BAAA,EAA+B;AAG9C,QAAA,MAAM,iBACJ,KAAA,GAAQ,CAAA,GAAK,OAAO,OAAA,CAAQ,MAAA,GAAS,QAAS,GAAA,GAAM,GAAA;AACtD,QAAA,IAAI,cAAA,IAAkB,IAAA,CAAK,OAAA,CAAQ,6BAAA,EAA+B;AAChE,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,IAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,YAAA,EAAe,MAAA,CAAO,QAAQ,MAAM,CAAA,sCAAA;AAAA,WAC1E;AAAA,QACF,CAAA,MAAO;AACL,UAAA,MAAM,MAAA,GAAS,uBAAuB,cAAc,CAAA,uBAAA,CAAA;AACpD,UAAA,IAAA,CAAK,QAAQ,MAAA,CAAO,KAAA;AAAA,YAClB,CAAA,+BAAA,EAAkC,EAAE,CAAA,GAAA,EAAM,MAAM,CAAA;AAAA,WAClD;AACA,UAAA,MAAM,IAAA,CAAK,QAAQ,yBAAA,CAA0B;AAAA,YAC3C,WAAA,EAAa,EAAA;AAAA,YACb,MAAA,EAAQ;AAAA,cACN,UAAA,EAAY,CAAA,6CAAA,EAAgD,MAAM,CAAA,EAAA,EAAK,MAAM,CAAA;AAAA;AAC/E,WACD,CAAA;AACD,UAAA,SAAA,GAAY,KAAA;AAAA,QACd;AAAA,MACF;AACA,MAAA,IAAI,SAAA,EAAW;AACb,QAAA,KAAA,MAAW,SAAA,IAAa,OAAO,OAAA,EAAS;AACtC,UAAA,OAAA,CAAQ,KAAK,SAAS,CAAA;AAAA,QACxB;AAAA,MACF;AAAA,IACF;AAEA,IAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,UAAA,CAAW,aAAA,CAAc;AAAA,MAC1C,IAAA,EAAM,OAAA;AAAA,MACN,KAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ,MAAA,EAAoC;AAChD,IAAA,MAAM,EAAE,OAAM,GAAI,MAAA;AAClB,IAAA,IAAI,CAAC,IAAA,CAAK,mBAAA,EAAoB,CAAE,QAAA,CAAS,KAAK,CAAA,EAAG;AAC/C,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,EAAE,MAAA,EAAQ,QAAA,EAAU,UAAA,KAAe,IAAA,CAAK,OAAA;AAC9C,IAAA,MAAM,YAAA,GAAe,SAAS,eAAA,EAAgB;AAC9C,IAAA,MAAA,CAAO,KAAA,CAAM,CAAA,oBAAA,EAAuB,YAAY,CAAA,UAAA,EAAa,KAAK,CAAA,MAAA,CAAQ,CAAA;AAE1E,IAAA,IAAI,CAAC,SAAS,YAAA,EAAc;AAC1B,MAAA;AAAA,IACF;AAEA,IAAA,MAAM,MAAA,GAAS,MAAM,QAAA,CAAS,YAAA,CAAa,QAAQ,MAAM,CAAA;AAEzD,IAAA,IAAI,MAAA,CAAO,SAAS,SAAA,EAAW;AAC7B,MAAA,MAAA,CAAO,KAAA;AAAA,QACL,CAAA,oBAAA,EAAuB,YAAY,CAAA,2BAAA,EAA8B,KAAK,CAAA,CAAA;AAAA,OACxE;AACA,MAAA;AAAA,IACF;AAEA,IAAA,IAAI,MAAA,CAAO,KAAA,CAAM,MAAA,GAAS,CAAA,EAAG;AAC3B,MAAA,MAAM,eAAA,GAAkB,MAAM,IAAA,CAAK,OAAA,CAAQ,yBAAA;AAAA,QACzC;AAAA,OACF;AAEA,MAAA,IAAI,CAAC,eAAA,EAAiB;AACpB,QAAA,MAAA,CAAO,KAAA;AAAA,UACL,uBAAuB,YAAY,CAAA,qEAAA;AAAA,SACrC;AAAA,MACF,CAAA,MAAO;AACL,QAAA,MAAM,OACJ,eAAA,CAAgB,MAAA,KAAW,SAAA,GACvB,MAAM,KAAK,OAAA,CAAQ,WAAA,CAAY,eAAA,CAAgB,EAAE,IACjD,MAAM,IAAA,CAAK,OAAA,CAAQ,YAAA,CAAa,gBAAgB,EAAE,CAAA;AAExD,QAAA,IAAI,CAAC,IAAA,EAAM;AACT,UAAA,MAAM,IAAI,KAAA;AAAA,YACR,yFAAyF,YAAY,CAAA,CAAA;AAAA,WACvG;AAAA,QACF;AACA,QAAA,MAAM,KAAK,OAAA,CAAQ,kBAAA;AAAA,UACjB,YAAA;AAAA,UACA,MAAA,CAAO,KAAA;AAAA,UACP,IAAA,CAAK;AAAA,SACP;AAAA,MACF;AAAA,IACF;AAEA,IAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,wBAAA,CAAyB,YAAA,EAAc,OAAO,OAAO,CAAA;AAExE,IAAA,MAAM,UAAA,CAAW,cAAc,MAAM,CAAA;AACrC,IAAA,MAAA,CAAO,KAAA;AAAA,MACL,CAAA,oBAAA,EAAuB,YAAY,CAAA,uBAAA,EAA0B,KAAK,CAAA,OAAA;AAAA,KACpE;AAAA,EACF;AAAA,EAEA,mBAAA,GAAgC;AAC9B,IAAA,MAAM,EAAE,QAAA,EAAS,GAAI,IAAA,CAAK,OAAA;AAC1B,IAAA,MAAM,SAAS,QAAA,CAAS,YAAA,GACpB,SAAS,YAAA,CAAa,mBAAA,KACtB,EAAC;AACL,IAAA,OAAO,MAAA;AAAA,EACT;AACF;;"}

package/dist/module/WrapperProviders.cjs.js

@@ -12,6 +12,7 @@ class WrapperProviders {
   constructor(options) {
     this.options = options;
   }
+  options;
   migrate;
   numberOfProvidersToConnect = 0;
   readySignal = types.createDeferred();

package/dist/module/WrapperProviders.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"WrapperProviders.cjs.js","sources":["../../src/module/WrapperProviders.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  LoggerService,\n  HttpAuthService,\n  PermissionsService,\n  RootConfigService,\n  SchedulerService,\n} from '@backstage/backend-plugin-api';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\nimport { stringifyError } from '@backstage/errors';\nimport {\n  EntityProvider,\n  EntityProviderConnection,\n} from '@backstage/plugin-catalog-node';\nimport { createDeferred } from '@backstage/types';\nimport express from 'express';\nimport { Knex } from 'knex';\nimport { Duration } from 'luxon';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { applyDatabaseMigrations } from '../database/migrations';\nimport { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine';\nimport { IncrementalProviderRouter } from '../router/routes';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '../types';\nimport { EventsService } from '@backstage/plugin-events-node';\n\n/**\n * Helps in the creation of the catalog entity providers that wrap the\n * incremental ones.\n */\nexport class WrapperProviders {\n  private migrate: Promise<void> | undefined;\n  private numberOfProvidersToConnect = 0;\n  private readonly readySignal = createDeferred();\n\n  constructor(\n    private readonly options: {\n      config: RootConfigService;\n      logger: LoggerService;\n      client: Knex;\n      scheduler: SchedulerService;\n      applyDatabaseMigrations?: typeof applyDatabaseMigrations;\n      events: EventsService;\n      metrics: MetricsService;\n      permissions: PermissionsService;\n      httpAuth: HttpAuthService;\n    },\n  ) {}\n\n  wrap(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    options: IncrementalEntityProviderOptions,\n  ): EntityProvider {\n    this.numberOfProvidersToConnect += 1;\n    return {\n      getProviderName: () => provider.getProviderName(),\n      connect: async connection => {\n        await this.startProvider(provider, options, connection);\n        this.numberOfProvidersToConnect -= 1;\n        if (this.numberOfProvidersToConnect === 0) {\n          this.readySignal.resolve();\n        }\n      },\n    };\n  }\n\n  adminRouter(): express.Router {\n    return new IncrementalProviderRouter(\n      new IncrementalIngestionDatabaseManager({ client: this.options.client }),\n      this.options.logger,\n      this.options.permissions,\n      this.options.httpAuth,\n    ).createRouter();\n  }\n\n  private async startProvider(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    providerOptions: IncrementalEntityProviderOptions,\n    connection: EntityProviderConnection,\n  ) {\n    const logger = this.options.logger.child({\n      entityProvider: provider.getProviderName(),\n    });\n\n    try {\n      if (!this.migrate) {\n        this.migrate = Promise.resolve().then(async () => {\n          const apply =\n            this.options.applyDatabaseMigrations ?? applyDatabaseMigrations;\n          await apply(this.options.client);\n        });\n      }\n\n      await this.migrate;\n\n      const { burstInterval, burstLength, restLength } = providerOptions;\n\n      logger.info(`Connecting`);\n\n      const manager = new IncrementalIngestionDatabaseManager({\n        client: this.options.client,\n      });\n      const engine = new IncrementalIngestionEngine({\n        ...providerOptions,\n        ready: this.readySignal,\n        manager,\n        logger,\n        provider,\n        restLength,\n        connection,\n        metrics: this.options.metrics,\n      });\n\n      let frequency = Duration.isDuration(burstInterval)\n        ? burstInterval\n        : Duration.fromObject(burstInterval);\n      if (frequency.as('milliseconds') < 5000) {\n        frequency = Duration.fromObject({ seconds: 5 }); // don't let it be silly low, to not overload the scheduler\n      }\n\n      let length = Duration.isDuration(burstLength)\n        ? burstLength\n        : Duration.fromObject(burstLength);\n      length = length.plus(Duration.fromObject({ minutes: 1 })); // some margin from the actual completion\n\n      await this.options.scheduler.scheduleTask({\n        id: provider.getProviderName(),\n        fn: engine.taskFn.bind(engine),\n        frequency,\n        timeout: length,\n      });\n\n      const topics = engine.supportsEventTopics();\n      if (topics.length > 0) {\n        logger.info(\n          `Provider ${provider.getProviderName()} subscribing to events for topics: ${topics.join(\n            ',',\n          )}`,\n        );\n        await this.options.events.subscribe({\n          topics,\n          id: `catalog-backend-module-incremental-ingestion:${provider.getProviderName()}`,\n          onEvent: evt => engine.onEvent(evt),\n        });\n      }\n    } catch (error) {\n      logger.warn(\n        `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${stringifyError(\n          error,\n        )}`,\n      );\n      throw error;\n    }\n  }\n}\n"],"names":["createDeferred","IncrementalProviderRouter","IncrementalIngestionDatabaseManager","applyDatabaseMigrations","IncrementalIngestionEngine","Duration","stringifyError"],"mappings":";;;;;;;;;;AA+CO,MAAM,gBAAA,CAAiB;AAAA,EAK5B,YACmB,OAAA,EAWjB;AAXiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA,EAWhB;AAAA,EAhBK,OAAA;AAAA,EACA,0BAAA,GAA6B,CAAA;AAAA,EACpB,cAAcA,oBAAA,EAAe;AAAA,EAgB9C,IAAA,CACE,UACA,OAAA,EACgB;AAChB,IAAA,IAAA,CAAK,0BAAA,IAA8B,CAAA;AACnC,IAAA,OAAO;AAAA,MACL,eAAA,EAAiB,MAAM,QAAA,CAAS,eAAA,EAAgB;AAAA,MAChD,OAAA,EAAS,OAAM,UAAA,KAAc;AAC3B,QAAA,MAAM,IAAA,CAAK,aAAA,CAAc,QAAA,EAAU,OAAA,EAAS,UAAU,CAAA;AACtD,QAAA,IAAA,CAAK,0BAAA,IAA8B,CAAA;AACnC,QAAA,IAAI,IAAA,CAAK,+BAA+B,CAAA,EAAG;AACzC,UAAA,IAAA,CAAK,YAAY,OAAA,EAAQ;AAAA,QAC3B;AAAA,MACF;AAAA,KACF;AAAA,EACF;AAAA,EAEA,WAAA,GAA8B;AAC5B,IAAA,OAAO,IAAIC,gCAAA;AAAA,MACT,IAAIC,uEAAA,CAAoC,EAAE,QAAQ,IAAA,CAAK,OAAA,CAAQ,QAAQ,CAAA;AAAA,MACvE,KAAK,OAAA,CAAQ,MAAA;AAAA,MACb,KAAK,OAAA,CAAQ,WAAA;AAAA,MACb,KAAK,OAAA,CAAQ;AAAA,MACb,YAAA,EAAa;AAAA,EACjB;AAAA,EAEA,MAAc,aAAA,CACZ,QAAA,EACA,eAAA,EACA,UAAA,EACA;AACA,IAAA,MAAM,MAAA,GAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM;AAAA,MACvC,cAAA,EAAgB,SAAS,eAAA;AAAgB,KAC1C,CAAA;AAED,IAAA,IAAI;AACF,MAAA,IAAI,CAAC,KAAK,OAAA,EAAS;AACjB,QAAA,IAAA,CAAK,OAAA,GAAU,OAAA,CAAQ,OAAA,EAAQ,CAAE,KAAK,YAAY;AAChD,UAAA,MAAM,KAAA,GACJ,IAAA,CAAK,OAAA,CAAQ,uBAAA,IAA2BC,kCAAA;AAC1C,UAAA,MAAM,KAAA,CAAM,IAAA,CAAK,OAAA,CAAQ,MAAM,CAAA;AAAA,QACjC,CAAC,CAAA;AAAA,MACH;AAEA,MAAA,MAAM,IAAA,CAAK,OAAA;AAEX,MAAA,MAAM,EAAE,aAAA,EAAe,WAAA,EAAa,UAAA,EAAW,GAAI,eAAA;AAEnD,MAAA,MAAA,CAAO,KAAK,CAAA,UAAA,CAAY,CAAA;AAExB,MAAA,MAAM,OAAA,GAAU,IAAID,uEAAA,CAAoC;AAAA,QACtD,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,MAAM,MAAA,GAAS,IAAIE,qDAAA,CAA2B;AAAA,QAC5C,GAAG,eAAA;AAAA,QACH,OAAO,IAAA,CAAK,WAAA;AAAA,QACZ,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,UAAA;AAAA,QACA,OAAA,EAAS,KAAK,OAAA,CAAQ;AAAA,OACvB,CAAA;AAED,MAAA,IAAI,SAAA,GAAYC,eAAS,UAAA,CAAW,aAAa,IAC7C,aAAA,GACAA,cAAA,CAAS,WAAW,aAAa,CAAA;AACrC,MAAA,IAAI,SAAA,CAAU,EAAA,CAAG,cAAc,CAAA,GAAI,GAAA,EAAM;AACvC,QAAA,SAAA,GAAYA,cAAA,CAAS,UAAA,CAAW,EAAE,OAAA,EAAS,GAAG,CAAA;AAAA,MAChD;AAEA,MAAA,IAAI,MAAA,GAASA,eAAS,UAAA,CAAW,WAAW,IACxC,WAAA,GACAA,cAAA,CAAS,WAAW,WAAW,CAAA;AACnC,MAAA,MAAA,GAAS,MAAA,CAAO,KAAKA,cAAA,CAAS,UAAA,CAAW,EAAE,OAAA,EAAS,CAAA,EAAG,CAAC,CAAA;AAExD,MAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa;AAAA,QACxC,EAAA,EAAI,SAAS,eAAA,EAAgB;AAAA,QAC7B,EAAA,EAAI,MAAA,CAAO,MAAA,CAAO,IAAA,CAAK,MAAM,CAAA;AAAA,QAC7B,SAAA;AAAA,QACA,OAAA,EAAS;AAAA,OACV,CAAA;AAED,MAAA,MAAM,MAAA,GAAS,OAAO,mBAAA,EAAoB;AAC1C,MAAA,IAAI,MAAA,CAAO,SAAS,CAAA,EAAG;AACrB,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,SAAA,EAAY,QAAA,CAAS,eAAA,EAAiB,sCAAsC,MAAA,CAAO,IAAA;AAAA,YACjF;AAAA,WACD,CAAA;AAAA,SACH;AACA,QAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,SAAA,CAAU;AAAA,UAClC,MAAA;AAAA,UACA,EAAA,EAAI,CAAA,6CAAA,EAAgD,QAAA,CAAS,eAAA,EAAiB,CAAA,CAAA;AAAA,UAC9E,OAAA,EAAS,CAAA,GAAA,KAAO,MAAA,CAAO,OAAA,CAAQ,GAAG;AAAA,SACnC,CAAA;AAAA,MACH;AAAA,IACF,SAAS,KAAA,EAAO;AACd,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,oDAAA,EAAuD,QAAA,CAAS,eAAA,EAAiB,CAAA,EAAA,EAAKC,qBAAA;AAAA,UACpF;AAAA,SACD,CAAA;AAAA,OACH;AACA,MAAA,MAAM,KAAA;AAAA,IACR;AAAA,EACF;AACF;;;;"}
+{"version":3,"file":"WrapperProviders.cjs.js","sources":["../../src/module/WrapperProviders.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  LoggerService,\n  HttpAuthService,\n  PermissionsService,\n  RootConfigService,\n  SchedulerService,\n} from '@backstage/backend-plugin-api';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\nimport { stringifyError } from '@backstage/errors';\nimport {\n  EntityProvider,\n  EntityProviderConnection,\n} from '@backstage/plugin-catalog-node';\nimport { createDeferred } from '@backstage/types';\nimport express from 'express';\nimport { Knex } from 'knex';\nimport { Duration } from 'luxon';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { applyDatabaseMigrations } from '../database/migrations';\nimport { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine';\nimport { IncrementalProviderRouter } from '../router/routes';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '../types';\nimport { EventsService } from '@backstage/plugin-events-node';\n\n/**\n * Helps in the creation of the catalog entity providers that wrap the\n * incremental ones.\n */\nexport class WrapperProviders {\n  private migrate: Promise<void> | undefined;\n  private numberOfProvidersToConnect = 0;\n  private readonly readySignal = createDeferred();\n\n  constructor(\n    private readonly options: {\n      config: RootConfigService;\n      logger: LoggerService;\n      client: Knex;\n      scheduler: SchedulerService;\n      applyDatabaseMigrations?: typeof applyDatabaseMigrations;\n      events: EventsService;\n      metrics: MetricsService;\n      permissions: PermissionsService;\n      httpAuth: HttpAuthService;\n    },\n  ) {}\n\n  wrap(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    options: IncrementalEntityProviderOptions,\n  ): EntityProvider {\n    this.numberOfProvidersToConnect += 1;\n    return {\n      getProviderName: () => provider.getProviderName(),\n      connect: async connection => {\n        await this.startProvider(provider, options, connection);\n        this.numberOfProvidersToConnect -= 1;\n        if (this.numberOfProvidersToConnect === 0) {\n          this.readySignal.resolve();\n        }\n      },\n    };\n  }\n\n  adminRouter(): express.Router {\n    return new IncrementalProviderRouter(\n      new IncrementalIngestionDatabaseManager({ client: this.options.client }),\n      this.options.logger,\n      this.options.permissions,\n      this.options.httpAuth,\n    ).createRouter();\n  }\n\n  private async startProvider(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    providerOptions: IncrementalEntityProviderOptions,\n    connection: EntityProviderConnection,\n  ) {\n    const logger = this.options.logger.child({\n      entityProvider: provider.getProviderName(),\n    });\n\n    try {\n      if (!this.migrate) {\n        this.migrate = Promise.resolve().then(async () => {\n          const apply =\n            this.options.applyDatabaseMigrations ?? applyDatabaseMigrations;\n          await apply(this.options.client);\n        });\n      }\n\n      await this.migrate;\n\n      const { burstInterval, burstLength, restLength } = providerOptions;\n\n      logger.info(`Connecting`);\n\n      const manager = new IncrementalIngestionDatabaseManager({\n        client: this.options.client,\n      });\n      const engine = new IncrementalIngestionEngine({\n        ...providerOptions,\n        ready: this.readySignal,\n        manager,\n        logger,\n        provider,\n        restLength,\n        connection,\n        metrics: this.options.metrics,\n      });\n\n      let frequency = Duration.isDuration(burstInterval)\n        ? burstInterval\n        : Duration.fromObject(burstInterval);\n      if (frequency.as('milliseconds') < 5000) {\n        frequency = Duration.fromObject({ seconds: 5 }); // don't let it be silly low, to not overload the scheduler\n      }\n\n      let length = Duration.isDuration(burstLength)\n        ? burstLength\n        : Duration.fromObject(burstLength);\n      length = length.plus(Duration.fromObject({ minutes: 1 })); // some margin from the actual completion\n\n      await this.options.scheduler.scheduleTask({\n        id: provider.getProviderName(),\n        fn: engine.taskFn.bind(engine),\n        frequency,\n        timeout: length,\n      });\n\n      const topics = engine.supportsEventTopics();\n      if (topics.length > 0) {\n        logger.info(\n          `Provider ${provider.getProviderName()} subscribing to events for topics: ${topics.join(\n            ',',\n          )}`,\n        );\n        await this.options.events.subscribe({\n          topics,\n          id: `catalog-backend-module-incremental-ingestion:${provider.getProviderName()}`,\n          onEvent: evt => engine.onEvent(evt),\n        });\n      }\n    } catch (error) {\n      logger.warn(\n        `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${stringifyError(\n          error,\n        )}`,\n      );\n      throw error;\n    }\n  }\n}\n"],"names":["createDeferred","IncrementalProviderRouter","IncrementalIngestionDatabaseManager","applyDatabaseMigrations","IncrementalIngestionEngine","Duration","stringifyError"],"mappings":";;;;;;;;;;AA+CO,MAAM,gBAAA,CAAiB;AAAA,EAK5B,YACmB,OAAA,EAWjB;AAXiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA,EAWhB;AAAA,EAXgB,OAAA;AAAA,EALX,OAAA;AAAA,EACA,0BAAA,GAA6B,CAAA;AAAA,EACpB,cAAcA,oBAAA,EAAe;AAAA,EAgB9C,IAAA,CACE,UACA,OAAA,EACgB;AAChB,IAAA,IAAA,CAAK,0BAAA,IAA8B,CAAA;AACnC,IAAA,OAAO;AAAA,MACL,eAAA,EAAiB,MAAM,QAAA,CAAS,eAAA,EAAgB;AAAA,MAChD,OAAA,EAAS,OAAM,UAAA,KAAc;AAC3B,QAAA,MAAM,IAAA,CAAK,aAAA,CAAc,QAAA,EAAU,OAAA,EAAS,UAAU,CAAA;AACtD,QAAA,IAAA,CAAK,0BAAA,IAA8B,CAAA;AACnC,QAAA,IAAI,IAAA,CAAK,+BAA+B,CAAA,EAAG;AACzC,UAAA,IAAA,CAAK,YAAY,OAAA,EAAQ;AAAA,QAC3B;AAAA,MACF;AAAA,KACF;AAAA,EACF;AAAA,EAEA,WAAA,GAA8B;AAC5B,IAAA,OAAO,IAAIC,gCAAA;AAAA,MACT,IAAIC,uEAAA,CAAoC,EAAE,QAAQ,IAAA,CAAK,OAAA,CAAQ,QAAQ,CAAA;AAAA,MACvE,KAAK,OAAA,CAAQ,MAAA;AAAA,MACb,KAAK,OAAA,CAAQ,WAAA;AAAA,MACb,KAAK,OAAA,CAAQ;AAAA,MACb,YAAA,EAAa;AAAA,EACjB;AAAA,EAEA,MAAc,aAAA,CACZ,QAAA,EACA,eAAA,EACA,UAAA,EACA;AACA,IAAA,MAAM,MAAA,GAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM;AAAA,MACvC,cAAA,EAAgB,SAAS,eAAA;AAAgB,KAC1C,CAAA;AAED,IAAA,IAAI;AACF,MAAA,IAAI,CAAC,KAAK,OAAA,EAAS;AACjB,QAAA,IAAA,CAAK,OAAA,GAAU,OAAA,CAAQ,OAAA,EAAQ,CAAE,KAAK,YAAY;AAChD,UAAA,MAAM,KAAA,GACJ,IAAA,CAAK,OAAA,CAAQ,uBAAA,IAA2BC,kCAAA;AAC1C,UAAA,MAAM,KAAA,CAAM,IAAA,CAAK,OAAA,CAAQ,MAAM,CAAA;AAAA,QACjC,CAAC,CAAA;AAAA,MACH;AAEA,MAAA,MAAM,IAAA,CAAK,OAAA;AAEX,MAAA,MAAM,EAAE,aAAA,EAAe,WAAA,EAAa,UAAA,EAAW,GAAI,eAAA;AAEnD,MAAA,MAAA,CAAO,KAAK,CAAA,UAAA,CAAY,CAAA;AAExB,MAAA,MAAM,OAAA,GAAU,IAAID,uEAAA,CAAoC;AAAA,QACtD,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,MAAM,MAAA,GAAS,IAAIE,qDAAA,CAA2B;AAAA,QAC5C,GAAG,eAAA;AAAA,QACH,OAAO,IAAA,CAAK,WAAA;AAAA,QACZ,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,UAAA;AAAA,QACA,OAAA,EAAS,KAAK,OAAA,CAAQ;AAAA,OACvB,CAAA;AAED,MAAA,IAAI,SAAA,GAAYC,eAAS,UAAA,CAAW,aAAa,IAC7C,aAAA,GACAA,cAAA,CAAS,WAAW,aAAa,CAAA;AACrC,MAAA,IAAI,SAAA,CAAU,EAAA,CAAG,cAAc,CAAA,GAAI,GAAA,EAAM;AACvC,QAAA,SAAA,GAAYA,cAAA,CAAS,UAAA,CAAW,EAAE,OAAA,EAAS,GAAG,CAAA;AAAA,MAChD;AAEA,MAAA,IAAI,MAAA,GAASA,eAAS,UAAA,CAAW,WAAW,IACxC,WAAA,GACAA,cAAA,CAAS,WAAW,WAAW,CAAA;AACnC,MAAA,MAAA,GAAS,MAAA,CAAO,KAAKA,cAAA,CAAS,UAAA,CAAW,EAAE,OAAA,EAAS,CAAA,EAAG,CAAC,CAAA;AAExD,MAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa;AAAA,QACxC,EAAA,EAAI,SAAS,eAAA,EAAgB;AAAA,QAC7B,EAAA,EAAI,MAAA,CAAO,MAAA,CAAO,IAAA,CAAK,MAAM,CAAA;AAAA,QAC7B,SAAA;AAAA,QACA,OAAA,EAAS;AAAA,OACV,CAAA;AAED,MAAA,MAAM,MAAA,GAAS,OAAO,mBAAA,EAAoB;AAC1C,MAAA,IAAI,MAAA,CAAO,SAAS,CAAA,EAAG;AACrB,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,SAAA,EAAY,QAAA,CAAS,eAAA,EAAiB,sCAAsC,MAAA,CAAO,IAAA;AAAA,YACjF;AAAA,WACD,CAAA;AAAA,SACH;AACA,QAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,SAAA,CAAU;AAAA,UAClC,MAAA;AAAA,UACA,EAAA,EAAI,CAAA,6CAAA,EAAgD,QAAA,CAAS,eAAA,EAAiB,CAAA,CAAA;AAAA,UAC9E,OAAA,EAAS,CAAA,GAAA,KAAO,MAAA,CAAO,OAAA,CAAQ,GAAG;AAAA,SACnC,CAAA;AAAA,MACH;AAAA,IACF,SAAS,KAAA,EAAO;AACd,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,oDAAA,EAAuD,QAAA,CAAS,eAAA,EAAiB,CAAA,EAAA,EAAKC,qBAAA;AAAA,UACpF;AAAA,SACD,CAAA;AAAA,OACH;AACA,MAAA,MAAM,KAAA;AAAA,IACR;AAAA,EACF;AACF;;"}

package/dist/module/catalogModuleIncrementalIngestionEntityProvider.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"catalogModuleIncrementalIngestionEntityProvider.cjs.js","sources":["../../src/module/catalogModuleIncrementalIngestionEntityProvider.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  coreServices,\n  createBackendModule,\n  createExtensionPoint,\n} from '@backstage/backend-plugin-api';\nimport { metricsServiceRef } from '@backstage/backend-plugin-api/alpha';\nimport { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node';\nimport { WrapperProviders } from './WrapperProviders';\nimport { eventsServiceRef } from '@backstage/plugin-events-node';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '../types';\n\n/**\n * @public\n * Interface for {@link incrementalIngestionProvidersExtensionPoint}.\n */\nexport interface IncrementalIngestionProviderExtensionPoint {\n  /** Adds a new incremental entity provider */\n  addProvider<TCursor, TContext>(config: {\n    options: IncrementalEntityProviderOptions;\n    provider: IncrementalEntityProvider<TCursor, TContext>;\n  }): void;\n}\n\n/**\n * @public\n *\n * Extension point for registering incremental ingestion providers.\n * The `catalogModuleIncrementalIngestionEntityProvider` must be installed for these providers to work.\n *\n * @example\n *\n * ```ts\n * backend.add(createBackendModule({\n *   pluginId: 'catalog',\n *   moduleId: 'my-incremental-provider',\n *   register(env) {\n *     env.registerInit({\n *       deps: {\n *         extension: incrementalIngestionProvidersExtensionPoint,\n *       },\n *       async init({ extension }) {\n *         extension.addProvider({\n *           burstInterval: ...,\n *           burstLength: ...,\n *           restLength: ...,\n *         }, {\n *           next(context, cursor) {\n *             ...\n *           },\n *           ...\n *         })\n *       })\n *     })\n *   }\n * }))\n * ```\n */\nexport const incrementalIngestionProvidersExtensionPoint =\n  createExtensionPoint<IncrementalIngestionProviderExtensionPoint>({\n    id: 'catalog.incrementalIngestionProvider.providers',\n  });\n\n/**\n * Registers the incremental entity provider with the catalog processing extension point.\n *\n * @public\n */\nexport const catalogModuleIncrementalIngestionEntityProvider =\n  createBackendModule({\n    pluginId: 'catalog',\n    moduleId: 'incremental-ingestion-entity-provider',\n    register(env) {\n      const addedProviders = new Array<{\n        provider: IncrementalEntityProvider<unknown, unknown>;\n        options: IncrementalEntityProviderOptions;\n      }>();\n\n      env.registerExtensionPoint(incrementalIngestionProvidersExtensionPoint, {\n        addProvider({ options, provider }) {\n          addedProviders.push({ options, provider });\n        },\n      });\n\n      env.registerInit({\n        deps: {\n          catalog: catalogProcessingExtensionPoint,\n          config: coreServices.rootConfig,\n          database: coreServices.database,\n          httpRouter: coreServices.httpRouter,\n          logger: coreServices.logger,\n          scheduler: coreServices.scheduler,\n          events: eventsServiceRef,\n          metrics: metricsServiceRef,\n          permissions: coreServices.permissions,\n          httpAuth: coreServices.httpAuth,\n        },\n        async init({\n          catalog,\n          config,\n          database,\n          httpRouter,\n          logger,\n          scheduler,\n          events,\n          metrics,\n          permissions,\n          httpAuth,\n        }) {\n          const client = await database.getClient();\n\n          const providers = new WrapperProviders({\n            config,\n            logger,\n            client,\n            scheduler,\n            events,\n            metrics,\n            permissions,\n            httpAuth,\n          });\n\n          for (const entry of addedProviders) {\n            const wrapped = providers.wrap(entry.provider, entry.options);\n            catalog.addEntityProvider(wrapped);\n          }\n\n          httpRouter.use(providers.adminRouter());\n        },\n      });\n    },\n  });\n"],"names":["createExtensionPoint","createBackendModule","catalogProcessingExtensionPoint","coreServices","eventsServiceRef","metricsServiceRef","WrapperProviders"],"mappings":";;;;;;;;AA4EO,MAAM,8CACXA,qCAAA,CAAiE;AAAA,EAC/D,EAAA,EAAI;AACN,CAAC;AAOI,MAAM,kDACXC,oCAAA,CAAoB;AAAA,EAClB,QAAA,EAAU,SAAA;AAAA,EACV,QAAA,EAAU,uCAAA;AAAA,EACV,SAAS,GAAA,EAAK;AACZ,IAAA,MAAM,cAAA,GAAiB,IAAI,KAAA,EAGxB;AAEH,IAAA,GAAA,CAAI,uBAAuB,2CAAA,EAA6C;AAAA,MACtE,WAAA,CAAY,EAAE,OAAA,EAAS,QAAA,EAAS,EAAG;AACjC,QAAA,cAAA,CAAe,IAAA,CAAK,EAAE,OAAA,EAAS,QAAA,EAAU,CAAA;AAAA,MAC3C;AAAA,KACD,CAAA;AAED,IAAA,GAAA,CAAI,YAAA,CAAa;AAAA,MACf,IAAA,EAAM;AAAA,QACJ,OAAA,EAASC,iDAAA;AAAA,QACT,QAAQC,6BAAA,CAAa,UAAA;AAAA,QACrB,UAAUA,6BAAA,CAAa,QAAA;AAAA,QACvB,YAAYA,6BAAA,CAAa,UAAA;AAAA,QACzB,QAAQA,6BAAA,CAAa,MAAA;AAAA,QACrB,WAAWA,6BAAA,CAAa,SAAA;AAAA,QACxB,MAAA,EAAQC,iCAAA;AAAA,QACR,OAAA,EAASC,uBAAA;AAAA,QACT,aAAaF,6BAAA,CAAa,WAAA;AAAA,QAC1B,UAAUA,6BAAA,CAAa;AAAA,OACzB;AAAA,MACA,MAAM,IAAA,CAAK;AAAA,QACT,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,MAAA;AAAA,QACA,SAAA;AAAA,QACA,MAAA;AAAA,QACA,OAAA;AAAA,QACA,WAAA;AAAA,QACA;AAAA,OACF,EAAG;AACD,QAAA,MAAM,MAAA,GAAS,MAAM,QAAA,CAAS,SAAA,EAAU;AAExC,QAAA,MAAM,SAAA,GAAY,IAAIG,iCAAA,CAAiB;AAAA,UACrC,MAAA;AAAA,UACA,MAAA;AAAA,UACA,MAAA;AAAA,UACA,SAAA;AAAA,UACA,MAAA;AAAA,UACA,OAAA;AAAA,UACA,WAAA;AAAA,UACA;AAAA,SACD,CAAA;AAED,QAAA,KAAA,MAAW,SAAS,cAAA,EAAgB;AAClC,UAAA,MAAM,UAAU,SAAA,CAAU,IAAA,CAAK,KAAA,CAAM,QAAA,EAAU,MAAM,OAAO,CAAA;AAC5D,UAAA,OAAA,CAAQ,kBAAkB,OAAO,CAAA;AAAA,QACnC;AAEA,QAAA,UAAA,CAAW,GAAA,CAAI,SAAA,CAAU,WAAA,EAAa,CAAA;AAAA,MACxC;AAAA,KACD,CAAA;AAAA,EACH;AACF,CAAC;;;;;"}
+{"version":3,"file":"catalogModuleIncrementalIngestionEntityProvider.cjs.js","sources":["../../src/module/catalogModuleIncrementalIngestionEntityProvider.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  coreServices,\n  createBackendModule,\n  createExtensionPoint,\n} from '@backstage/backend-plugin-api';\nimport { metricsServiceRef } from '@backstage/backend-plugin-api/alpha';\nimport { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node';\nimport { WrapperProviders } from './WrapperProviders';\nimport { eventsServiceRef } from '@backstage/plugin-events-node';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '../types';\n\n/**\n * @public\n * Interface for {@link incrementalIngestionProvidersExtensionPoint}.\n */\nexport interface IncrementalIngestionProviderExtensionPoint {\n  /** Adds a new incremental entity provider */\n  addProvider<TCursor, TContext>(config: {\n    options: IncrementalEntityProviderOptions;\n    provider: IncrementalEntityProvider<TCursor, TContext>;\n  }): void;\n}\n\n/**\n * @public\n *\n * Extension point for registering incremental ingestion providers.\n * The `catalogModuleIncrementalIngestionEntityProvider` must be installed for these providers to work.\n *\n * @example\n *\n * ```ts\n * backend.add(createBackendModule({\n *   pluginId: 'catalog',\n *   moduleId: 'my-incremental-provider',\n *   register(env) {\n *     env.registerInit({\n *       deps: {\n *         extension: incrementalIngestionProvidersExtensionPoint,\n *       },\n *       async init({ extension }) {\n *         extension.addProvider({\n *           burstInterval: ...,\n *           burstLength: ...,\n *           restLength: ...,\n *         }, {\n *           next(context, cursor) {\n *             ...\n *           },\n *           ...\n *         })\n *       })\n *     })\n *   }\n * }))\n * ```\n */\nexport const incrementalIngestionProvidersExtensionPoint =\n  createExtensionPoint<IncrementalIngestionProviderExtensionPoint>({\n    id: 'catalog.incrementalIngestionProvider.providers',\n  });\n\n/**\n * Registers the incremental entity provider with the catalog processing extension point.\n *\n * @public\n */\nexport const catalogModuleIncrementalIngestionEntityProvider =\n  createBackendModule({\n    pluginId: 'catalog',\n    moduleId: 'incremental-ingestion-entity-provider',\n    register(env) {\n      const addedProviders = new Array<{\n        provider: IncrementalEntityProvider<unknown, unknown>;\n        options: IncrementalEntityProviderOptions;\n      }>();\n\n      env.registerExtensionPoint(incrementalIngestionProvidersExtensionPoint, {\n        addProvider({ options, provider }) {\n          addedProviders.push({ options, provider });\n        },\n      });\n\n      env.registerInit({\n        deps: {\n          catalog: catalogProcessingExtensionPoint,\n          config: coreServices.rootConfig,\n          database: coreServices.database,\n          httpRouter: coreServices.httpRouter,\n          logger: coreServices.logger,\n          scheduler: coreServices.scheduler,\n          events: eventsServiceRef,\n          metrics: metricsServiceRef,\n          permissions: coreServices.permissions,\n          httpAuth: coreServices.httpAuth,\n        },\n        async init({\n          catalog,\n          config,\n          database,\n          httpRouter,\n          logger,\n          scheduler,\n          events,\n          metrics,\n          permissions,\n          httpAuth,\n        }) {\n          const client = await database.getClient();\n\n          const providers = new WrapperProviders({\n            config,\n            logger,\n            client,\n            scheduler,\n            events,\n            metrics,\n            permissions,\n            httpAuth,\n          });\n\n          for (const entry of addedProviders) {\n            const wrapped = providers.wrap(entry.provider, entry.options);\n            catalog.addEntityProvider(wrapped);\n          }\n\n          httpRouter.use(providers.adminRouter());\n        },\n      });\n    },\n  });\n"],"names":["createExtensionPoint","createBackendModule","catalogProcessingExtensionPoint","coreServices","eventsServiceRef","metricsServiceRef","WrapperProviders"],"mappings":";;;;;;;;AA4EO,MAAM,8CACXA,qCAAA,CAAiE;AAAA,EAC/D,EAAA,EAAI;AACN,CAAC;AAOI,MAAM,kDACXC,oCAAA,CAAoB;AAAA,EAClB,QAAA,EAAU,SAAA;AAAA,EACV,QAAA,EAAU,uCAAA;AAAA,EACV,SAAS,GAAA,EAAK;AACZ,IAAA,MAAM,cAAA,GAAiB,IAAI,KAAA,EAGxB;AAEH,IAAA,GAAA,CAAI,uBAAuB,2CAAA,EAA6C;AAAA,MACtE,WAAA,CAAY,EAAE,OAAA,EAAS,QAAA,EAAS,EAAG;AACjC,QAAA,cAAA,CAAe,IAAA,CAAK,EAAE,OAAA,EAAS,QAAA,EAAU,CAAA;AAAA,MAC3C;AAAA,KACD,CAAA;AAED,IAAA,GAAA,CAAI,YAAA,CAAa;AAAA,MACf,IAAA,EAAM;AAAA,QACJ,OAAA,EAASC,iDAAA;AAAA,QACT,QAAQC,6BAAA,CAAa,UAAA;AAAA,QACrB,UAAUA,6BAAA,CAAa,QAAA;AAAA,QACvB,YAAYA,6BAAA,CAAa,UAAA;AAAA,QACzB,QAAQA,6BAAA,CAAa,MAAA;AAAA,QACrB,WAAWA,6BAAA,CAAa,SAAA;AAAA,QACxB,MAAA,EAAQC,iCAAA;AAAA,QACR,OAAA,EAASC,uBAAA;AAAA,QACT,aAAaF,6BAAA,CAAa,WAAA;AAAA,QAC1B,UAAUA,6BAAA,CAAa;AAAA,OACzB;AAAA,MACA,MAAM,IAAA,CAAK;AAAA,QACT,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,MAAA;AAAA,QACA,SAAA;AAAA,QACA,MAAA;AAAA,QACA,OAAA;AAAA,QACA,WAAA;AAAA,QACA;AAAA,OACF,EAAG;AACD,QAAA,MAAM,MAAA,GAAS,MAAM,QAAA,CAAS,SAAA,EAAU;AAExC,QAAA,MAAM,SAAA,GAAY,IAAIG,iCAAA,CAAiB;AAAA,UACrC,MAAA;AAAA,UACA,MAAA;AAAA,UACA,MAAA;AAAA,UACA,SAAA;AAAA,UACA,MAAA;AAAA,UACA,OAAA;AAAA,UACA,WAAA;AAAA,UACA;AAAA,SACD,CAAA;AAED,QAAA,KAAA,MAAW,SAAS,cAAA,EAAgB;AAClC,UAAA,MAAM,UAAU,SAAA,CAAU,IAAA,CAAK,KAAA,CAAM,QAAA,EAAU,MAAM,OAAO,CAAA;AAC5D,UAAA,OAAA,CAAQ,kBAAkB,OAAO,CAAA;AAAA,QACnC;AAEA,QAAA,UAAA,CAAW,GAAA,CAAI,SAAA,CAAU,WAAA,EAAa,CAAA;AAAA,MACxC;AAAA,KACD,CAAA;AAAA,EACH;AACF,CAAC;;;"}

package/dist/router/routes.cjs.js

@@ -18,6 +18,8 @@ class IncrementalProviderRouter {
     this.manager = manager;
     this.logger = logger;
   }
+  permissions;
+  httpAuth;
   manager;
   logger;
   createRouter() {

package/dist/router/routes.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"routes.cjs.js","sources":["../../src/router/routes.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  HttpAuthService,\n  LoggerService,\n  PermissionsService,\n} from '@backstage/backend-plugin-api';\nimport { NotAllowedError } from '@backstage/errors';\nimport {\n  catalogIngestionManagePermission,\n  catalogIngestionReadPermission,\n} from '@backstage/plugin-catalog-common/alpha';\nimport { AuthorizeResult } from '@backstage/plugin-permission-common';\nimport express from 'express';\nimport Router from 'express-promise-router';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\n\nexport class IncrementalProviderRouter {\n  private manager: IncrementalIngestionDatabaseManager;\n  private logger: LoggerService;\n\n  constructor(\n    manager: IncrementalIngestionDatabaseManager,\n    logger: LoggerService,\n    private readonly permissions: PermissionsService,\n    private readonly httpAuth: HttpAuthService,\n  ) {\n    this.manager = manager;\n    this.logger = logger;\n  }\n\n  createRouter(): express.Router {\n    const router = Router();\n    router.use(express.json());\n    router.use('/incremental', async (req, _res, next) => {\n      const permission =\n        req.method === 'GET' || req.method === 'HEAD'\n          ? catalogIngestionReadPermission\n          : catalogIngestionManagePermission;\n      const [decision] = await this.permissions.authorize([{ permission }], {\n        credentials: await this.httpAuth.credentials(req),\n      });\n      if (decision.result === AuthorizeResult.DENY) {\n        throw new NotAllowedError('Unauthorized');\n      }\n      next();\n    });\n\n    // Get the overall health of all incremental providers\n    router.get('/incremental/health', async (_, res) => {\n      const records = await this.manager.healthcheck();\n      const providers = records.map(record => record.provider_name);\n      const duplicates = [\n        ...new Set(providers.filter((e, i, a) => a.indexOf(e) !== i)),\n      ];\n\n      if (duplicates.length > 0) {\n        res.json({ healthy: false, duplicateIngestions: duplicates });\n      } else {\n        res.json({ healthy: true });\n      }\n    });\n\n    // Clean up and pause all providers\n    router.post('/incremental/cleanup', async (_, res) => {\n      const result = await this.manager.cleanupProviders();\n      res.json(result);\n    });\n\n    // Get basic status of the provider\n    router.get('/incremental/providers/:provider', async (req, res) => {\n      const { provider } = req.params;\n      const record = await this.manager.getCurrentIngestionRecord(provider);\n      if (record) {\n        res.json({\n          success: true,\n          status: {\n            current_action: record.status,\n            next_action_at: new Date(record.next_action_at),\n          },\n          last_error: record.last_error,\n        });\n      } else {\n        const providers: string[] = await this.manager.listProviders();\n        if (providers.includes(provider)) {\n          res.json({\n            success: true,\n            status: {\n              current_action: 'rest complete, waiting to start',\n            },\n          });\n        } else {\n          this.logger.error(\n            `${provider} - No ingestion record found in the database!`,\n          );\n          res.status(404).json({\n            success: false,\n            status: {},\n            last_error: `Provider '${provider}' not found`,\n          });\n        }\n      }\n    });\n\n    // Trigger the provider's next action\n    router.post(\n      `/incremental/providers/:provider/trigger`,\n      async (req, res) => {\n        const { provider } = req.params;\n        const record = await this.manager.getCurrentIngestionRecord(provider);\n        if (record) {\n          await this.manager.triggerNextProviderAction(provider);\n          res.json({\n            success: true,\n            message: `${provider}: Next action triggered.`,\n          });\n        } else {\n          const providers: string[] = await this.manager.listProviders();\n          if (providers.includes(provider)) {\n            this.logger.debug(`${provider} - Ingestion record found`);\n            res.json({\n              success: true,\n              message: 'Unable to trigger next action (provider is restarting)',\n            });\n          } else {\n            res.status(404).json({\n              success: false,\n              message: `Provider '${provider}' not found`,\n            });\n          }\n        }\n      },\n    );\n\n    // Start a brand-new ingestion cycle for the provider.\n    // (Cancel's the current run if active, or marks it complete if resting)\n    router.post(`/incremental/providers/:provider/start`, async (req, res) => {\n      const { provider } = req.params;\n\n      const record = await this.manager.getCurrentIngestionRecord(provider);\n      if (record) {\n        const ingestionId = record.id;\n        if (record.status === 'resting') {\n          await this.manager.setProviderComplete(ingestionId);\n        } else {\n          await this.manager.setProviderCanceling(ingestionId);\n        }\n        res.json({\n          success: true,\n          message: `${provider}: Next cycle triggered.`,\n        });\n      } else {\n        const providers: string[] = await this.manager.listProviders();\n        if (providers.includes(provider)) {\n          this.logger.debug(`${provider} - Ingestion record found`);\n          res.json({\n            success: true,\n            message: 'Provider is already restarting',\n          });\n        } else {\n          res.status(404).json({\n            success: false,\n            message: `Provider '${provider}' not found`,\n          });\n        }\n      }\n    });\n\n    router.get(`/incremental/providers`, async (_req, res) => {\n      const providers = await this.manager.listProviders();\n\n      res.json({\n        success: true,\n        providers,\n      });\n    });\n\n    // Stop the provider and pause it for 24 hours\n    router.post(`/incremental/providers/:provider/cancel`, async (req, res) => {\n      const { provider } = req.params;\n      const record = await this.manager.getCurrentIngestionRecord(provider);\n      if (record) {\n        const next_action_at = new Date();\n        next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1000);\n        await this.manager.updateByName(provider, {\n          next_action: 'nothing (done)',\n          ingestion_completed_at: new Date(),\n          next_action_at,\n          status: 'resting',\n        });\n        res.json({\n          success: true,\n          message: `${provider}: Current ingestion canceled.`,\n        });\n      } else {\n        const providers: string[] = await this.manager.listProviders();\n        if (providers.includes(provider)) {\n          this.logger.debug(`${provider} - Ingestion record found`);\n          res.json({\n            success: true,\n            message: 'Provider is currently restarting, please wait.',\n          });\n        } else {\n          res.status(404).json({\n            success: false,\n            message: `Provider '${provider}' not found`,\n          });\n        }\n      }\n    });\n\n    // Wipe out all ingestion records for the provider and pause for 24 hours\n    router.delete('/incremental/providers/:provider', async (req, res) => {\n      const { provider } = req.params;\n      const result = await this.manager.purgeAndResetProvider(provider);\n      res.json(result);\n    });\n\n    // Get the ingestion marks for the current cycle\n    router.get(`/incremental/providers/:provider/marks`, async (req, res) => {\n      const { provider } = req.params;\n      const record = await this.manager.getCurrentIngestionRecord(provider);\n      if (record) {\n        const id = record.id;\n        const records = await this.manager.getAllMarks(id);\n        res.json({ success: true, records });\n      } else {\n        const providers: string[] = await this.manager.listProviders();\n        if (providers.includes(provider)) {\n          this.logger.debug(`${provider} - Ingestion record found`);\n          res.json({\n            success: true,\n            message: 'No records yet (provider is restarting)',\n          });\n        } else {\n          this.logger.error(\n            `${provider} - No ingestion record found in the database!`,\n          );\n          res.status(404).json({\n            success: false,\n            status: {},\n            last_error: `Provider '${provider}' not found`,\n          });\n        }\n      }\n    });\n\n    router.delete(\n      `/incremental/providers/:provider/marks`,\n      async (req, res) => {\n        const { provider } = req.params;\n        const deletions = await this.manager.clearFinishedIngestions(provider);\n\n        res.json({\n          success: true,\n          message: `Expired marks for provider '${provider}' removed.`,\n          deletions,\n        });\n      },\n    );\n\n    return router;\n  }\n}\n"],"names":["Router","express","catalogIngestionReadPermission","catalogIngestionManagePermission","AuthorizeResult","NotAllowedError"],"mappings":";;;;;;;;;;;;;AA+BO,MAAM,yBAAA,CAA0B;AAAA,EAIrC,WAAA,CACE,OAAA,EACA,MAAA,EACiB,WAAA,EACA,QAAA,EACjB;AAFiB,IAAA,IAAA,CAAA,WAAA,GAAA,WAAA;AACA,IAAA,IAAA,CAAA,QAAA,GAAA,QAAA;AAEjB,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AACf,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AAAA,EAChB;AAAA,EAXQ,OAAA;AAAA,EACA,MAAA;AAAA,EAYR,YAAA,GAA+B;AAC7B,IAAA,MAAM,SAASA,uBAAA,EAAO;AACtB,IAAA,MAAA,CAAO,GAAA,CAAIC,wBAAA,CAAQ,IAAA,EAAM,CAAA;AACzB,IAAA,MAAA,CAAO,GAAA,CAAI,cAAA,EAAgB,OAAO,GAAA,EAAK,MAAM,IAAA,KAAS;AACpD,MAAA,MAAM,aACJ,GAAA,CAAI,MAAA,KAAW,SAAS,GAAA,CAAI,MAAA,KAAW,SACnCC,oCAAA,GACAC,sCAAA;AACN,MAAA,MAAM,CAAC,QAAQ,CAAA,GAAI,MAAM,IAAA,CAAK,WAAA,CAAY,SAAA,CAAU,CAAC,EAAE,UAAA,EAAY,CAAA,EAAG;AAAA,QACpE,WAAA,EAAa,MAAM,IAAA,CAAK,QAAA,CAAS,YAAY,GAAG;AAAA,OACjD,CAAA;AACD,MAAA,IAAI,QAAA,CAAS,MAAA,KAAWC,sCAAA,CAAgB,IAAA,EAAM;AAC5C,QAAA,MAAM,IAAIC,uBAAgB,cAAc,CAAA;AAAA,MAC1C;AACA,MAAA,IAAA,EAAK;AAAA,IACP,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,GAAA,CAAI,qBAAA,EAAuB,OAAO,CAAA,EAAG,GAAA,KAAQ;AAClD,MAAA,MAAM,OAAA,GAAU,MAAM,IAAA,CAAK,OAAA,CAAQ,WAAA,EAAY;AAC/C,MAAA,MAAM,SAAA,GAAY,OAAA,CAAQ,GAAA,CAAI,CAAA,MAAA,KAAU,OAAO,aAAa,CAAA;AAC5D,MAAA,MAAM,UAAA,GAAa;AAAA,QACjB,GAAG,IAAI,GAAA,CAAI,SAAA,CAAU,OAAO,CAAC,CAAA,EAAG,CAAA,EAAG,CAAA,KAAM,CAAA,CAAE,OAAA,CAAQ,CAAC,CAAA,KAAM,CAAC,CAAC;AAAA,OAC9D;AAEA,MAAA,IAAI,UAAA,CAAW,SAAS,CAAA,EAAG;AACzB,QAAA,GAAA,CAAI,KAAK,EAAE,OAAA,EAAS,KAAA,EAAO,mBAAA,EAAqB,YAAY,CAAA;AAAA,MAC9D,CAAA,MAAO;AACL,QAAA,GAAA,CAAI,IAAA,CAAK,EAAE,OAAA,EAAS,IAAA,EAAM,CAAA;AAAA,MAC5B;AAAA,IACF,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,IAAA,CAAK,sBAAA,EAAwB,OAAO,CAAA,EAAG,GAAA,KAAQ;AACpD,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,gBAAA,EAAiB;AACnD,MAAA,GAAA,CAAI,KAAK,MAAM,CAAA;AAAA,IACjB,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,GAAA,CAAI,kCAAA,EAAoC,OAAO,GAAA,EAAK,GAAA,KAAQ;AACjE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,GAAA,CAAI,IAAA,CAAK;AAAA,UACP,OAAA,EAAS,IAAA;AAAA,UACT,MAAA,EAAQ;AAAA,YACN,gBAAgB,MAAA,CAAO,MAAA;AAAA,YACvB,cAAA,EAAgB,IAAI,IAAA,CAAK,MAAA,CAAO,cAAc;AAAA,WAChD;AAAA,UACA,YAAY,MAAA,CAAO;AAAA,SACpB,CAAA;AAAA,MACH,CAAA,MAAO;AACL,QAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,QAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,MAAA,EAAQ;AAAA,cACN,cAAA,EAAgB;AAAA;AAClB,WACD,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,YACV,GAAG,QAAQ,CAAA,6CAAA;AAAA,WACb;AACA,UAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,YACnB,OAAA,EAAS,KAAA;AAAA,YACT,QAAQ,EAAC;AAAA,YACT,UAAA,EAAY,aAAa,QAAQ,CAAA,WAAA;AAAA,WAClC,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,CAAA,wCAAA,CAAA;AAAA,MACA,OAAO,KAAK,GAAA,KAAQ;AAClB,QAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,QAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,QAAA,IAAI,MAAA,EAAQ;AACV,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,yBAAA,CAA0B,QAAQ,CAAA;AACrD,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,OAAA,EAAS,GAAG,QAAQ,CAAA,wBAAA;AAAA,WACrB,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,UAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,YAAA,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA,yBAAA,CAA2B,CAAA;AACxD,YAAA,GAAA,CAAI,IAAA,CAAK;AAAA,cACP,OAAA,EAAS,IAAA;AAAA,cACT,OAAA,EAAS;AAAA,aACV,CAAA;AAAA,UACH,CAAA,MAAO;AACL,YAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,cACnB,OAAA,EAAS,KAAA;AAAA,cACT,OAAA,EAAS,aAAa,QAAQ,CAAA,WAAA;AAAA,aAC/B,CAAA;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,KACF;AAIA,IAAA,MAAA,CAAO,IAAA,CAAK,CAAA,sCAAA,CAAA,EAA0C,OAAO,GAAA,EAAK,GAAA,KAAQ;AACxE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AAEzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,MAAM,cAAc,MAAA,CAAO,EAAA;AAC3B,QAAA,IAAI,MAAA,CAAO,WAAW,SAAA,EAAW;AAC/B,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,mBAAA,CAAoB,WAAW,CAAA;AAAA,QACpD,CAAA,MAAO;AACL,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,oBAAA,CAAqB,WAAW,CAAA;AAAA,QACrD;AACA,QAAA,GAAA,CAAI,IAAA,CAAK;AAAA,UACP,OAAA,EAAS,IAAA;AAAA,UACT,OAAA,EAAS,GAAG,QAAQ,CAAA,uBAAA;AAAA,SACrB,CAAA;AAAA,MACH,CAAA,MAAO;AACL,QAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,QAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA,yBAAA,CAA2B,CAAA;AACxD,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,OAAA,EAAS;AAAA,WACV,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,YACnB,OAAA,EAAS,KAAA;AAAA,YACT,OAAA,EAAS,aAAa,QAAQ,CAAA,WAAA;AAAA,WAC/B,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAED,IAAA,MAAA,CAAO,GAAA,CAAI,CAAA,sBAAA,CAAA,EAA0B,OAAO,IAAA,EAAM,GAAA,KAAQ;AACxD,MAAA,MAAM,SAAA,GAAY,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAEnD,MAAA,GAAA,CAAI,IAAA,CAAK;AAAA,QACP,OAAA,EAAS,IAAA;AAAA,QACT;AAAA,OACD,CAAA;AAAA,IACH,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,IAAA,CAAK,CAAA,uCAAA,CAAA,EAA2C,OAAO,GAAA,EAAK,GAAA,KAAQ;AACzE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,MAAM,cAAA,uBAAqB,IAAA,EAAK;AAChC,QAAA,cAAA,CAAe,QAAQ,cAAA,CAAe,OAAA,KAAY,EAAA,GAAK,EAAA,GAAK,KAAK,GAAI,CAAA;AACrE,QAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,YAAA,CAAa,QAAA,EAAU;AAAA,UACxC,WAAA,EAAa,gBAAA;AAAA,UACb,sBAAA,sBAA4B,IAAA,EAAK;AAAA,UACjC,cAAA;AAAA,UACA,MAAA,EAAQ;AAAA,SACT,CAAA;AACD,QAAA,GAAA,CAAI,IAAA,CAAK;AAAA,UACP,OAAA,EAAS,IAAA;AAAA,UACT,OAAA,EAAS,GAAG,QAAQ,CAAA,6BAAA;AAAA,SACrB,CAAA;AAAA,MACH,CAAA,MAAO;AACL,QAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,QAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA,yBAAA,CAA2B,CAAA;AACxD,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,OAAA,EAAS;AAAA,WACV,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,YACnB,OAAA,EAAS,KAAA;AAAA,YACT,OAAA,EAAS,aAAa,QAAQ,CAAA,WAAA;AAAA,WAC/B,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,MAAA,CAAO,kCAAA,EAAoC,OAAO,GAAA,EAAK,GAAA,KAAQ;AACpE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,sBAAsB,QAAQ,CAAA;AAChE,MAAA,GAAA,CAAI,KAAK,MAAM,CAAA;AAAA,IACjB,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,GAAA,CAAI,CAAA,sCAAA,CAAA,EAA0C,OAAO,GAAA,EAAK,GAAA,KAAQ;AACvE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,MAAM,KAAK,MAAA,CAAO,EAAA;AAClB,QAAA,MAAM,OAAA,GAAU,MAAM,IAAA,CAAK,OAAA,CAAQ,YAAY,EAAE,CAAA;AACjD,QAAA,GAAA,CAAI,IAAA,CAAK,EAAE,OAAA,EAAS,IAAA,EAAM,SAAS,CAAA;AAAA,MACrC,CAAA,MAAO;AACL,QAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,QAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA,yBAAA,CAA2B,CAAA;AACxD,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,OAAA,EAAS;AAAA,WACV,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,YACV,GAAG,QAAQ,CAAA,6CAAA;AAAA,WACb;AACA,UAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,YACnB,OAAA,EAAS,KAAA;AAAA,YACT,QAAQ,EAAC;AAAA,YACT,UAAA,EAAY,aAAa,QAAQ,CAAA,WAAA;AAAA,WAClC,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAED,IAAA,MAAA,CAAO,MAAA;AAAA,MACL,CAAA,sCAAA,CAAA;AAAA,MACA,OAAO,KAAK,GAAA,KAAQ;AAClB,QAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,QAAA,MAAM,SAAA,GAAY,MAAM,IAAA,CAAK,OAAA,CAAQ,wBAAwB,QAAQ,CAAA;AAErE,QAAA,GAAA,CAAI,IAAA,CAAK;AAAA,UACP,OAAA,EAAS,IAAA;AAAA,UACT,OAAA,EAAS,+BAA+B,QAAQ,CAAA,UAAA,CAAA;AAAA,UAChD;AAAA,SACD,CAAA;AAAA,MACH;AAAA,KACF;AAEA,IAAA,OAAO,MAAA;AAAA,EACT;AACF;;;;"}
+{"version":3,"file":"routes.cjs.js","sources":["../../src/router/routes.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  HttpAuthService,\n  LoggerService,\n  PermissionsService,\n} from '@backstage/backend-plugin-api';\nimport { NotAllowedError } from '@backstage/errors';\nimport {\n  catalogIngestionManagePermission,\n  catalogIngestionReadPermission,\n} from '@backstage/plugin-catalog-common/alpha';\nimport { AuthorizeResult } from '@backstage/plugin-permission-common';\nimport express from 'express';\nimport Router from 'express-promise-router';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\n\nexport class IncrementalProviderRouter {\n  private manager: IncrementalIngestionDatabaseManager;\n  private logger: LoggerService;\n\n  constructor(\n    manager: IncrementalIngestionDatabaseManager,\n    logger: LoggerService,\n    private readonly permissions: PermissionsService,\n    private readonly httpAuth: HttpAuthService,\n  ) {\n    this.manager = manager;\n    this.logger = logger;\n  }\n\n  createRouter(): express.Router {\n    const router = Router();\n    router.use(express.json());\n    router.use('/incremental', async (req, _res, next) => {\n      const permission =\n        req.method === 'GET' || req.method === 'HEAD'\n          ? catalogIngestionReadPermission\n          : catalogIngestionManagePermission;\n      const [decision] = await this.permissions.authorize([{ permission }], {\n        credentials: await this.httpAuth.credentials(req),\n      });\n      if (decision.result === AuthorizeResult.DENY) {\n        throw new NotAllowedError('Unauthorized');\n      }\n      next();\n    });\n\n    // Get the overall health of all incremental providers\n    router.get('/incremental/health', async (_, res) => {\n      const records = await this.manager.healthcheck();\n      const providers = records.map(record => record.provider_name);\n      const duplicates = [\n        ...new Set(providers.filter((e, i, a) => a.indexOf(e) !== i)),\n      ];\n\n      if (duplicates.length > 0) {\n        res.json({ healthy: false, duplicateIngestions: duplicates });\n      } else {\n        res.json({ healthy: true });\n      }\n    });\n\n    // Clean up and pause all providers\n    router.post('/incremental/cleanup', async (_, res) => {\n      const result = await this.manager.cleanupProviders();\n      res.json(result);\n    });\n\n    // Get basic status of the provider\n    router.get('/incremental/providers/:provider', async (req, res) => {\n      const { provider } = req.params;\n      const record = await this.manager.getCurrentIngestionRecord(provider);\n      if (record) {\n        res.json({\n          success: true,\n          status: {\n            current_action: record.status,\n            next_action_at: new Date(record.next_action_at),\n          },\n          last_error: record.last_error,\n        });\n      } else {\n        const providers: string[] = await this.manager.listProviders();\n        if (providers.includes(provider)) {\n          res.json({\n            success: true,\n            status: {\n              current_action: 'rest complete, waiting to start',\n            },\n          });\n        } else {\n          this.logger.error(\n            `${provider} - No ingestion record found in the database!`,\n          );\n          res.status(404).json({\n            success: false,\n            status: {},\n            last_error: `Provider '${provider}' not found`,\n          });\n        }\n      }\n    });\n\n    // Trigger the provider's next action\n    router.post(\n      `/incremental/providers/:provider/trigger`,\n      async (req, res) => {\n        const { provider } = req.params;\n        const record = await this.manager.getCurrentIngestionRecord(provider);\n        if (record) {\n          await this.manager.triggerNextProviderAction(provider);\n          res.json({\n            success: true,\n            message: `${provider}: Next action triggered.`,\n          });\n        } else {\n          const providers: string[] = await this.manager.listProviders();\n          if (providers.includes(provider)) {\n            this.logger.debug(`${provider} - Ingestion record found`);\n            res.json({\n              success: true,\n              message: 'Unable to trigger next action (provider is restarting)',\n            });\n          } else {\n            res.status(404).json({\n              success: false,\n              message: `Provider '${provider}' not found`,\n            });\n          }\n        }\n      },\n    );\n\n    // Start a brand-new ingestion cycle for the provider.\n    // (Cancel's the current run if active, or marks it complete if resting)\n    router.post(`/incremental/providers/:provider/start`, async (req, res) => {\n      const { provider } = req.params;\n\n      const record = await this.manager.getCurrentIngestionRecord(provider);\n      if (record) {\n        const ingestionId = record.id;\n        if (record.status === 'resting') {\n          await this.manager.setProviderComplete(ingestionId);\n        } else {\n          await this.manager.setProviderCanceling(ingestionId);\n        }\n        res.json({\n          success: true,\n          message: `${provider}: Next cycle triggered.`,\n        });\n      } else {\n        const providers: string[] = await this.manager.listProviders();\n        if (providers.includes(provider)) {\n          this.logger.debug(`${provider} - Ingestion record found`);\n          res.json({\n            success: true,\n            message: 'Provider is already restarting',\n          });\n        } else {\n          res.status(404).json({\n            success: false,\n            message: `Provider '${provider}' not found`,\n          });\n        }\n      }\n    });\n\n    router.get(`/incremental/providers`, async (_req, res) => {\n      const providers = await this.manager.listProviders();\n\n      res.json({\n        success: true,\n        providers,\n      });\n    });\n\n    // Stop the provider and pause it for 24 hours\n    router.post(`/incremental/providers/:provider/cancel`, async (req, res) => {\n      const { provider } = req.params;\n      const record = await this.manager.getCurrentIngestionRecord(provider);\n      if (record) {\n        const next_action_at = new Date();\n        next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1000);\n        await this.manager.updateByName(provider, {\n          next_action: 'nothing (done)',\n          ingestion_completed_at: new Date(),\n          next_action_at,\n          status: 'resting',\n        });\n        res.json({\n          success: true,\n          message: `${provider}: Current ingestion canceled.`,\n        });\n      } else {\n        const providers: string[] = await this.manager.listProviders();\n        if (providers.includes(provider)) {\n          this.logger.debug(`${provider} - Ingestion record found`);\n          res.json({\n            success: true,\n            message: 'Provider is currently restarting, please wait.',\n          });\n        } else {\n          res.status(404).json({\n            success: false,\n            message: `Provider '${provider}' not found`,\n          });\n        }\n      }\n    });\n\n    // Wipe out all ingestion records for the provider and pause for 24 hours\n    router.delete('/incremental/providers/:provider', async (req, res) => {\n      const { provider } = req.params;\n      const result = await this.manager.purgeAndResetProvider(provider);\n      res.json(result);\n    });\n\n    // Get the ingestion marks for the current cycle\n    router.get(`/incremental/providers/:provider/marks`, async (req, res) => {\n      const { provider } = req.params;\n      const record = await this.manager.getCurrentIngestionRecord(provider);\n      if (record) {\n        const id = record.id;\n        const records = await this.manager.getAllMarks(id);\n        res.json({ success: true, records });\n      } else {\n        const providers: string[] = await this.manager.listProviders();\n        if (providers.includes(provider)) {\n          this.logger.debug(`${provider} - Ingestion record found`);\n          res.json({\n            success: true,\n            message: 'No records yet (provider is restarting)',\n          });\n        } else {\n          this.logger.error(\n            `${provider} - No ingestion record found in the database!`,\n          );\n          res.status(404).json({\n            success: false,\n            status: {},\n            last_error: `Provider '${provider}' not found`,\n          });\n        }\n      }\n    });\n\n    router.delete(\n      `/incremental/providers/:provider/marks`,\n      async (req, res) => {\n        const { provider } = req.params;\n        const deletions = await this.manager.clearFinishedIngestions(provider);\n\n        res.json({\n          success: true,\n          message: `Expired marks for provider '${provider}' removed.`,\n          deletions,\n        });\n      },\n    );\n\n    return router;\n  }\n}\n"],"names":["Router","express","catalogIngestionReadPermission","catalogIngestionManagePermission","AuthorizeResult","NotAllowedError"],"mappings":";;;;;;;;;;;;;AA+BO,MAAM,yBAAA,CAA0B;AAAA,EAIrC,WAAA,CACE,OAAA,EACA,MAAA,EACiB,WAAA,EACA,QAAA,EACjB;AAFiB,IAAA,IAAA,CAAA,WAAA,GAAA,WAAA;AACA,IAAA,IAAA,CAAA,QAAA,GAAA,QAAA;AAEjB,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AACf,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AAAA,EAChB;AAAA,EALmB,WAAA;AAAA,EACA,QAAA;AAAA,EAPX,OAAA;AAAA,EACA,MAAA;AAAA,EAYR,YAAA,GAA+B;AAC7B,IAAA,MAAM,SAASA,uBAAA,EAAO;AACtB,IAAA,MAAA,CAAO,GAAA,CAAIC,wBAAA,CAAQ,IAAA,EAAM,CAAA;AACzB,IAAA,MAAA,CAAO,GAAA,CAAI,cAAA,EAAgB,OAAO,GAAA,EAAK,MAAM,IAAA,KAAS;AACpD,MAAA,MAAM,aACJ,GAAA,CAAI,MAAA,KAAW,SAAS,GAAA,CAAI,MAAA,KAAW,SACnCC,oCAAA,GACAC,sCAAA;AACN,MAAA,MAAM,CAAC,QAAQ,CAAA,GAAI,MAAM,IAAA,CAAK,WAAA,CAAY,SAAA,CAAU,CAAC,EAAE,UAAA,EAAY,CAAA,EAAG;AAAA,QACpE,WAAA,EAAa,MAAM,IAAA,CAAK,QAAA,CAAS,YAAY,GAAG;AAAA,OACjD,CAAA;AACD,MAAA,IAAI,QAAA,CAAS,MAAA,KAAWC,sCAAA,CAAgB,IAAA,EAAM;AAC5C,QAAA,MAAM,IAAIC,uBAAgB,cAAc,CAAA;AAAA,MAC1C;AACA,MAAA,IAAA,EAAK;AAAA,IACP,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,GAAA,CAAI,qBAAA,EAAuB,OAAO,CAAA,EAAG,GAAA,KAAQ;AAClD,MAAA,MAAM,OAAA,GAAU,MAAM,IAAA,CAAK,OAAA,CAAQ,WAAA,EAAY;AAC/C,MAAA,MAAM,SAAA,GAAY,OAAA,CAAQ,GAAA,CAAI,CAAA,MAAA,KAAU,OAAO,aAAa,CAAA;AAC5D,MAAA,MAAM,UAAA,GAAa;AAAA,QACjB,GAAG,IAAI,GAAA,CAAI,SAAA,CAAU,OAAO,CAAC,CAAA,EAAG,CAAA,EAAG,CAAA,KAAM,CAAA,CAAE,OAAA,CAAQ,CAAC,CAAA,KAAM,CAAC,CAAC;AAAA,OAC9D;AAEA,MAAA,IAAI,UAAA,CAAW,SAAS,CAAA,EAAG;AACzB,QAAA,GAAA,CAAI,KAAK,EAAE,OAAA,EAAS,KAAA,EAAO,mBAAA,EAAqB,YAAY,CAAA;AAAA,MAC9D,CAAA,MAAO;AACL,QAAA,GAAA,CAAI,IAAA,CAAK,EAAE,OAAA,EAAS,IAAA,EAAM,CAAA;AAAA,MAC5B;AAAA,IACF,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,IAAA,CAAK,sBAAA,EAAwB,OAAO,CAAA,EAAG,GAAA,KAAQ;AACpD,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,gBAAA,EAAiB;AACnD,MAAA,GAAA,CAAI,KAAK,MAAM,CAAA;AAAA,IACjB,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,GAAA,CAAI,kCAAA,EAAoC,OAAO,GAAA,EAAK,GAAA,KAAQ;AACjE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,GAAA,CAAI,IAAA,CAAK;AAAA,UACP,OAAA,EAAS,IAAA;AAAA,UACT,MAAA,EAAQ;AAAA,YACN,gBAAgB,MAAA,CAAO,MAAA;AAAA,YACvB,cAAA,EAAgB,IAAI,IAAA,CAAK,MAAA,CAAO,cAAc;AAAA,WAChD;AAAA,UACA,YAAY,MAAA,CAAO;AAAA,SACpB,CAAA;AAAA,MACH,CAAA,MAAO;AACL,QAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,QAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,MAAA,EAAQ;AAAA,cACN,cAAA,EAAgB;AAAA;AAClB,WACD,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,YACV,GAAG,QAAQ,CAAA,6CAAA;AAAA,WACb;AACA,UAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,YACnB,OAAA,EAAS,KAAA;AAAA,YACT,QAAQ,EAAC;AAAA,YACT,UAAA,EAAY,aAAa,QAAQ,CAAA,WAAA;AAAA,WAClC,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,CAAA,wCAAA,CAAA;AAAA,MACA,OAAO,KAAK,GAAA,KAAQ;AAClB,QAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,QAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,QAAA,IAAI,MAAA,EAAQ;AACV,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,yBAAA,CAA0B,QAAQ,CAAA;AACrD,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,OAAA,EAAS,GAAG,QAAQ,CAAA,wBAAA;AAAA,WACrB,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,UAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,YAAA,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA,yBAAA,CAA2B,CAAA;AACxD,YAAA,GAAA,CAAI,IAAA,CAAK;AAAA,cACP,OAAA,EAAS,IAAA;AAAA,cACT,OAAA,EAAS;AAAA,aACV,CAAA;AAAA,UACH,CAAA,MAAO;AACL,YAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,cACnB,OAAA,EAAS,KAAA;AAAA,cACT,OAAA,EAAS,aAAa,QAAQ,CAAA,WAAA;AAAA,aAC/B,CAAA;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,KACF;AAIA,IAAA,MAAA,CAAO,IAAA,CAAK,CAAA,sCAAA,CAAA,EAA0C,OAAO,GAAA,EAAK,GAAA,KAAQ;AACxE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AAEzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,MAAM,cAAc,MAAA,CAAO,EAAA;AAC3B,QAAA,IAAI,MAAA,CAAO,WAAW,SAAA,EAAW;AAC/B,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,mBAAA,CAAoB,WAAW,CAAA;AAAA,QACpD,CAAA,MAAO;AACL,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,oBAAA,CAAqB,WAAW,CAAA;AAAA,QACrD;AACA,QAAA,GAAA,CAAI,IAAA,CAAK;AAAA,UACP,OAAA,EAAS,IAAA;AAAA,UACT,OAAA,EAAS,GAAG,QAAQ,CAAA,uBAAA;AAAA,SACrB,CAAA;AAAA,MACH,CAAA,MAAO;AACL,QAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,QAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA,yBAAA,CAA2B,CAAA;AACxD,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,OAAA,EAAS;AAAA,WACV,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,YACnB,OAAA,EAAS,KAAA;AAAA,YACT,OAAA,EAAS,aAAa,QAAQ,CAAA,WAAA;AAAA,WAC/B,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAED,IAAA,MAAA,CAAO,GAAA,CAAI,CAAA,sBAAA,CAAA,EAA0B,OAAO,IAAA,EAAM,GAAA,KAAQ;AACxD,MAAA,MAAM,SAAA,GAAY,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAEnD,MAAA,GAAA,CAAI,IAAA,CAAK;AAAA,QACP,OAAA,EAAS,IAAA;AAAA,QACT;AAAA,OACD,CAAA;AAAA,IACH,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,IAAA,CAAK,CAAA,uCAAA,CAAA,EAA2C,OAAO,GAAA,EAAK,GAAA,KAAQ;AACzE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,MAAM,cAAA,uBAAqB,IAAA,EAAK;AAChC,QAAA,cAAA,CAAe,QAAQ,cAAA,CAAe,OAAA,KAAY,EAAA,GAAK,EAAA,GAAK,KAAK,GAAI,CAAA;AACrE,QAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,YAAA,CAAa,QAAA,EAAU;AAAA,UACxC,WAAA,EAAa,gBAAA;AAAA,UACb,sBAAA,sBAA4B,IAAA,EAAK;AAAA,UACjC,cAAA;AAAA,UACA,MAAA,EAAQ;AAAA,SACT,CAAA;AACD,QAAA,GAAA,CAAI,IAAA,CAAK;AAAA,UACP,OAAA,EAAS,IAAA;AAAA,UACT,OAAA,EAAS,GAAG,QAAQ,CAAA,6BAAA;AAAA,SACrB,CAAA;AAAA,MACH,CAAA,MAAO;AACL,QAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,QAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA,yBAAA,CAA2B,CAAA;AACxD,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,OAAA,EAAS;AAAA,WACV,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,YACnB,OAAA,EAAS,KAAA;AAAA,YACT,OAAA,EAAS,aAAa,QAAQ,CAAA,WAAA;AAAA,WAC/B,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,MAAA,CAAO,kCAAA,EAAoC,OAAO,GAAA,EAAK,GAAA,KAAQ;AACpE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,sBAAsB,QAAQ,CAAA;AAChE,MAAA,GAAA,CAAI,KAAK,MAAM,CAAA;AAAA,IACjB,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,GAAA,CAAI,CAAA,sCAAA,CAAA,EAA0C,OAAO,GAAA,EAAK,GAAA,KAAQ;AACvE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,MAAM,KAAK,MAAA,CAAO,EAAA;AAClB,QAAA,MAAM,OAAA,GAAU,MAAM,IAAA,CAAK,OAAA,CAAQ,YAAY,EAAE,CAAA;AACjD,QAAA,GAAA,CAAI,IAAA,CAAK,EAAE,OAAA,EAAS,IAAA,EAAM,SAAS,CAAA;AAAA,MACrC,CAAA,MAAO;AACL,QAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,QAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA,yBAAA,CAA2B,CAAA;AACxD,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,OAAA,EAAS;AAAA,WACV,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,YACV,GAAG,QAAQ,CAAA,6CAAA;AAAA,WACb;AACA,UAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,YACnB,OAAA,EAAS,KAAA;AAAA,YACT,QAAQ,EAAC;AAAA,YACT,UAAA,EAAY,aAAa,QAAQ,CAAA,WAAA;AAAA,WAClC,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAED,IAAA,MAAA,CAAO,MAAA;AAAA,MACL,CAAA,sCAAA,CAAA;AAAA,MACA,OAAO,KAAK,GAAA,KAAQ;AAClB,QAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,QAAA,MAAM,SAAA,GAAY,MAAM,IAAA,CAAK,OAAA,CAAQ,wBAAwB,QAAQ,CAAA;AAErE,QAAA,GAAA,CAAI,IAAA,CAAK;AAAA,UACP,OAAA,EAAS,IAAA;AAAA,UACT,OAAA,EAAS,+BAA+B,QAAQ,CAAA,UAAA,CAAA;AAAA,UAChD;AAAA,SACD,CAAA;AAAA,MACH;AAAA,KACF;AAEA,IAAA,OAAO,MAAA;AAAA,EACT;AACF;;"}

package/migrations/20260811133818_ingestion_mark_entities_source_key.js

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2026 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.up = async function up(knex) {
+  await knex.schema.alterTable('ingestion_mark_entities', table => {
+    table
+      .string('source_key')
+      .nullable()
+      .comment('The provider name of the ingestion that produced this mark');
+  });
+
+  await knex('ingestion_mark_entities').update({
+    source_key: knex('ingestions')
+      .select('ingestions.provider_name')
+      .innerJoin(
+        'ingestion_marks',
+        'ingestion_marks.ingestion_id',
+        'ingestions.id',
+      )
+      .where(
+        'ingestion_marks.id',
+        knex.ref('ingestion_mark_entities.ingestion_mark_id'),
+      ),
+  });
+
+  await knex.schema.alterTable('ingestion_mark_entities', table => {
+    table.renameColumn('ref', 'entity_ref');
+  });
+
+  // Prior to this migration, `(source_key, entity_ref)` was never enforced
+  // to be unique, so pre-existing installations may have duplicate rows.
+  // Keep only the row tied to the most recently created mark, breaking ties
+  // by the highest `id`.
+  await knex.raw(`
+    DELETE FROM ingestion_mark_entities WHERE id IN (
+      SELECT id FROM (
+        SELECT
+          ime.id AS id,
+          ROW_NUMBER() OVER (
+            PARTITION BY ime.source_key, ime.entity_ref
+            ORDER BY im.created_at DESC, ime.id DESC
+          ) AS rn
+        FROM ingestion_mark_entities ime
+        JOIN ingestion_marks im ON im.id = ime.ingestion_mark_id
+      ) ranked WHERE ranked.rn > 1
+    )
+  `);
+
+  await knex.schema.alterTable('ingestion_mark_entities', table => {
+    table.string('source_key').notNullable().alter();
+    table.unique(['source_key', 'entity_ref'], {
+      indexName: 'ingestion_mark_entities_source_key_entity_ref_uniq',
+    });
+  });
+};
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.down = async function down(knex) {
+  await knex.schema.alterTable('ingestion_mark_entities', table => {
+    table.dropUnique(
+      ['source_key', 'entity_ref'],
+      'ingestion_mark_entities_source_key_entity_ref_uniq',
+    );
+  });
+
+  await knex.schema.alterTable('ingestion_mark_entities', table => {
+    table.renameColumn('entity_ref', 'ref');
+  });
+
+  await knex.schema.alterTable('ingestion_mark_entities', table => {
+    table.dropColumn('source_key');
+  });
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend-module-incremental-ingestion",
-  "version": "0.8.0-next.1",
+  "version": "0.9.0-next.0",
   "description": "An entity provider for streaming large asset sources into the catalog",
   "backstage": {
     "role": "backend-plugin-module",
@@ -55,16 +55,16 @@
     "test": "backstage-cli package test"
   },
   "dependencies": {
-    "@backstage/backend-defaults": "0.17.9-next.1",
-    "@backstage/backend-plugin-api": "1.10.1-next.1",
-    "@backstage/catalog-model": "1.10.0",
-    "@backstage/config": "1.3.9-next.0",
-    "@backstage/errors": "1.3.1",
-    "@backstage/plugin-catalog-backend": "4.0.0-next.1",
-    "@backstage/plugin-catalog-common": "1.2.0-next.0",
-    "@backstage/plugin-catalog-node": "2.2.5-next.1",
-    "@backstage/plugin-events-node": "0.4.26-next.0",
-    "@backstage/plugin-permission-common": "0.9.11-next.0",
+    "@backstage/backend-defaults": "0.18.1-next.0",
+    "@backstage/backend-plugin-api": "1.10.2-next.0",
+    "@backstage/catalog-model": "1.10.2-next.0",
+    "@backstage/config": "1.3.10-next.0",
+    "@backstage/errors": "1.3.2-next.0",
+    "@backstage/plugin-catalog-backend": "4.0.1-next.0",
+    "@backstage/plugin-catalog-common": "1.2.1-next.0",
+    "@backstage/plugin-catalog-node": "2.2.6-next.0",
+    "@backstage/plugin-events-node": "0.4.27-next.0",
+    "@backstage/plugin-permission-common": "0.9.12-next.0",
     "@backstage/types": "1.2.2",
     "express": "^4.22.0",
     "express-promise-router": "^4.1.0",
@@ -72,8 +72,8 @@
     "luxon": "^3.0.0"
   },
   "devDependencies": {
-    "@backstage/backend-test-utils": "1.11.7-next.1",
-    "@backstage/cli": "0.36.6-next.1",
+    "@backstage/backend-test-utils": "1.11.8-next.0",
+    "@backstage/cli": "0.36.7-next.0",
     "@types/express": "^4.17.6",
     "@types/supertest": "^2.0.8",
     "supertest": "^7.0.0"