@backstage/plugin-catalog-backend-module-incremental-ingestion 0.5.5-next.1 → 0.5.5-next.2

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @backstage/plugin-catalog-backend-module-incremental-ingestion
 
+## 0.5.5-next.2
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/plugin-catalog-node@1.13.1-next.1
+  - @backstage/plugin-catalog-backend@1.26.2-next.2
+  - @backstage/backend-plugin-api@1.0.1-next.1
+  - @backstage/catalog-model@1.7.0
+  - @backstage/config@1.2.0
+  - @backstage/errors@1.2.4
+  - @backstage/plugin-events-node@0.4.1-next.1
+  - @backstage/plugin-permission-common@0.8.1
+
 ## 0.5.5-next.1
 
 ### Patch Changes

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend-module-incremental-ingestion__alpha",
-  "version": "0.5.5-next.1",
+  "version": "0.5.5-next.2",
   "main": "../dist/alpha.cjs.js",
   "types": "../dist/alpha.d.ts"
 }

package/dist/alpha.cjs.js

@@ -2,136 +2,10 @@
 
 Object.defineProperty(exports, '__esModule', { value: true });
 
-var backendPluginApi = require('@backstage/backend-plugin-api');
-var alpha = require('@backstage/plugin-catalog-node/alpha');
-var errors = require('@backstage/errors');
-var luxon = require('luxon');
-var util = require('./cjs/util-B3Qn9g5Y.cjs.js');
-require('perf_hooks');
-require('uuid');
-require('@backstage/catalog-model');
-require('@backstage/backend-common');
-require('express');
-require('express-promise-router');
+var catalogModuleIncrementalIngestionEntityProvider = require('./module/catalogModuleIncrementalIngestionEntityProvider.cjs.js');
 
-class WrapperProviders {
-  constructor(options) {
-    this.options = options;
-  }
-  migrate;
-  numberOfProvidersToConnect = 0;
-  readySignal = new util.Deferred();
-  wrap(provider, options) {
-    this.numberOfProvidersToConnect += 1;
-    return {
-      getProviderName: () => provider.getProviderName(),
-      connect: async (connection) => {
-        await this.startProvider(provider, options, connection);
-        this.numberOfProvidersToConnect -= 1;
-        if (this.numberOfProvidersToConnect === 0) {
-          this.readySignal.resolve();
-        }
-      }
-    };
-  }
-  async adminRouter() {
-    return await new util.IncrementalProviderRouter(
-      new util.IncrementalIngestionDatabaseManager({ client: this.options.client }),
-      this.options.logger
-    ).createRouter();
-  }
-  async startProvider(provider, providerOptions, connection) {
-    const logger = this.options.logger.child({
-      entityProvider: provider.getProviderName()
-    });
-    try {
-      if (!this.migrate) {
-        this.migrate = Promise.resolve().then(async () => {
-          const apply = this.options.applyDatabaseMigrations ?? util.applyDatabaseMigrations;
-          await apply(this.options.client);
-        });
-      }
-      await this.migrate;
-      const { burstInterval, burstLength, restLength } = providerOptions;
-      logger.info(`Connecting`);
-      const manager = new util.IncrementalIngestionDatabaseManager({
-        client: this.options.client
-      });
-      const engine = new util.IncrementalIngestionEngine({
-        ...providerOptions,
-        ready: this.readySignal,
-        manager,
-        logger,
-        provider,
-        restLength,
-        connection
-      });
-      const frequency = luxon.Duration.isDuration(burstInterval) ? burstInterval : luxon.Duration.fromObject(burstInterval);
-      const length = luxon.Duration.isDuration(burstLength) ? burstLength : luxon.Duration.fromObject(burstLength);
-      await this.options.scheduler.scheduleTask({
-        id: provider.getProviderName(),
-        fn: engine.taskFn.bind(engine),
-        frequency,
-        timeout: length
-      });
-    } catch (error) {
-      logger.warn(
-        `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${errors.stringifyError(
-          error
-        )}`
-      );
-      throw error;
-    }
-  }
-}
 
-const incrementalIngestionProvidersExtensionPoint = backendPluginApi.createExtensionPoint({
-  id: "catalog.incrementalIngestionProvider.providers"
-});
-const catalogModuleIncrementalIngestionEntityProvider = backendPluginApi.createBackendModule({
-  pluginId: "catalog",
-  moduleId: "incremental-ingestion-entity-provider",
-  register(env) {
-    const addedProviders = new Array();
-    env.registerExtensionPoint(incrementalIngestionProvidersExtensionPoint, {
-      addProvider({ options, provider }) {
-        addedProviders.push({ options, provider });
-      }
-    });
-    env.registerInit({
-      deps: {
-        catalog: alpha.catalogProcessingExtensionPoint,
-        config: backendPluginApi.coreServices.rootConfig,
-        database: backendPluginApi.coreServices.database,
-        httpRouter: backendPluginApi.coreServices.httpRouter,
-        logger: backendPluginApi.coreServices.logger,
-        scheduler: backendPluginApi.coreServices.scheduler
-      },
-      async init({
-        catalog,
-        config,
-        database,
-        httpRouter,
-        logger,
-        scheduler
-      }) {
-        const client = await database.getClient();
-        const providers = new WrapperProviders({
-          config,
-          logger,
-          client,
-          scheduler
-        });
-        for (const entry of addedProviders) {
-          const wrapped = providers.wrap(entry.provider, entry.options);
-          catalog.addEntityProvider(wrapped);
-        }
-        httpRouter.use(await providers.adminRouter());
-      }
-    });
-  }
-});
 
-exports.default = catalogModuleIncrementalIngestionEntityProvider;
-exports.incrementalIngestionProvidersExtensionPoint = incrementalIngestionProvidersExtensionPoint;
+exports.default = catalogModuleIncrementalIngestionEntityProvider.catalogModuleIncrementalIngestionEntityProvider;
+exports.incrementalIngestionProvidersExtensionPoint = catalogModuleIncrementalIngestionEntityProvider.incrementalIngestionProvidersExtensionPoint;
 //# sourceMappingURL=alpha.cjs.js.map

package/dist/alpha.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"alpha.cjs.js","sources":["../src/module/WrapperProviders.ts","../src/module/catalogModuleIncrementalIngestionEntityProvider.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  LoggerService,\n  RootConfigService,\n  SchedulerService,\n} from '@backstage/backend-plugin-api';\nimport { stringifyError } from '@backstage/errors';\nimport {\n  EntityProvider,\n  EntityProviderConnection,\n} from '@backstage/plugin-catalog-node';\nimport express from 'express';\nimport { Knex } from 'knex';\nimport { Duration } from 'luxon';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { applyDatabaseMigrations } from '../database/migrations';\nimport { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine';\nimport { IncrementalProviderRouter } from '../router/routes';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '../types';\nimport { Deferred } from '../util';\n\n/**\n * Helps in the creation of the catalog entity providers that wrap the\n * incremental ones.\n */\nexport class WrapperProviders {\n  private migrate: Promise<void> | undefined;\n  private numberOfProvidersToConnect = 0;\n  private readonly readySignal = new Deferred<void>();\n\n  constructor(\n    private readonly options: {\n      config: RootConfigService;\n      logger: LoggerService;\n      client: Knex;\n      scheduler: SchedulerService;\n      applyDatabaseMigrations?: typeof applyDatabaseMigrations;\n    },\n  ) {}\n\n  wrap(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    options: IncrementalEntityProviderOptions,\n  ): EntityProvider {\n    this.numberOfProvidersToConnect += 1;\n    return {\n      getProviderName: () => provider.getProviderName(),\n      connect: async connection => {\n        await this.startProvider(provider, options, connection);\n        this.numberOfProvidersToConnect -= 1;\n        if (this.numberOfProvidersToConnect === 0) {\n          this.readySignal.resolve();\n        }\n      },\n    };\n  }\n\n  async adminRouter(): Promise<express.Router> {\n    return await new IncrementalProviderRouter(\n      new IncrementalIngestionDatabaseManager({ client: this.options.client }),\n      this.options.logger,\n    ).createRouter();\n  }\n\n  private async startProvider(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    providerOptions: IncrementalEntityProviderOptions,\n    connection: EntityProviderConnection,\n  ) {\n    const logger = this.options.logger.child({\n      entityProvider: provider.getProviderName(),\n    });\n\n    try {\n      if (!this.migrate) {\n        this.migrate = Promise.resolve().then(async () => {\n          const apply =\n            this.options.applyDatabaseMigrations ?? applyDatabaseMigrations;\n          await apply(this.options.client);\n        });\n      }\n\n      await this.migrate;\n\n      const { burstInterval, burstLength, restLength } = providerOptions;\n\n      logger.info(`Connecting`);\n\n      const manager = new IncrementalIngestionDatabaseManager({\n        client: this.options.client,\n      });\n      const engine = new IncrementalIngestionEngine({\n        ...providerOptions,\n        ready: this.readySignal,\n        manager,\n        logger,\n        provider,\n        restLength,\n        connection,\n      });\n\n      const frequency = Duration.isDuration(burstInterval)\n        ? burstInterval\n        : Duration.fromObject(burstInterval);\n      const length = Duration.isDuration(burstLength)\n        ? burstLength\n        : Duration.fromObject(burstLength);\n\n      await this.options.scheduler.scheduleTask({\n        id: provider.getProviderName(),\n        fn: engine.taskFn.bind(engine),\n        frequency,\n        timeout: length,\n      });\n    } catch (error) {\n      logger.warn(\n        `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${stringifyError(\n          error,\n        )}`,\n      );\n      throw error;\n    }\n  }\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  coreServices,\n  createBackendModule,\n  createExtensionPoint,\n} from '@backstage/backend-plugin-api';\nimport { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '@backstage/plugin-catalog-backend-module-incremental-ingestion';\nimport { WrapperProviders } from './WrapperProviders';\n\n/**\n * @alpha\n * Interface for {@link incrementalIngestionProvidersExtensionPoint}.\n */\nexport interface IncrementalIngestionProviderExtensionPoint {\n  /** Adds a new incremental entity provider */\n  addProvider<TCursor, TContext>(config: {\n    options: IncrementalEntityProviderOptions;\n    provider: IncrementalEntityProvider<TCursor, TContext>;\n  }): void;\n}\n\n/**\n * @alpha\n *\n * Extension point for registering incremental ingestion providers.\n * The `catalogModuleIncrementalIngestionEntityProvider` must be installed for these providers to work.\n *\n * @example\n *\n * ```ts\n * backend.add(createBackendModule({\n *   pluginId: 'catalog',\n *   moduleId: 'my-incremental-provider',\n *   register(env) {\n *     env.registerInit({\n *       deps: {\n *         extension: incrementalIngestionProvidersExtensionPoint,\n *       },\n *       async init({ extension }) {\n *         extension.addProvider({\n *           burstInterval: ...,\n *           burstLength: ...,\n *           restLength: ...,\n *         }, {\n *           next(context, cursor) {\n *             ...\n *           },\n *           ...\n *         })\n *       })\n *     })\n *   }\n * }))\n * ```\n */\nexport const incrementalIngestionProvidersExtensionPoint =\n  createExtensionPoint<IncrementalIngestionProviderExtensionPoint>({\n    id: 'catalog.incrementalIngestionProvider.providers',\n  });\n\n/**\n * Registers the incremental entity provider with the catalog processing extension point.\n *\n * @alpha\n */\nexport const catalogModuleIncrementalIngestionEntityProvider =\n  createBackendModule({\n    pluginId: 'catalog',\n    moduleId: 'incremental-ingestion-entity-provider',\n    register(env) {\n      const addedProviders = new Array<{\n        provider: IncrementalEntityProvider<unknown, unknown>;\n        options: IncrementalEntityProviderOptions;\n      }>();\n\n      env.registerExtensionPoint(incrementalIngestionProvidersExtensionPoint, {\n        addProvider({ options, provider }) {\n          addedProviders.push({ options, provider });\n        },\n      });\n\n      env.registerInit({\n        deps: {\n          catalog: catalogProcessingExtensionPoint,\n          config: coreServices.rootConfig,\n          database: coreServices.database,\n          httpRouter: coreServices.httpRouter,\n          logger: coreServices.logger,\n          scheduler: coreServices.scheduler,\n        },\n        async init({\n          catalog,\n          config,\n          database,\n          httpRouter,\n          logger,\n          scheduler,\n        }) {\n          const client = await database.getClient();\n\n          const providers = new WrapperProviders({\n            config,\n            logger,\n            client,\n            scheduler,\n          });\n\n          for (const entry of addedProviders) {\n            const wrapped = providers.wrap(entry.provider, entry.options);\n            catalog.addEntityProvider(wrapped);\n          }\n\n          httpRouter.use(await providers.adminRouter());\n        },\n      });\n    },\n  });\n"],"names":["Deferred","IncrementalProviderRouter","IncrementalIngestionDatabaseManager","applyDatabaseMigrations","IncrementalIngestionEngine","Duration","stringifyError","createExtensionPoint","createBackendModule","catalogProcessingExtensionPoint","coreServices"],"mappings":";;;;;;;;;;;;;;;;AA2CO,MAAM,gBAAiB,CAAA;AAAA,EAK5B,YACmB,OAOjB,EAAA;AAPiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA,CAAA;AAAA,GAOhB;AAAA,EAZK,OAAA,CAAA;AAAA,EACA,0BAA6B,GAAA,CAAA,CAAA;AAAA,EACpB,WAAA,GAAc,IAAIA,aAAe,EAAA,CAAA;AAAA,EAYlD,IAAA,CACE,UACA,OACgB,EAAA;AAChB,IAAA,IAAA,CAAK,0BAA8B,IAAA,CAAA,CAAA;AACnC,IAAO,OAAA;AAAA,MACL,eAAA,EAAiB,MAAM,QAAA,CAAS,eAAgB,EAAA;AAAA,MAChD,OAAA,EAAS,OAAM,UAAc,KAAA;AAC3B,QAAA,MAAM,IAAK,CAAA,aAAA,CAAc,QAAU,EAAA,OAAA,EAAS,UAAU,CAAA,CAAA;AACtD,QAAA,IAAA,CAAK,0BAA8B,IAAA,CAAA,CAAA;AACnC,QAAI,IAAA,IAAA,CAAK,+BAA+B,CAAG,EAAA;AACzC,UAAA,IAAA,CAAK,YAAY,OAAQ,EAAA,CAAA;AAAA,SAC3B;AAAA,OACF;AAAA,KACF,CAAA;AAAA,GACF;AAAA,EAEA,MAAM,WAAuC,GAAA;AAC3C,IAAA,OAAO,MAAM,IAAIC,8BAAA;AAAA,MACf,IAAIC,wCAAoC,CAAA,EAAE,QAAQ,IAAK,CAAA,OAAA,CAAQ,QAAQ,CAAA;AAAA,MACvE,KAAK,OAAQ,CAAA,MAAA;AAAA,MACb,YAAa,EAAA,CAAA;AAAA,GACjB;AAAA,EAEA,MAAc,aAAA,CACZ,QACA,EAAA,eAAA,EACA,UACA,EAAA;AACA,IAAA,MAAM,MAAS,GAAA,IAAA,CAAK,OAAQ,CAAA,MAAA,CAAO,KAAM,CAAA;AAAA,MACvC,cAAA,EAAgB,SAAS,eAAgB,EAAA;AAAA,KAC1C,CAAA,CAAA;AAED,IAAI,IAAA;AACF,MAAI,IAAA,CAAC,KAAK,OAAS,EAAA;AACjB,QAAA,IAAA,CAAK,OAAU,GAAA,OAAA,CAAQ,OAAQ,EAAA,CAAE,KAAK,YAAY;AAChD,UAAM,MAAA,KAAA,GACJ,IAAK,CAAA,OAAA,CAAQ,uBAA2B,IAAAC,4BAAA,CAAA;AAC1C,UAAM,MAAA,KAAA,CAAM,IAAK,CAAA,OAAA,CAAQ,MAAM,CAAA,CAAA;AAAA,SAChC,CAAA,CAAA;AAAA,OACH;AAEA,MAAA,MAAM,IAAK,CAAA,OAAA,CAAA;AAEX,MAAA,MAAM,EAAE,aAAA,EAAe,WAAa,EAAA,UAAA,EAAe,GAAA,eAAA,CAAA;AAEnD,MAAA,MAAA,CAAO,KAAK,CAAY,UAAA,CAAA,CAAA,CAAA;AAExB,MAAM,MAAA,OAAA,GAAU,IAAID,wCAAoC,CAAA;AAAA,QACtD,MAAA,EAAQ,KAAK,OAAQ,CAAA,MAAA;AAAA,OACtB,CAAA,CAAA;AACD,MAAM,MAAA,MAAA,GAAS,IAAIE,+BAA2B,CAAA;AAAA,QAC5C,GAAG,eAAA;AAAA,QACH,OAAO,IAAK,CAAA,WAAA;AAAA,QACZ,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,UAAA;AAAA,OACD,CAAA,CAAA;AAED,MAAM,MAAA,SAAA,GAAYC,eAAS,UAAW,CAAA,aAAa,IAC/C,aACA,GAAAA,cAAA,CAAS,WAAW,aAAa,CAAA,CAAA;AACrC,MAAM,MAAA,MAAA,GAASA,eAAS,UAAW,CAAA,WAAW,IAC1C,WACA,GAAAA,cAAA,CAAS,WAAW,WAAW,CAAA,CAAA;AAEnC,MAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,SAAA,CAAU,YAAa,CAAA;AAAA,QACxC,EAAA,EAAI,SAAS,eAAgB,EAAA;AAAA,QAC7B,EAAI,EAAA,MAAA,CAAO,MAAO,CAAA,IAAA,CAAK,MAAM,CAAA;AAAA,QAC7B,SAAA;AAAA,QACA,OAAS,EAAA,MAAA;AAAA,OACV,CAAA,CAAA;AAAA,aACM,KAAO,EAAA;AACd,MAAO,MAAA,CAAA,IAAA;AAAA,QACL,CAAuD,oDAAA,EAAA,QAAA,CAAS,eAAgB,EAAC,CAAK,EAAA,EAAAC,qBAAA;AAAA,UACpF,KAAA;AAAA,SACD,CAAA,CAAA;AAAA,OACH,CAAA;AACA,MAAM,MAAA,KAAA,CAAA;AAAA,KACR;AAAA,GACF;AACF;;ACnEO,MAAM,8CACXC,qCAAiE,CAAA;AAAA,EAC/D,EAAI,EAAA,gDAAA;AACN,CAAC,EAAA;AAOI,MAAM,kDACXC,oCAAoB,CAAA;AAAA,EAClB,QAAU,EAAA,SAAA;AAAA,EACV,QAAU,EAAA,uCAAA;AAAA,EACV,SAAS,GAAK,EAAA;AACZ,IAAM,MAAA,cAAA,GAAiB,IAAI,KAGxB,EAAA,CAAA;AAEH,IAAA,GAAA,CAAI,uBAAuB,2CAA6C,EAAA;AAAA,MACtE,WAAY,CAAA,EAAE,OAAS,EAAA,QAAA,EAAY,EAAA;AACjC,QAAA,cAAA,CAAe,IAAK,CAAA,EAAE,OAAS,EAAA,QAAA,EAAU,CAAA,CAAA;AAAA,OAC3C;AAAA,KACD,CAAA,CAAA;AAED,IAAA,GAAA,CAAI,YAAa,CAAA;AAAA,MACf,IAAM,EAAA;AAAA,QACJ,OAAS,EAAAC,qCAAA;AAAA,QACT,QAAQC,6BAAa,CAAA,UAAA;AAAA,QACrB,UAAUA,6BAAa,CAAA,QAAA;AAAA,QACvB,YAAYA,6BAAa,CAAA,UAAA;AAAA,QACzB,QAAQA,6BAAa,CAAA,MAAA;AAAA,QACrB,WAAWA,6BAAa,CAAA,SAAA;AAAA,OAC1B;AAAA,MACA,MAAM,IAAK,CAAA;AAAA,QACT,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,MAAA;AAAA,QACA,SAAA;AAAA,OACC,EAAA;AACD,QAAM,MAAA,MAAA,GAAS,MAAM,QAAA,CAAS,SAAU,EAAA,CAAA;AAExC,QAAM,MAAA,SAAA,GAAY,IAAI,gBAAiB,CAAA;AAAA,UACrC,MAAA;AAAA,UACA,MAAA;AAAA,UACA,MAAA;AAAA,UACA,SAAA;AAAA,SACD,CAAA,CAAA;AAED,QAAA,KAAA,MAAW,SAAS,cAAgB,EAAA;AAClC,UAAA,MAAM,UAAU,SAAU,CAAA,IAAA,CAAK,KAAM,CAAA,QAAA,EAAU,MAAM,OAAO,CAAA,CAAA;AAC5D,UAAA,OAAA,CAAQ,kBAAkB,OAAO,CAAA,CAAA;AAAA,SACnC;AAEA,QAAA,UAAA,CAAW,GAAI,CAAA,MAAM,SAAU,CAAA,WAAA,EAAa,CAAA,CAAA;AAAA,OAC9C;AAAA,KACD,CAAA,CAAA;AAAA,GACH;AACF,CAAC;;;;;"}
+{"version":3,"file":"alpha.cjs.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;"}

package/dist/database/IncrementalIngestionDatabaseManager.cjs.js

@@ -0,0 +1,487 @@
+'use strict';
+
+var catalogModel = require('@backstage/catalog-model');
+var uuid = require('uuid');
+
+class IncrementalIngestionDatabaseManager {
+  client;
+  constructor(options) {
+    this.client = options.client;
+  }
+  /**
+   * Performs an update to the ingestion record with matching `id`.
+   * @param options - IngestionRecordUpdate
+   */
+  async updateIngestionRecordById(options) {
+    await this.client.transaction(async (tx) => {
+      const { ingestionId, update } = options;
+      await tx("ingestions").where("id", ingestionId).update(update);
+    });
+  }
+  /**
+   * Performs an update to the ingestion record with matching provider name. Will only update active records.
+   * @param provider - string
+   * @param update - Partial<IngestionUpsertIFace>
+   */
+  async updateIngestionRecordByProvider(provider, update) {
+    await this.client.transaction(async (tx) => {
+      await tx("ingestions").where("provider_name", provider).andWhere("completion_ticket", "open").update(update);
+    });
+  }
+  /**
+   * Performs an insert into the `ingestions` table with the supplied values.
+   * @param record - IngestionUpsertIFace
+   */
+  async insertIngestionRecord(record) {
+    await this.client.transaction(async (tx) => {
+      await tx("ingestions").insert(record);
+    });
+  }
+  async deleteMarkEntities(tx, ids) {
+    const chunks = [];
+    for (let i = 0; i < ids.length; i += 100) {
+      const chunk = ids.slice(i, i + 100);
+      chunks.push(chunk);
+    }
+    let deleted = 0;
+    for (const chunk of chunks) {
+      const chunkDeleted = await tx("ingestion_mark_entities").delete().whereIn(
+        "id",
+        chunk.map((entry) => entry.id)
+      );
+      deleted += chunkDeleted;
+    }
+    return deleted;
+  }
+  /**
+   * Finds the current ingestion record for the named provider.
+   * @param provider - string
+   * @returns IngestionRecord | undefined
+   */
+  async getCurrentIngestionRecord(provider) {
+    return await this.client.transaction(async (tx) => {
+      const record = await tx("ingestions").where("provider_name", provider).andWhere("completion_ticket", "open").first();
+      return record;
+    });
+  }
+  /**
+   * Finds the last ingestion record for the named provider.
+   * @param provider - string
+   * @returns IngestionRecord | undefined
+   */
+  async getPreviousIngestionRecord(provider) {
+    return await this.client.transaction(async (tx) => {
+      return await tx("ingestions").where("provider_name", provider).andWhereNot("completion_ticket", "open").first();
+    });
+  }
+  /**
+   * Removes all entries from `ingestion_marks_entities`, `ingestion_marks`, and `ingestions`
+   * for prior ingestions that completed (i.e., have a `completion_ticket` value other than 'open').
+   * @param provider - string
+   * @returns A count of deletions for each record type.
+   */
+  async clearFinishedIngestions(provider) {
+    return await this.client.transaction(async (tx) => {
+      const markEntitiesDeleted = await tx("ingestion_mark_entities").delete().whereIn(
+        "ingestion_mark_id",
+        tx("ingestion_marks").select("id").whereIn(
+          "ingestion_id",
+          tx("ingestions").select("id").where("provider_name", provider).andWhereNot("completion_ticket", "open")
+        )
+      );
+      const marksDeleted = await tx("ingestion_marks").delete().whereIn(
+        "ingestion_id",
+        tx("ingestions").select("id").where("provider_name", provider).andWhereNot("completion_ticket", "open")
+      );
+      const ingestionsDeleted = await tx("ingestions").delete().where("provider_name", provider).andWhereNot("completion_ticket", "open");
+      return {
+        deletions: {
+          markEntitiesDeleted,
+          marksDeleted,
+          ingestionsDeleted
+        }
+      };
+    });
+  }
+  /**
+   * Automatically cleans up duplicate ingestion records if they were accidentally created.
+   * Any ingestion record where the `rest_completed_at` is null (meaning it is active) AND
+   * the ingestionId is incorrect is a duplicate ingestion record.
+   * @param ingestionId - string
+   * @param provider - string
+   */
+  async clearDuplicateIngestions(ingestionId, provider) {
+    await this.client.transaction(async (tx) => {
+      const invalid = await tx("ingestions").where("provider_name", provider).andWhere("rest_completed_at", null).andWhereNot("id", ingestionId);
+      if (invalid.length > 0) {
+        await tx("ingestions").delete().whereIn("id", invalid);
+        await tx("ingestion_mark_entities").delete().whereIn(
+          "ingestion_mark_id",
+          tx("ingestion_marks").select("id").whereIn("ingestion_id", invalid)
+        );
+        await tx("ingestion_marks").delete().whereIn("ingestion_id", invalid);
+      }
+    });
+  }
+  /**
+   * This method fully purges and resets all ingestion records for the named provider, and
+   * leaves it in a paused state.
+   * @param provider - string
+   * @returns Counts of all deleted ingestion records
+   */
+  async purgeAndResetProvider(provider) {
+    return await this.client.transaction(async (tx) => {
+      const ingestionIDs = await tx("ingestions").select("id").where("provider_name", provider);
+      const markIDs = ingestionIDs.length > 0 ? await tx("ingestion_marks").select("id").whereIn(
+        "ingestion_id",
+        ingestionIDs.map((entry) => entry.id)
+      ) : [];
+      const markEntityIDs = markIDs.length > 0 ? await tx("ingestion_mark_entities").select("id").whereIn(
+        "ingestion_mark_id",
+        markIDs.map((entry) => entry.id)
+      ) : [];
+      const markEntitiesDeleted = await this.deleteMarkEntities(
+        tx,
+        markEntityIDs
+      );
+      const marksDeleted = markIDs.length > 0 ? await tx("ingestion_marks").delete().whereIn(
+        "ingestion_id",
+        ingestionIDs.map((entry) => entry.id)
+      ) : 0;
+      const ingestionsDeleted = await tx("ingestions").delete().where("provider_name", provider);
+      const next_action_at = /* @__PURE__ */ new Date();
+      next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1e3);
+      await this.insertIngestionRecord({
+        id: uuid.v4(),
+        next_action: "rest",
+        provider_name: provider,
+        next_action_at,
+        ingestion_completed_at: /* @__PURE__ */ new Date(),
+        status: "resting",
+        completion_ticket: "open"
+      });
+      return { provider, ingestionsDeleted, marksDeleted, markEntitiesDeleted };
+    });
+  }
+  /**
+   * This method is used to remove entity records from the ingestion_mark_entities
+   * table by their entity reference.
+   */
+  async deleteEntityRecordsByRef(entities) {
+    const refs = entities.map((e) => e.entityRef);
+    await this.client.transaction(async (tx) => {
+      await tx("ingestion_mark_entities").delete().whereIn("ref", refs);
+    });
+  }
+  /**
+   * Creates a new ingestion record.
+   * @param provider - string
+   * @returns A new ingestion record
+   */
+  async createProviderIngestionRecord(provider) {
+    const ingestionId = uuid.v4();
+    const nextAction = "ingest";
+    try {
+      await this.insertIngestionRecord({
+        id: ingestionId,
+        next_action: nextAction,
+        provider_name: provider,
+        status: "bursting",
+        completion_ticket: "open"
+      });
+      return { ingestionId, nextAction, attempts: 0, nextActionAt: Date.now() };
+    } catch (_e) {
+      return void 0;
+    }
+  }
+  /**
+   * Computes which entities to remove, if any, at the end of a burst.
+   * @param provider - string
+   * @param ingestionId - string
+   * @returns All entities to remove for this burst.
+   */
+  async computeRemoved(provider, ingestionId) {
+    const previousIngestion = await this.getPreviousIngestionRecord(provider);
+    return await this.client.transaction(async (tx) => {
+      const count = await tx("ingestion_mark_entities").count({ total: "ingestion_mark_entities.ref" }).join(
+        "ingestion_marks",
+        "ingestion_marks.id",
+        "ingestion_mark_entities.ingestion_mark_id"
+      ).join("ingestions", "ingestions.id", "ingestion_marks.ingestion_id").where("ingestions.id", ingestionId);
+      const total = count.reduce((acc, cur) => acc + cur.total, 0);
+      const removed = [];
+      if (previousIngestion) {
+        const stale = await tx("ingestion_mark_entities").select("ingestion_mark_entities.ref").join(
+          "ingestion_marks",
+          "ingestion_marks.id",
+          "ingestion_mark_entities.ingestion_mark_id"
+        ).join("ingestions", "ingestions.id", "ingestion_marks.ingestion_id").where("ingestions.id", previousIngestion.id);
+        for (const entityRef of stale) {
+          removed.push({ entityRef: entityRef.ref });
+        }
+      }
+      return { total, removed };
+    });
+  }
+  /**
+   * Performs a lookup of all providers that have duplicate active ingestion records.
+   * @returns An array of all duplicate active ingestions
+   */
+  async healthcheck() {
+    return await this.client.transaction(async (tx) => {
+      const records = await tx(
+        "ingestions"
+      ).distinct("id", "provider_name").where("rest_completed_at", null);
+      return records;
+    });
+  }
+  /**
+   * Skips any wait time for the next action to run.
+   * @param provider - string
+   */
+  async triggerNextProviderAction(provider) {
+    await this.updateIngestionRecordByProvider(provider, {
+      next_action_at: /* @__PURE__ */ new Date()
+    });
+  }
+  /**
+   * Purges the following tables:
+   * * `ingestions`
+   * * `ingestion_marks`
+   * * `ingestion_mark_entities`
+   *
+   * This function leaves the ingestions table with all providers in a paused state.
+   * @returns Results from cleaning up all ingestion tables.
+   */
+  async cleanupProviders() {
+    const providers = await this.listProviders();
+    const ingestionsDeleted = await this.purgeTable("ingestions");
+    const next_action_at = /* @__PURE__ */ new Date();
+    next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1e3);
+    for (const provider of providers) {
+      await this.insertIngestionRecord({
+        id: uuid.v4(),
+        next_action: "rest",
+        provider_name: provider,
+        next_action_at,
+        ingestion_completed_at: /* @__PURE__ */ new Date(),
+        status: "resting",
+        completion_ticket: "open"
+      });
+    }
+    const ingestionMarksDeleted = await this.purgeTable("ingestion_marks");
+    const markEntitiesDeleted = await this.purgeTable(
+      "ingestion_mark_entities"
+    );
+    return { ingestionsDeleted, ingestionMarksDeleted, markEntitiesDeleted };
+  }
+  /**
+   * Configures the current ingestion record to ingest a burst.
+   * @param ingestionId - string
+   */
+  async setProviderIngesting(ingestionId) {
+    await this.updateIngestionRecordById({
+      ingestionId,
+      update: { next_action: "ingest" }
+    });
+  }
+  /**
+   * Indicates the provider is currently ingesting a burst.
+   * @param ingestionId - string
+   */
+  async setProviderBursting(ingestionId) {
+    await this.updateIngestionRecordById({
+      ingestionId,
+      update: { status: "bursting" }
+    });
+  }
+  /**
+   * Finalizes the current ingestion record to indicate that the post-ingestion rest period is complete.
+   * @param ingestionId - string
+   */
+  async setProviderComplete(ingestionId) {
+    await this.updateIngestionRecordById({
+      ingestionId,
+      update: {
+        next_action: "nothing (done)",
+        rest_completed_at: /* @__PURE__ */ new Date(),
+        status: "complete",
+        completion_ticket: uuid.v4()
+      }
+    });
+  }
+  /**
+   * Marks ingestion as complete and starts the post-ingestion rest cycle.
+   * @param ingestionId - string
+   * @param restLength - Duration
+   */
+  async setProviderResting(ingestionId, restLength) {
+    await this.updateIngestionRecordById({
+      ingestionId,
+      update: {
+        next_action: "rest",
+        next_action_at: new Date(Date.now() + restLength.as("milliseconds")),
+        ingestion_completed_at: /* @__PURE__ */ new Date(),
+        status: "resting"
+      }
+    });
+  }
+  /**
+   * Marks ingestion as paused after a burst completes.
+   * @param ingestionId - string
+   */
+  async setProviderInterstitial(ingestionId) {
+    await this.updateIngestionRecordById({
+      ingestionId,
+      update: { attempts: 0, status: "interstitial" }
+    });
+  }
+  /**
+   * Starts the cancel process for the current ingestion.
+   * @param ingestionId - string
+   * @param message - string (optional)
+   */
+  async setProviderCanceling(ingestionId, message) {
+    const update = {
+      next_action: "cancel",
+      last_error: message ? message : void 0,
+      next_action_at: /* @__PURE__ */ new Date(),
+      status: "canceling"
+    };
+    await this.updateIngestionRecordById({ ingestionId, update });
+  }
+  /**
+   * Completes the cancel process and triggers a new ingestion.
+   * @param ingestionId - string
+   */
+  async setProviderCanceled(ingestionId) {
+    await this.updateIngestionRecordById({
+      ingestionId,
+      update: {
+        next_action: "nothing (canceled)",
+        rest_completed_at: /* @__PURE__ */ new Date(),
+        status: "complete",
+        completion_ticket: uuid.v4()
+      }
+    });
+  }
+  /**
+   * Configures the current ingestion to wait and retry, due to a data source error.
+   * @param ingestionId - string
+   * @param attempts - number
+   * @param error - Error
+   * @param backoffLength - number
+   */
+  async setProviderBackoff(ingestionId, attempts, error, backoffLength) {
+    await this.updateIngestionRecordById({
+      ingestionId,
+      update: {
+        next_action: "backoff",
+        attempts: attempts + 1,
+        last_error: String(error),
+        next_action_at: new Date(Date.now() + backoffLength),
+        status: "backing off"
+      }
+    });
+  }
+  /**
+   * Returns the last record from `ingestion_marks` for the supplied ingestionId.
+   * @param ingestionId - string
+   * @returns MarkRecord | undefined
+   */
+  async getLastMark(ingestionId) {
+    return await this.client.transaction(async (tx) => {
+      const mark = await tx("ingestion_marks").where("ingestion_id", ingestionId).orderBy("sequence", "desc").first();
+      return this.#decodeMark(this.client, mark);
+    });
+  }
+  /**
+   * Returns the first record from `ingestion_marks` for the supplied ingestionId.
+   * @param ingestionId - string
+   * @returns MarkRecord | undefined
+   */
+  async getFirstMark(ingestionId) {
+    return await this.client.transaction(async (tx) => {
+      const mark = await tx("ingestion_marks").where("ingestion_id", ingestionId).orderBy("sequence", "asc").first();
+      return this.#decodeMark(this.client, mark);
+    });
+  }
+  async getAllMarks(ingestionId) {
+    return await this.client.transaction(async (tx) => {
+      const marks = await tx("ingestion_marks").where("ingestion_id", ingestionId).orderBy("sequence", "desc");
+      return marks.map((m) => this.#decodeMark(this.client, m));
+    });
+  }
+  /**
+   * Performs an insert into the `ingestion_marks` table with the supplied values.
+   * @param options - MarkRecordInsert
+   */
+  async createMark(options) {
+    const { record } = options;
+    await this.client.transaction(async (tx) => {
+      await tx("ingestion_marks").insert(record);
+    });
+  }
+  // Handles the fact that sqlite does not support json columns; they just
+  // persist the stringified data instead
+  #decodeMark(knex, record) {
+    if (record && knex.client.config.client.includes("sqlite3")) {
+      return {
+        ...record,
+        cursor: JSON.parse(record.cursor)
+      };
+    }
+    return record;
+  }
+  /**
+   * Performs an upsert to the `ingestion_mark_entities` table for all deferred entities.
+   * @param markId - string
+   * @param entities - DeferredEntity[]
+   */
+  async createMarkEntities(markId, entities) {
+    const refs = entities.map((e) => catalogModel.stringifyEntityRef(e.entity));
+    await this.client.transaction(async (tx) => {
+      const existingRefsArray = (await tx("ingestion_mark_entities").select("ref").whereIn("ref", refs)).map((e) => e.ref);
+      const existingRefsSet = new Set(existingRefsArray);
+      const newRefs = refs.filter((e) => !existingRefsSet.has(e));
+      await tx("ingestion_mark_entities").update("ingestion_mark_id", markId).whereIn("ref", existingRefsArray);
+      if (newRefs.length > 0) {
+        await tx("ingestion_mark_entities").insert(
+          newRefs.map((ref) => ({
+            id: uuid.v4(),
+            ingestion_mark_id: markId,
+            ref
+          }))
+        );
+      }
+    });
+  }
+  /**
+   * Deletes the entire content of a table, and returns the number of records deleted.
+   * @param table - string
+   * @returns number
+   */
+  async purgeTable(table) {
+    return await this.client.transaction(async (tx) => {
+      return await tx(table).delete();
+    });
+  }
+  /**
+   * Returns a list of all providers.
+   * @returns string[]
+   */
+  async listProviders() {
+    return await this.client.transaction(async (tx) => {
+      const providers = await tx(
+        "ingestions"
+      ).distinct("provider_name");
+      return providers.map((entry) => entry.provider_name);
+    });
+  }
+  async updateByName(provider, update) {
+    await this.updateIngestionRecordByProvider(provider, update);
+  }
+}
+
+exports.IncrementalIngestionDatabaseManager = IncrementalIngestionDatabaseManager;
+//# sourceMappingURL=IncrementalIngestionDatabaseManager.cjs.js.map