@backstage/plugin-catalog-backend-module-incremental-ingestion 0.4.1-next.2 → 0.4.1

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # @backstage/plugin-catalog-backend-module-incremental-ingestion
 
+## 0.4.1
+
+### Patch Changes
+
+- 629cbd194a87: Use `coreServices.rootConfig` instead of `coreService.config`
+- e2b6396a1274: Export new alpha `incrementalIngestionProvidersExtensionPoint` for registering incremental providers, rather than the providers being passed as options to the backend module.
+- Updated dependencies
+  - @backstage/backend-common@0.19.2
+  - @backstage/plugin-catalog-backend@1.12.0
+  - @backstage/backend-plugin-api@0.6.0
+  - @backstage/plugin-catalog-node@1.4.1
+  - @backstage/plugin-events-node@0.2.9
+  - @backstage/backend-tasks@0.5.5
+  - @backstage/catalog-model@1.4.1
+  - @backstage/config@1.0.8
+  - @backstage/errors@1.2.1
+  - @backstage/plugin-permission-common@0.7.7
+
 ## 0.4.1-next.2
 
 ### Patch Changes

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend-module-incremental-ingestion",
-  "version": "0.4.1-next.2",
+  "version": "0.4.1",
   "main": "../dist/alpha.cjs.js",
   "types": "../dist/alpha.d.ts"
 }

package/dist/alpha.cjs.js

@@ -94,45 +94,53 @@ class WrapperProviders {
   }
 }
 
-const catalogModuleIncrementalIngestionEntityProvider = backendPluginApi.createBackendModule(
-  (options) => ({
-    pluginId: "catalog",
-    moduleId: "incrementalIngestionEntityProvider",
-    register(env) {
-      env.registerInit({
-        deps: {
-          catalog: alpha.catalogProcessingExtensionPoint,
-          config: backendPluginApi.coreServices.rootConfig,
-          database: backendPluginApi.coreServices.database,
-          httpRouter: backendPluginApi.coreServices.httpRouter,
-          logger: backendPluginApi.coreServices.logger,
-          scheduler: backendPluginApi.coreServices.scheduler
-        },
-        async init({
-          catalog,
+const incrementalIngestionProvidersExtensionPoint = backendPluginApi.createExtensionPoint({
+  id: "catalog.incrementalIngestionProvider.providers"
+});
+const catalogModuleIncrementalIngestionEntityProvider = backendPluginApi.createBackendModule({
+  pluginId: "catalog",
+  moduleId: "incrementalIngestionEntityProvider",
+  register(env) {
+    const addedProviders = new Array();
+    env.registerExtensionPoint(incrementalIngestionProvidersExtensionPoint, {
+      addProvider({ options, provider }) {
+        addedProviders.push({ options, provider });
+      }
+    });
+    env.registerInit({
+      deps: {
+        catalog: alpha.catalogProcessingExtensionPoint,
+        config: backendPluginApi.coreServices.rootConfig,
+        database: backendPluginApi.coreServices.database,
+        httpRouter: backendPluginApi.coreServices.httpRouter,
+        logger: backendPluginApi.coreServices.logger,
+        scheduler: backendPluginApi.coreServices.scheduler
+      },
+      async init({
+        catalog,
+        config,
+        database,
+        httpRouter,
+        logger,
+        scheduler
+      }) {
+        const client = await database.getClient();
+        const providers = new WrapperProviders({
           config,
-          database,
-          httpRouter,
           logger,
+          client,
           scheduler
-        }) {
-          const client = await database.getClient();
-          const providers = new WrapperProviders({
-            config,
-            logger,
-            client,
-            scheduler
-          });
-          for (const entry of options.providers) {
-            const wrapped = providers.wrap(entry.provider, entry.options);
-            catalog.addEntityProvider(wrapped);
-          }
-          httpRouter.use(await providers.adminRouter());
+        });
+        for (const entry of addedProviders) {
+          const wrapped = providers.wrap(entry.provider, entry.options);
+          catalog.addEntityProvider(wrapped);
         }
-      });
-    }
-  })
-);
+        httpRouter.use(await providers.adminRouter());
+      }
+    });
+  }
+});
 
 exports.catalogModuleIncrementalIngestionEntityProvider = catalogModuleIncrementalIngestionEntityProvider;
+exports.incrementalIngestionProvidersExtensionPoint = incrementalIngestionProvidersExtensionPoint;
 //# sourceMappingURL=alpha.cjs.js.map

package/dist/alpha.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"alpha.cjs.js","sources":["../src/module/WrapperProviders.ts","../src/module/catalogModuleIncrementalIngestionEntityProvider.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  RootConfigService,\n  LoggerService,\n  SchedulerService,\n} from '@backstage/backend-plugin-api';\nimport { loggerToWinstonLogger } from '@backstage/backend-common';\nimport { stringifyError } from '@backstage/errors';\nimport {\n  EntityProvider,\n  EntityProviderConnection,\n} from '@backstage/plugin-catalog-node';\nimport express from 'express';\nimport { Knex } from 'knex';\nimport { Duration } from 'luxon';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { applyDatabaseMigrations } from '../database/migrations';\nimport { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine';\nimport { IncrementalProviderRouter } from '../router/routes';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '../types';\nimport { Deferred } from '../util';\n\n/**\n * Helps in the creation of the catalog entity providers that wrap the\n * incremental ones.\n */\nexport class WrapperProviders {\n  private migrate: Promise<void> | undefined;\n  private numberOfProvidersToConnect = 0;\n  private readonly readySignal = new Deferred<void>();\n\n  constructor(\n    private readonly options: {\n      config: RootConfigService;\n      logger: LoggerService;\n      client: Knex;\n      scheduler: SchedulerService;\n      applyDatabaseMigrations?: typeof applyDatabaseMigrations;\n    },\n  ) {}\n\n  wrap(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    options: IncrementalEntityProviderOptions,\n  ): EntityProvider {\n    this.numberOfProvidersToConnect += 1;\n    return {\n      getProviderName: () => provider.getProviderName(),\n      connect: async connection => {\n        await this.startProvider(provider, options, connection);\n        this.numberOfProvidersToConnect -= 1;\n        if (this.numberOfProvidersToConnect === 0) {\n          this.readySignal.resolve();\n        }\n      },\n    };\n  }\n\n  async adminRouter(): Promise<express.Router> {\n    return await new IncrementalProviderRouter(\n      new IncrementalIngestionDatabaseManager({ client: this.options.client }),\n      loggerToWinstonLogger(this.options.logger),\n    ).createRouter();\n  }\n\n  private async startProvider(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    providerOptions: IncrementalEntityProviderOptions,\n    connection: EntityProviderConnection,\n  ) {\n    const logger = loggerToWinstonLogger(\n      this.options.logger.child({\n        entityProvider: provider.getProviderName(),\n      }),\n    );\n\n    try {\n      if (!this.migrate) {\n        this.migrate = Promise.resolve().then(async () => {\n          const apply =\n            this.options.applyDatabaseMigrations ?? applyDatabaseMigrations;\n          await apply(this.options.client);\n        });\n      }\n\n      await this.migrate;\n\n      const { burstInterval, burstLength, restLength } = providerOptions;\n\n      logger.info(`Connecting`);\n\n      const manager = new IncrementalIngestionDatabaseManager({\n        client: this.options.client,\n      });\n      const engine = new IncrementalIngestionEngine({\n        ...providerOptions,\n        ready: this.readySignal,\n        manager,\n        logger,\n        provider,\n        restLength,\n        connection,\n      });\n\n      const frequency = Duration.isDuration(burstInterval)\n        ? burstInterval\n        : Duration.fromObject(burstInterval);\n      const length = Duration.isDuration(burstLength)\n        ? burstLength\n        : Duration.fromObject(burstLength);\n\n      await this.options.scheduler.scheduleTask({\n        id: provider.getProviderName(),\n        fn: engine.taskFn.bind(engine),\n        frequency,\n        timeout: length,\n      });\n    } catch (error) {\n      logger.warn(\n        `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${stringifyError(\n          error,\n        )}`,\n      );\n      throw error;\n    }\n  }\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  coreServices,\n  createBackendModule,\n} from '@backstage/backend-plugin-api';\nimport { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '@backstage/plugin-catalog-backend-module-incremental-ingestion';\nimport { WrapperProviders } from './WrapperProviders';\n\n/**\n * Registers the incremental entity provider with the catalog processing extension point.\n *\n * @alpha\n */\nexport const catalogModuleIncrementalIngestionEntityProvider =\n  createBackendModule(\n    (options: {\n      providers: Array<{\n        provider: IncrementalEntityProvider<unknown, unknown>;\n        options: IncrementalEntityProviderOptions;\n      }>;\n    }) => ({\n      pluginId: 'catalog',\n      moduleId: 'incrementalIngestionEntityProvider',\n      register(env) {\n        env.registerInit({\n          deps: {\n            catalog: catalogProcessingExtensionPoint,\n            config: coreServices.rootConfig,\n            database: coreServices.database,\n            httpRouter: coreServices.httpRouter,\n            logger: coreServices.logger,\n            scheduler: coreServices.scheduler,\n          },\n          async init({\n            catalog,\n            config,\n            database,\n            httpRouter,\n            logger,\n            scheduler,\n          }) {\n            const client = await database.getClient();\n\n            const providers = new WrapperProviders({\n              config,\n              logger,\n              client,\n              scheduler,\n            });\n\n            for (const entry of options.providers) {\n              const wrapped = providers.wrap(entry.provider, entry.options);\n              catalog.addEntityProvider(wrapped);\n            }\n\n            httpRouter.use(await providers.adminRouter());\n          },\n        });\n      },\n    }),\n  );\n"],"names":["Deferred","IncrementalProviderRouter","IncrementalIngestionDatabaseManager","loggerToWinstonLogger","applyDatabaseMigrations","IncrementalIngestionEngine","Duration","stringifyError","createBackendModule","catalogProcessingExtensionPoint","coreServices"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA4CO,MAAM,gBAAiB,CAAA;AAAA,EAK5B,YACmB,OAOjB,EAAA;AAPiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA,CAAA;AALnB,IAAQ,aAAA,CAAA,IAAA,EAAA,SAAA,CAAA,CAAA;AACR,IAAA,aAAA,CAAA,IAAA,EAAQ,4BAA6B,EAAA,CAAA,CAAA,CAAA;AACrC,IAAiB,aAAA,CAAA,IAAA,EAAA,aAAA,EAAc,IAAIA,aAAe,EAAA,CAAA,CAAA;AAAA,GAU/C;AAAA,EAEH,IAAA,CACE,UACA,OACgB,EAAA;AAChB,IAAA,IAAA,CAAK,0BAA8B,IAAA,CAAA,CAAA;AACnC,IAAO,OAAA;AAAA,MACL,eAAA,EAAiB,MAAM,QAAA,CAAS,eAAgB,EAAA;AAAA,MAChD,OAAA,EAAS,OAAM,UAAc,KAAA;AAC3B,QAAA,MAAM,IAAK,CAAA,aAAA,CAAc,QAAU,EAAA,OAAA,EAAS,UAAU,CAAA,CAAA;AACtD,QAAA,IAAA,CAAK,0BAA8B,IAAA,CAAA,CAAA;AACnC,QAAI,IAAA,IAAA,CAAK,+BAA+B,CAAG,EAAA;AACzC,UAAA,IAAA,CAAK,YAAY,OAAQ,EAAA,CAAA;AAAA,SAC3B;AAAA,OACF;AAAA,KACF,CAAA;AAAA,GACF;AAAA,EAEA,MAAM,WAAuC,GAAA;AAC3C,IAAA,OAAO,MAAM,IAAIC,8BAAA;AAAA,MACf,IAAIC,wCAAoC,CAAA,EAAE,QAAQ,IAAK,CAAA,OAAA,CAAQ,QAAQ,CAAA;AAAA,MACvEC,mCAAA,CAAsB,IAAK,CAAA,OAAA,CAAQ,MAAM,CAAA;AAAA,MACzC,YAAa,EAAA,CAAA;AAAA,GACjB;AAAA,EAEA,MAAc,aAAA,CACZ,QACA,EAAA,eAAA,EACA,UACA,EAAA;AACA,IAAA,MAAM,MAAS,GAAAA,mCAAA;AAAA,MACb,IAAA,CAAK,OAAQ,CAAA,MAAA,CAAO,KAAM,CAAA;AAAA,QACxB,cAAA,EAAgB,SAAS,eAAgB,EAAA;AAAA,OAC1C,CAAA;AAAA,KACH,CAAA;AAEA,IAAI,IAAA;AACF,MAAI,IAAA,CAAC,KAAK,OAAS,EAAA;AACjB,QAAA,IAAA,CAAK,OAAU,GAAA,OAAA,CAAQ,OAAQ,EAAA,CAAE,KAAK,YAAY;AAhG1D,UAAA,IAAA,EAAA,CAAA;AAiGU,UAAA,MAAM,KACJ,GAAA,CAAA,EAAA,GAAA,IAAA,CAAK,OAAQ,CAAA,uBAAA,KAAb,IAAwC,GAAA,EAAA,GAAAC,4BAAA,CAAA;AAC1C,UAAM,MAAA,KAAA,CAAM,IAAK,CAAA,OAAA,CAAQ,MAAM,CAAA,CAAA;AAAA,SAChC,CAAA,CAAA;AAAA,OACH;AAEA,MAAA,MAAM,IAAK,CAAA,OAAA,CAAA;AAEX,MAAA,MAAM,EAAE,aAAA,EAAe,WAAa,EAAA,UAAA,EAAe,GAAA,eAAA,CAAA;AAEnD,MAAA,MAAA,CAAO,KAAK,CAAY,UAAA,CAAA,CAAA,CAAA;AAExB,MAAM,MAAA,OAAA,GAAU,IAAIF,wCAAoC,CAAA;AAAA,QACtD,MAAA,EAAQ,KAAK,OAAQ,CAAA,MAAA;AAAA,OACtB,CAAA,CAAA;AACD,MAAM,MAAA,MAAA,GAAS,IAAIG,+BAA2B,CAAA;AAAA,QAC5C,GAAG,eAAA;AAAA,QACH,OAAO,IAAK,CAAA,WAAA;AAAA,QACZ,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,UAAA;AAAA,OACD,CAAA,CAAA;AAED,MAAM,MAAA,SAAA,GAAYC,eAAS,UAAW,CAAA,aAAa,IAC/C,aACA,GAAAA,cAAA,CAAS,WAAW,aAAa,CAAA,CAAA;AACrC,MAAM,MAAA,MAAA,GAASA,eAAS,UAAW,CAAA,WAAW,IAC1C,WACA,GAAAA,cAAA,CAAS,WAAW,WAAW,CAAA,CAAA;AAEnC,MAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,SAAA,CAAU,YAAa,CAAA;AAAA,QACxC,EAAA,EAAI,SAAS,eAAgB,EAAA;AAAA,QAC7B,EAAI,EAAA,MAAA,CAAO,MAAO,CAAA,IAAA,CAAK,MAAM,CAAA;AAAA,QAC7B,SAAA;AAAA,QACA,OAAS,EAAA,MAAA;AAAA,OACV,CAAA,CAAA;AAAA,aACM,KAAO,EAAA;AACd,MAAO,MAAA,CAAA,IAAA;AAAA,QACL,CAAuD,oDAAA,EAAA,QAAA,CAAS,eAAgB,EAAC,CAAK,EAAA,EAAAC,qBAAA;AAAA,UACpF,KAAA;AAAA,SACD,CAAA,CAAA;AAAA,OACH,CAAA;AACA,MAAM,MAAA,KAAA,CAAA;AAAA,KACR;AAAA,GACF;AACF;;AChHO,MAAM,+CACX,GAAAC,oCAAA;AAAA,EACE,CAAC,OAKM,MAAA;AAAA,IACL,QAAU,EAAA,SAAA;AAAA,IACV,QAAU,EAAA,oCAAA;AAAA,IACV,SAAS,GAAK,EAAA;AACZ,MAAA,GAAA,CAAI,YAAa,CAAA;AAAA,QACf,IAAM,EAAA;AAAA,UACJ,OAAS,EAAAC,qCAAA;AAAA,UACT,QAAQC,6BAAa,CAAA,UAAA;AAAA,UACrB,UAAUA,6BAAa,CAAA,QAAA;AAAA,UACvB,YAAYA,6BAAa,CAAA,UAAA;AAAA,UACzB,QAAQA,6BAAa,CAAA,MAAA;AAAA,UACrB,WAAWA,6BAAa,CAAA,SAAA;AAAA,SAC1B;AAAA,QACA,MAAM,IAAK,CAAA;AAAA,UACT,OAAA;AAAA,UACA,MAAA;AAAA,UACA,QAAA;AAAA,UACA,UAAA;AAAA,UACA,MAAA;AAAA,UACA,SAAA;AAAA,SACC,EAAA;AACD,UAAM,MAAA,MAAA,GAAS,MAAM,QAAA,CAAS,SAAU,EAAA,CAAA;AAExC,UAAM,MAAA,SAAA,GAAY,IAAI,gBAAiB,CAAA;AAAA,YACrC,MAAA;AAAA,YACA,MAAA;AAAA,YACA,MAAA;AAAA,YACA,SAAA;AAAA,WACD,CAAA,CAAA;AAED,UAAW,KAAA,MAAA,KAAA,IAAS,QAAQ,SAAW,EAAA;AACrC,YAAA,MAAM,UAAU,SAAU,CAAA,IAAA,CAAK,KAAM,CAAA,QAAA,EAAU,MAAM,OAAO,CAAA,CAAA;AAC5D,YAAA,OAAA,CAAQ,kBAAkB,OAAO,CAAA,CAAA;AAAA,WACnC;AAEA,UAAA,UAAA,CAAW,GAAI,CAAA,MAAM,SAAU,CAAA,WAAA,EAAa,CAAA,CAAA;AAAA,SAC9C;AAAA,OACD,CAAA,CAAA;AAAA,KACH;AAAA,GACF,CAAA;AACF;;;;"}
+{"version":3,"file":"alpha.cjs.js","sources":["../src/module/WrapperProviders.ts","../src/module/catalogModuleIncrementalIngestionEntityProvider.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  RootConfigService,\n  LoggerService,\n  SchedulerService,\n} from '@backstage/backend-plugin-api';\nimport { loggerToWinstonLogger } from '@backstage/backend-common';\nimport { stringifyError } from '@backstage/errors';\nimport {\n  EntityProvider,\n  EntityProviderConnection,\n} from '@backstage/plugin-catalog-node';\nimport express from 'express';\nimport { Knex } from 'knex';\nimport { Duration } from 'luxon';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { applyDatabaseMigrations } from '../database/migrations';\nimport { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine';\nimport { IncrementalProviderRouter } from '../router/routes';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '../types';\nimport { Deferred } from '../util';\n\n/**\n * Helps in the creation of the catalog entity providers that wrap the\n * incremental ones.\n */\nexport class WrapperProviders {\n  private migrate: Promise<void> | undefined;\n  private numberOfProvidersToConnect = 0;\n  private readonly readySignal = new Deferred<void>();\n\n  constructor(\n    private readonly options: {\n      config: RootConfigService;\n      logger: LoggerService;\n      client: Knex;\n      scheduler: SchedulerService;\n      applyDatabaseMigrations?: typeof applyDatabaseMigrations;\n    },\n  ) {}\n\n  wrap(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    options: IncrementalEntityProviderOptions,\n  ): EntityProvider {\n    this.numberOfProvidersToConnect += 1;\n    return {\n      getProviderName: () => provider.getProviderName(),\n      connect: async connection => {\n        await this.startProvider(provider, options, connection);\n        this.numberOfProvidersToConnect -= 1;\n        if (this.numberOfProvidersToConnect === 0) {\n          this.readySignal.resolve();\n        }\n      },\n    };\n  }\n\n  async adminRouter(): Promise<express.Router> {\n    return await new IncrementalProviderRouter(\n      new IncrementalIngestionDatabaseManager({ client: this.options.client }),\n      loggerToWinstonLogger(this.options.logger),\n    ).createRouter();\n  }\n\n  private async startProvider(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    providerOptions: IncrementalEntityProviderOptions,\n    connection: EntityProviderConnection,\n  ) {\n    const logger = loggerToWinstonLogger(\n      this.options.logger.child({\n        entityProvider: provider.getProviderName(),\n      }),\n    );\n\n    try {\n      if (!this.migrate) {\n        this.migrate = Promise.resolve().then(async () => {\n          const apply =\n            this.options.applyDatabaseMigrations ?? applyDatabaseMigrations;\n          await apply(this.options.client);\n        });\n      }\n\n      await this.migrate;\n\n      const { burstInterval, burstLength, restLength } = providerOptions;\n\n      logger.info(`Connecting`);\n\n      const manager = new IncrementalIngestionDatabaseManager({\n        client: this.options.client,\n      });\n      const engine = new IncrementalIngestionEngine({\n        ...providerOptions,\n        ready: this.readySignal,\n        manager,\n        logger,\n        provider,\n        restLength,\n        connection,\n      });\n\n      const frequency = Duration.isDuration(burstInterval)\n        ? burstInterval\n        : Duration.fromObject(burstInterval);\n      const length = Duration.isDuration(burstLength)\n        ? burstLength\n        : Duration.fromObject(burstLength);\n\n      await this.options.scheduler.scheduleTask({\n        id: provider.getProviderName(),\n        fn: engine.taskFn.bind(engine),\n        frequency,\n        timeout: length,\n      });\n    } catch (error) {\n      logger.warn(\n        `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${stringifyError(\n          error,\n        )}`,\n      );\n      throw error;\n    }\n  }\n}\n","/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  coreServices,\n  createBackendModule,\n  createExtensionPoint,\n} from '@backstage/backend-plugin-api';\nimport { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '@backstage/plugin-catalog-backend-module-incremental-ingestion';\nimport { WrapperProviders } from './WrapperProviders';\n\n/**\n * @alpha\n * Interface for {@link incrementalIngestionProvidersExtensionPoint}.\n */\nexport interface IncrementalIngestionProviderExtensionPoint {\n  /** Adds a new incremental entity provider */\n  addProvider<TCursor, TContext>(config: {\n    options: IncrementalEntityProviderOptions;\n    provider: IncrementalEntityProvider<TCursor, TContext>;\n  }): void;\n}\n\n/**\n * @alpha\n *\n * Extension point for registering incremental ingestion providers.\n * The `catalogModuleIncrementalIngestionEntityProvider` must be installed for these providers to work.\n *\n * @example\n *\n * ```ts\n * backend.add(createBackendModule({\n *   pluginId: 'catalog',\n *   moduleId: 'myIncrementalProvider',\n *   register(env) {\n *     env.registerInit({\n *       deps: {\n *         extension: incrementalIngestionProvidersExtensionPoint,\n *       },\n *       async init({ extension }) {\n *         extension.addProvider({\n *           burstInterval: ...,\n *           burstLength: ...,\n *           restLength: ...,\n *         }, {\n *           next(context, cursor) {\n *             ...\n *           },\n *           ...\n *         })\n *       })\n *     })\n *   }\n * }))\n * ```\n */\nexport const incrementalIngestionProvidersExtensionPoint =\n  createExtensionPoint<IncrementalIngestionProviderExtensionPoint>({\n    id: 'catalog.incrementalIngestionProvider.providers',\n  });\n\n/**\n * Registers the incremental entity provider with the catalog processing extension point.\n *\n * @alpha\n */\nexport const catalogModuleIncrementalIngestionEntityProvider =\n  createBackendModule({\n    pluginId: 'catalog',\n    moduleId: 'incrementalIngestionEntityProvider',\n    register(env) {\n      const addedProviders = new Array<{\n        provider: IncrementalEntityProvider<unknown, unknown>;\n        options: IncrementalEntityProviderOptions;\n      }>();\n\n      env.registerExtensionPoint(incrementalIngestionProvidersExtensionPoint, {\n        addProvider({ options, provider }) {\n          addedProviders.push({ options, provider });\n        },\n      });\n\n      env.registerInit({\n        deps: {\n          catalog: catalogProcessingExtensionPoint,\n          config: coreServices.rootConfig,\n          database: coreServices.database,\n          httpRouter: coreServices.httpRouter,\n          logger: coreServices.logger,\n          scheduler: coreServices.scheduler,\n        },\n        async init({\n          catalog,\n          config,\n          database,\n          httpRouter,\n          logger,\n          scheduler,\n        }) {\n          const client = await database.getClient();\n\n          const providers = new WrapperProviders({\n            config,\n            logger,\n            client,\n            scheduler,\n          });\n\n          for (const entry of addedProviders) {\n            const wrapped = providers.wrap(entry.provider, entry.options);\n            catalog.addEntityProvider(wrapped);\n          }\n\n          httpRouter.use(await providers.adminRouter());\n        },\n      });\n    },\n  });\n"],"names":["Deferred","IncrementalProviderRouter","IncrementalIngestionDatabaseManager","loggerToWinstonLogger","applyDatabaseMigrations","IncrementalIngestionEngine","Duration","stringifyError","createExtensionPoint","createBackendModule","catalogProcessingExtensionPoint","coreServices"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA4CO,MAAM,gBAAiB,CAAA;AAAA,EAK5B,YACmB,OAOjB,EAAA;AAPiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA,CAAA;AALnB,IAAQ,aAAA,CAAA,IAAA,EAAA,SAAA,CAAA,CAAA;AACR,IAAA,aAAA,CAAA,IAAA,EAAQ,4BAA6B,EAAA,CAAA,CAAA,CAAA;AACrC,IAAiB,aAAA,CAAA,IAAA,EAAA,aAAA,EAAc,IAAIA,aAAe,EAAA,CAAA,CAAA;AAAA,GAU/C;AAAA,EAEH,IAAA,CACE,UACA,OACgB,EAAA;AAChB,IAAA,IAAA,CAAK,0BAA8B,IAAA,CAAA,CAAA;AACnC,IAAO,OAAA;AAAA,MACL,eAAA,EAAiB,MAAM,QAAA,CAAS,eAAgB,EAAA;AAAA,MAChD,OAAA,EAAS,OAAM,UAAc,KAAA;AAC3B,QAAA,MAAM,IAAK,CAAA,aAAA,CAAc,QAAU,EAAA,OAAA,EAAS,UAAU,CAAA,CAAA;AACtD,QAAA,IAAA,CAAK,0BAA8B,IAAA,CAAA,CAAA;AACnC,QAAI,IAAA,IAAA,CAAK,+BAA+B,CAAG,EAAA;AACzC,UAAA,IAAA,CAAK,YAAY,OAAQ,EAAA,CAAA;AAAA,SAC3B;AAAA,OACF;AAAA,KACF,CAAA;AAAA,GACF;AAAA,EAEA,MAAM,WAAuC,GAAA;AAC3C,IAAA,OAAO,MAAM,IAAIC,8BAAA;AAAA,MACf,IAAIC,wCAAoC,CAAA,EAAE,QAAQ,IAAK,CAAA,OAAA,CAAQ,QAAQ,CAAA;AAAA,MACvEC,mCAAA,CAAsB,IAAK,CAAA,OAAA,CAAQ,MAAM,CAAA;AAAA,MACzC,YAAa,EAAA,CAAA;AAAA,GACjB;AAAA,EAEA,MAAc,aAAA,CACZ,QACA,EAAA,eAAA,EACA,UACA,EAAA;AACA,IAAA,MAAM,MAAS,GAAAA,mCAAA;AAAA,MACb,IAAA,CAAK,OAAQ,CAAA,MAAA,CAAO,KAAM,CAAA;AAAA,QACxB,cAAA,EAAgB,SAAS,eAAgB,EAAA;AAAA,OAC1C,CAAA;AAAA,KACH,CAAA;AAEA,IAAI,IAAA;AACF,MAAI,IAAA,CAAC,KAAK,OAAS,EAAA;AACjB,QAAA,IAAA,CAAK,OAAU,GAAA,OAAA,CAAQ,OAAQ,EAAA,CAAE,KAAK,YAAY;AAhG1D,UAAA,IAAA,EAAA,CAAA;AAiGU,UAAA,MAAM,KACJ,GAAA,CAAA,EAAA,GAAA,IAAA,CAAK,OAAQ,CAAA,uBAAA,KAAb,IAAwC,GAAA,EAAA,GAAAC,4BAAA,CAAA;AAC1C,UAAM,MAAA,KAAA,CAAM,IAAK,CAAA,OAAA,CAAQ,MAAM,CAAA,CAAA;AAAA,SAChC,CAAA,CAAA;AAAA,OACH;AAEA,MAAA,MAAM,IAAK,CAAA,OAAA,CAAA;AAEX,MAAA,MAAM,EAAE,aAAA,EAAe,WAAa,EAAA,UAAA,EAAe,GAAA,eAAA,CAAA;AAEnD,MAAA,MAAA,CAAO,KAAK,CAAY,UAAA,CAAA,CAAA,CAAA;AAExB,MAAM,MAAA,OAAA,GAAU,IAAIF,wCAAoC,CAAA;AAAA,QACtD,MAAA,EAAQ,KAAK,OAAQ,CAAA,MAAA;AAAA,OACtB,CAAA,CAAA;AACD,MAAM,MAAA,MAAA,GAAS,IAAIG,+BAA2B,CAAA;AAAA,QAC5C,GAAG,eAAA;AAAA,QACH,OAAO,IAAK,CAAA,WAAA;AAAA,QACZ,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,UAAA;AAAA,OACD,CAAA,CAAA;AAED,MAAM,MAAA,SAAA,GAAYC,eAAS,UAAW,CAAA,aAAa,IAC/C,aACA,GAAAA,cAAA,CAAS,WAAW,aAAa,CAAA,CAAA;AACrC,MAAM,MAAA,MAAA,GAASA,eAAS,UAAW,CAAA,WAAW,IAC1C,WACA,GAAAA,cAAA,CAAS,WAAW,WAAW,CAAA,CAAA;AAEnC,MAAM,MAAA,IAAA,CAAK,OAAQ,CAAA,SAAA,CAAU,YAAa,CAAA;AAAA,QACxC,EAAA,EAAI,SAAS,eAAgB,EAAA;AAAA,QAC7B,EAAI,EAAA,MAAA,CAAO,MAAO,CAAA,IAAA,CAAK,MAAM,CAAA;AAAA,QAC7B,SAAA;AAAA,QACA,OAAS,EAAA,MAAA;AAAA,OACV,CAAA,CAAA;AAAA,aACM,KAAO,EAAA;AACd,MAAO,MAAA,CAAA,IAAA;AAAA,QACL,CAAuD,oDAAA,EAAA,QAAA,CAAS,eAAgB,EAAC,CAAK,EAAA,EAAAC,qBAAA;AAAA,UACpF,KAAA;AAAA,SACD,CAAA,CAAA;AAAA,OACH,CAAA;AACA,MAAM,MAAA,KAAA,CAAA;AAAA,KACR;AAAA,GACF;AACF;;ACtEO,MAAM,8CACXC,qCAAiE,CAAA;AAAA,EAC/D,EAAI,EAAA,gDAAA;AACN,CAAC,EAAA;AAOI,MAAM,kDACXC,oCAAoB,CAAA;AAAA,EAClB,QAAU,EAAA,SAAA;AAAA,EACV,QAAU,EAAA,oCAAA;AAAA,EACV,SAAS,GAAK,EAAA;AACZ,IAAM,MAAA,cAAA,GAAiB,IAAI,KAGxB,EAAA,CAAA;AAEH,IAAA,GAAA,CAAI,uBAAuB,2CAA6C,EAAA;AAAA,MACtE,WAAY,CAAA,EAAE,OAAS,EAAA,QAAA,EAAY,EAAA;AACjC,QAAA,cAAA,CAAe,IAAK,CAAA,EAAE,OAAS,EAAA,QAAA,EAAU,CAAA,CAAA;AAAA,OAC3C;AAAA,KACD,CAAA,CAAA;AAED,IAAA,GAAA,CAAI,YAAa,CAAA;AAAA,MACf,IAAM,EAAA;AAAA,QACJ,OAAS,EAAAC,qCAAA;AAAA,QACT,QAAQC,6BAAa,CAAA,UAAA;AAAA,QACrB,UAAUA,6BAAa,CAAA,QAAA;AAAA,QACvB,YAAYA,6BAAa,CAAA,UAAA;AAAA,QACzB,QAAQA,6BAAa,CAAA,MAAA;AAAA,QACrB,WAAWA,6BAAa,CAAA,SAAA;AAAA,OAC1B;AAAA,MACA,MAAM,IAAK,CAAA;AAAA,QACT,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,MAAA;AAAA,QACA,SAAA;AAAA,OACC,EAAA;AACD,QAAM,MAAA,MAAA,GAAS,MAAM,QAAA,CAAS,SAAU,EAAA,CAAA;AAExC,QAAM,MAAA,SAAA,GAAY,IAAI,gBAAiB,CAAA;AAAA,UACrC,MAAA;AAAA,UACA,MAAA;AAAA,UACA,MAAA;AAAA,UACA,SAAA;AAAA,SACD,CAAA,CAAA;AAED,QAAA,KAAA,MAAW,SAAS,cAAgB,EAAA;AAClC,UAAA,MAAM,UAAU,SAAU,CAAA,IAAA,CAAK,KAAM,CAAA,QAAA,EAAU,MAAM,OAAO,CAAA,CAAA;AAC5D,UAAA,OAAA,CAAQ,kBAAkB,OAAO,CAAA,CAAA;AAAA,SACnC;AAEA,QAAA,UAAA,CAAW,GAAI,CAAA,MAAM,SAAU,CAAA,WAAA,EAAa,CAAA,CAAA;AAAA,OAC9C;AAAA,KACD,CAAA,CAAA;AAAA,GACH;AACF,CAAC;;;;;"}

package/dist/alpha.d.ts

@@ -1,16 +1,57 @@
 import * as _backstage_backend_plugin_api from '@backstage/backend-plugin-api';
-import { IncrementalEntityProvider, IncrementalEntityProviderOptions } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
+import { IncrementalEntityProviderOptions, IncrementalEntityProvider } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
 
+/**
+ * @alpha
+ * Interface for {@link incrementalIngestionProvidersExtensionPoint}.
+ */
+interface IncrementalIngestionProviderExtensionPoint {
+    /** Adds a new incremental entity provider */
+    addProvider<TCursor, TContext>(config: {
+        options: IncrementalEntityProviderOptions;
+        provider: IncrementalEntityProvider<TCursor, TContext>;
+    }): void;
+}
+/**
+ * @alpha
+ *
+ * Extension point for registering incremental ingestion providers.
+ * The `catalogModuleIncrementalIngestionEntityProvider` must be installed for these providers to work.
+ *
+ * @example
+ *
+ * ```ts
+ * backend.add(createBackendModule({
+ *   pluginId: 'catalog',
+ *   moduleId: 'myIncrementalProvider',
+ *   register(env) {
+ *     env.registerInit({
+ *       deps: {
+ *         extension: incrementalIngestionProvidersExtensionPoint,
+ *       },
+ *       async init({ extension }) {
+ *         extension.addProvider({
+ *           burstInterval: ...,
+ *           burstLength: ...,
+ *           restLength: ...,
+ *         }, {
+ *           next(context, cursor) {
+ *             ...
+ *           },
+ *           ...
+ *         })
+ *       })
+ *     })
+ *   }
+ * }))
+ * ```
+ */
+declare const incrementalIngestionProvidersExtensionPoint: _backstage_backend_plugin_api.ExtensionPoint<IncrementalIngestionProviderExtensionPoint>;
 /**
  * Registers the incremental entity provider with the catalog processing extension point.
  *
  * @alpha
  */
-declare const catalogModuleIncrementalIngestionEntityProvider: (options: {
-    providers: {
-        provider: IncrementalEntityProvider<unknown, unknown>;
-        options: IncrementalEntityProviderOptions;
-    }[];
-}) => _backstage_backend_plugin_api.BackendFeature;
+declare const catalogModuleIncrementalIngestionEntityProvider: () => _backstage_backend_plugin_api.BackendFeature;
 
-export { catalogModuleIncrementalIngestionEntityProvider };
+export { IncrementalIngestionProviderExtensionPoint, catalogModuleIncrementalIngestionEntityProvider, incrementalIngestionProvidersExtensionPoint };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@backstage/plugin-catalog-backend-module-incremental-ingestion",
   "description": "An entity provider for streaming large asset sources into the catalog",
-  "version": "0.4.1-next.2",
+  "version": "0.4.1",
   "main": "./dist/index.cjs.js",
   "types": "./dist/index.d.ts",
   "license": "Apache-2.0",
@@ -43,15 +43,15 @@
     "postpack": "backstage-cli package postpack"
   },
   "dependencies": {
-    "@backstage/backend-common": "^0.19.2-next.2",
-    "@backstage/backend-plugin-api": "^0.6.0-next.2",
-    "@backstage/backend-tasks": "^0.5.5-next.2",
+    "@backstage/backend-common": "^0.19.2",
+    "@backstage/backend-plugin-api": "^0.6.0",
+    "@backstage/backend-tasks": "^0.5.5",
     "@backstage/catalog-model": "^1.4.1",
     "@backstage/config": "^1.0.8",
     "@backstage/errors": "^1.2.1",
-    "@backstage/plugin-catalog-backend": "^1.12.0-next.2",
-    "@backstage/plugin-catalog-node": "^1.4.1-next.2",
-    "@backstage/plugin-events-node": "^0.2.9-next.2",
+    "@backstage/plugin-catalog-backend": "^1.12.0",
+    "@backstage/plugin-catalog-node": "^1.4.1",
+    "@backstage/plugin-events-node": "^0.2.9",
     "@backstage/plugin-permission-common": "^0.7.7",
     "@types/express": "^4.17.6",
     "@types/luxon": "^3.0.0",
@@ -64,10 +64,10 @@
     "winston": "^3.2.1"
   },
   "devDependencies": {
-    "@backstage/backend-app-api": "^0.5.0-next.2",
-    "@backstage/backend-defaults": "^0.2.0-next.2",
-    "@backstage/backend-test-utils": "^0.2.0-next.2",
-    "@backstage/cli": "^0.22.10-next.1"
+    "@backstage/backend-app-api": "^0.5.0",
+    "@backstage/backend-defaults": "^0.2.0",
+    "@backstage/backend-test-utils": "^0.2.0",
+    "@backstage/cli": "^0.22.10"
   },
   "files": [
     "dist",