@backstage/plugin-catalog-backend-module-incremental-ingestion 0.7.15 → 0.8.0-next.1

package/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @backstage/plugin-catalog-backend-module-incremental-ingestion
 
+## 0.8.0-next.1
+
+### Minor Changes
+
+- 305e2f7: **BREAKING**: The incremental ingestion administrative routes now enforce separate read and manage permissions. Installations with custom permission policies must add decisions for the new permissions.
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/plugin-catalog-common@1.2.0-next.0
+  - @backstage/plugin-catalog-backend@4.0.0-next.1
+  - @backstage/config@1.3.9-next.0
+  - @backstage/backend-defaults@0.17.9-next.1
+  - @backstage/plugin-catalog-node@2.2.5-next.1
+  - @backstage/backend-plugin-api@1.10.1-next.1
+  - @backstage/plugin-permission-common@0.9.11-next.0
+
+## 0.7.16-next.0
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/backend-defaults@0.17.9-next.0
+  - @backstage/plugin-catalog-backend@3.9.2-next.0
+  - @backstage/backend-plugin-api@1.10.1-next.0
+  - @backstage/plugin-catalog-node@2.2.5-next.0
+  - @backstage/plugin-events-node@0.4.26-next.0
+
 ## 0.7.15
 
 ### Patch Changes

package/dist/module/WrapperProviders.cjs.js

@@ -31,7 +31,9 @@ class WrapperProviders {
   adminRouter() {
     return new routes.IncrementalProviderRouter(
       new IncrementalIngestionDatabaseManager.IncrementalIngestionDatabaseManager({ client: this.options.client }),
-      this.options.logger
+      this.options.logger,
+      this.options.permissions,
+      this.options.httpAuth
     ).createRouter();
   }
   async startProvider(provider, providerOptions, connection) {

package/dist/module/WrapperProviders.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"WrapperProviders.cjs.js","sources":["../../src/module/WrapperProviders.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  LoggerService,\n  RootConfigService,\n  SchedulerService,\n} from '@backstage/backend-plugin-api';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\nimport { stringifyError } from '@backstage/errors';\nimport {\n  EntityProvider,\n  EntityProviderConnection,\n} from '@backstage/plugin-catalog-node';\nimport { createDeferred } from '@backstage/types';\nimport express from 'express';\nimport { Knex } from 'knex';\nimport { Duration } from 'luxon';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { applyDatabaseMigrations } from '../database/migrations';\nimport { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine';\nimport { IncrementalProviderRouter } from '../router/routes';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '../types';\nimport { EventsService } from '@backstage/plugin-events-node';\n\n/**\n * Helps in the creation of the catalog entity providers that wrap the\n * incremental ones.\n */\nexport class WrapperProviders {\n  private migrate: Promise<void> | undefined;\n  private numberOfProvidersToConnect = 0;\n  private readonly readySignal = createDeferred();\n\n  constructor(\n    private readonly options: {\n      config: RootConfigService;\n      logger: LoggerService;\n      client: Knex;\n      scheduler: SchedulerService;\n      applyDatabaseMigrations?: typeof applyDatabaseMigrations;\n      events: EventsService;\n      metrics: MetricsService;\n    },\n  ) {}\n\n  wrap(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    options: IncrementalEntityProviderOptions,\n  ): EntityProvider {\n    this.numberOfProvidersToConnect += 1;\n    return {\n      getProviderName: () => provider.getProviderName(),\n      connect: async connection => {\n        await this.startProvider(provider, options, connection);\n        this.numberOfProvidersToConnect -= 1;\n        if (this.numberOfProvidersToConnect === 0) {\n          this.readySignal.resolve();\n        }\n      },\n    };\n  }\n\n  adminRouter(): express.Router {\n    return new IncrementalProviderRouter(\n      new IncrementalIngestionDatabaseManager({ client: this.options.client }),\n      this.options.logger,\n    ).createRouter();\n  }\n\n  private async startProvider(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    providerOptions: IncrementalEntityProviderOptions,\n    connection: EntityProviderConnection,\n  ) {\n    const logger = this.options.logger.child({\n      entityProvider: provider.getProviderName(),\n    });\n\n    try {\n      if (!this.migrate) {\n        this.migrate = Promise.resolve().then(async () => {\n          const apply =\n            this.options.applyDatabaseMigrations ?? applyDatabaseMigrations;\n          await apply(this.options.client);\n        });\n      }\n\n      await this.migrate;\n\n      const { burstInterval, burstLength, restLength } = providerOptions;\n\n      logger.info(`Connecting`);\n\n      const manager = new IncrementalIngestionDatabaseManager({\n        client: this.options.client,\n      });\n      const engine = new IncrementalIngestionEngine({\n        ...providerOptions,\n        ready: this.readySignal,\n        manager,\n        logger,\n        provider,\n        restLength,\n        connection,\n        metrics: this.options.metrics,\n      });\n\n      let frequency = Duration.isDuration(burstInterval)\n        ? burstInterval\n        : Duration.fromObject(burstInterval);\n      if (frequency.as('milliseconds') < 5000) {\n        frequency = Duration.fromObject({ seconds: 5 }); // don't let it be silly low, to not overload the scheduler\n      }\n\n      let length = Duration.isDuration(burstLength)\n        ? burstLength\n        : Duration.fromObject(burstLength);\n      length = length.plus(Duration.fromObject({ minutes: 1 })); // some margin from the actual completion\n\n      await this.options.scheduler.scheduleTask({\n        id: provider.getProviderName(),\n        fn: engine.taskFn.bind(engine),\n        frequency,\n        timeout: length,\n      });\n\n      const topics = engine.supportsEventTopics();\n      if (topics.length > 0) {\n        logger.info(\n          `Provider ${provider.getProviderName()} subscribing to events for topics: ${topics.join(\n            ',',\n          )}`,\n        );\n        await this.options.events.subscribe({\n          topics,\n          id: `catalog-backend-module-incremental-ingestion:${provider.getProviderName()}`,\n          onEvent: evt => engine.onEvent(evt),\n        });\n      }\n    } catch (error) {\n      logger.warn(\n        `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${stringifyError(\n          error,\n        )}`,\n      );\n      throw error;\n    }\n  }\n}\n"],"names":["createDeferred","IncrementalProviderRouter","IncrementalIngestionDatabaseManager","applyDatabaseMigrations","IncrementalIngestionEngine","Duration","stringifyError"],"mappings":";;;;;;;;;;AA6CO,MAAM,gBAAA,CAAiB;AAAA,EAK5B,YACmB,OAAA,EASjB;AATiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA,EAShB;AAAA,EAdK,OAAA;AAAA,EACA,0BAAA,GAA6B,CAAA;AAAA,EACpB,cAAcA,oBAAA,EAAe;AAAA,EAc9C,IAAA,CACE,UACA,OAAA,EACgB;AAChB,IAAA,IAAA,CAAK,0BAAA,IAA8B,CAAA;AACnC,IAAA,OAAO;AAAA,MACL,eAAA,EAAiB,MAAM,QAAA,CAAS,eAAA,EAAgB;AAAA,MAChD,OAAA,EAAS,OAAM,UAAA,KAAc;AAC3B,QAAA,MAAM,IAAA,CAAK,aAAA,CAAc,QAAA,EAAU,OAAA,EAAS,UAAU,CAAA;AACtD,QAAA,IAAA,CAAK,0BAAA,IAA8B,CAAA;AACnC,QAAA,IAAI,IAAA,CAAK,+BAA+B,CAAA,EAAG;AACzC,UAAA,IAAA,CAAK,YAAY,OAAA,EAAQ;AAAA,QAC3B;AAAA,MACF;AAAA,KACF;AAAA,EACF;AAAA,EAEA,WAAA,GAA8B;AAC5B,IAAA,OAAO,IAAIC,gCAAA;AAAA,MACT,IAAIC,uEAAA,CAAoC,EAAE,QAAQ,IAAA,CAAK,OAAA,CAAQ,QAAQ,CAAA;AAAA,MACvE,KAAK,OAAA,CAAQ;AAAA,MACb,YAAA,EAAa;AAAA,EACjB;AAAA,EAEA,MAAc,aAAA,CACZ,QAAA,EACA,eAAA,EACA,UAAA,EACA;AACA,IAAA,MAAM,MAAA,GAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM;AAAA,MACvC,cAAA,EAAgB,SAAS,eAAA;AAAgB,KAC1C,CAAA;AAED,IAAA,IAAI;AACF,MAAA,IAAI,CAAC,KAAK,OAAA,EAAS;AACjB,QAAA,IAAA,CAAK,OAAA,GAAU,OAAA,CAAQ,OAAA,EAAQ,CAAE,KAAK,YAAY;AAChD,UAAA,MAAM,KAAA,GACJ,IAAA,CAAK,OAAA,CAAQ,uBAAA,IAA2BC,kCAAA;AAC1C,UAAA,MAAM,KAAA,CAAM,IAAA,CAAK,OAAA,CAAQ,MAAM,CAAA;AAAA,QACjC,CAAC,CAAA;AAAA,MACH;AAEA,MAAA,MAAM,IAAA,CAAK,OAAA;AAEX,MAAA,MAAM,EAAE,aAAA,EAAe,WAAA,EAAa,UAAA,EAAW,GAAI,eAAA;AAEnD,MAAA,MAAA,CAAO,KAAK,CAAA,UAAA,CAAY,CAAA;AAExB,MAAA,MAAM,OAAA,GAAU,IAAID,uEAAA,CAAoC;AAAA,QACtD,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,MAAM,MAAA,GAAS,IAAIE,qDAAA,CAA2B;AAAA,QAC5C,GAAG,eAAA;AAAA,QACH,OAAO,IAAA,CAAK,WAAA;AAAA,QACZ,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,UAAA;AAAA,QACA,OAAA,EAAS,KAAK,OAAA,CAAQ;AAAA,OACvB,CAAA;AAED,MAAA,IAAI,SAAA,GAAYC,eAAS,UAAA,CAAW,aAAa,IAC7C,aAAA,GACAA,cAAA,CAAS,WAAW,aAAa,CAAA;AACrC,MAAA,IAAI,SAAA,CAAU,EAAA,CAAG,cAAc,CAAA,GAAI,GAAA,EAAM;AACvC,QAAA,SAAA,GAAYA,cAAA,CAAS,UAAA,CAAW,EAAE,OAAA,EAAS,GAAG,CAAA;AAAA,MAChD;AAEA,MAAA,IAAI,MAAA,GAASA,eAAS,UAAA,CAAW,WAAW,IACxC,WAAA,GACAA,cAAA,CAAS,WAAW,WAAW,CAAA;AACnC,MAAA,MAAA,GAAS,MAAA,CAAO,KAAKA,cAAA,CAAS,UAAA,CAAW,EAAE,OAAA,EAAS,CAAA,EAAG,CAAC,CAAA;AAExD,MAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa;AAAA,QACxC,EAAA,EAAI,SAAS,eAAA,EAAgB;AAAA,QAC7B,EAAA,EAAI,MAAA,CAAO,MAAA,CAAO,IAAA,CAAK,MAAM,CAAA;AAAA,QAC7B,SAAA;AAAA,QACA,OAAA,EAAS;AAAA,OACV,CAAA;AAED,MAAA,MAAM,MAAA,GAAS,OAAO,mBAAA,EAAoB;AAC1C,MAAA,IAAI,MAAA,CAAO,SAAS,CAAA,EAAG;AACrB,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,SAAA,EAAY,QAAA,CAAS,eAAA,EAAiB,sCAAsC,MAAA,CAAO,IAAA;AAAA,YACjF;AAAA,WACD,CAAA;AAAA,SACH;AACA,QAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,SAAA,CAAU;AAAA,UAClC,MAAA;AAAA,UACA,EAAA,EAAI,CAAA,6CAAA,EAAgD,QAAA,CAAS,eAAA,EAAiB,CAAA,CAAA;AAAA,UAC9E,OAAA,EAAS,CAAA,GAAA,KAAO,MAAA,CAAO,OAAA,CAAQ,GAAG;AAAA,SACnC,CAAA;AAAA,MACH;AAAA,IACF,SAAS,KAAA,EAAO;AACd,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,oDAAA,EAAuD,QAAA,CAAS,eAAA,EAAiB,CAAA,EAAA,EAAKC,qBAAA;AAAA,UACpF;AAAA,SACD,CAAA;AAAA,OACH;AACA,MAAA,MAAM,KAAA;AAAA,IACR;AAAA,EACF;AACF;;;;"}
+{"version":3,"file":"WrapperProviders.cjs.js","sources":["../../src/module/WrapperProviders.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  LoggerService,\n  HttpAuthService,\n  PermissionsService,\n  RootConfigService,\n  SchedulerService,\n} from '@backstage/backend-plugin-api';\nimport { MetricsService } from '@backstage/backend-plugin-api/alpha';\nimport { stringifyError } from '@backstage/errors';\nimport {\n  EntityProvider,\n  EntityProviderConnection,\n} from '@backstage/plugin-catalog-node';\nimport { createDeferred } from '@backstage/types';\nimport express from 'express';\nimport { Knex } from 'knex';\nimport { Duration } from 'luxon';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { applyDatabaseMigrations } from '../database/migrations';\nimport { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine';\nimport { IncrementalProviderRouter } from '../router/routes';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '../types';\nimport { EventsService } from '@backstage/plugin-events-node';\n\n/**\n * Helps in the creation of the catalog entity providers that wrap the\n * incremental ones.\n */\nexport class WrapperProviders {\n  private migrate: Promise<void> | undefined;\n  private numberOfProvidersToConnect = 0;\n  private readonly readySignal = createDeferred();\n\n  constructor(\n    private readonly options: {\n      config: RootConfigService;\n      logger: LoggerService;\n      client: Knex;\n      scheduler: SchedulerService;\n      applyDatabaseMigrations?: typeof applyDatabaseMigrations;\n      events: EventsService;\n      metrics: MetricsService;\n      permissions: PermissionsService;\n      httpAuth: HttpAuthService;\n    },\n  ) {}\n\n  wrap(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    options: IncrementalEntityProviderOptions,\n  ): EntityProvider {\n    this.numberOfProvidersToConnect += 1;\n    return {\n      getProviderName: () => provider.getProviderName(),\n      connect: async connection => {\n        await this.startProvider(provider, options, connection);\n        this.numberOfProvidersToConnect -= 1;\n        if (this.numberOfProvidersToConnect === 0) {\n          this.readySignal.resolve();\n        }\n      },\n    };\n  }\n\n  adminRouter(): express.Router {\n    return new IncrementalProviderRouter(\n      new IncrementalIngestionDatabaseManager({ client: this.options.client }),\n      this.options.logger,\n      this.options.permissions,\n      this.options.httpAuth,\n    ).createRouter();\n  }\n\n  private async startProvider(\n    provider: IncrementalEntityProvider<unknown, unknown>,\n    providerOptions: IncrementalEntityProviderOptions,\n    connection: EntityProviderConnection,\n  ) {\n    const logger = this.options.logger.child({\n      entityProvider: provider.getProviderName(),\n    });\n\n    try {\n      if (!this.migrate) {\n        this.migrate = Promise.resolve().then(async () => {\n          const apply =\n            this.options.applyDatabaseMigrations ?? applyDatabaseMigrations;\n          await apply(this.options.client);\n        });\n      }\n\n      await this.migrate;\n\n      const { burstInterval, burstLength, restLength } = providerOptions;\n\n      logger.info(`Connecting`);\n\n      const manager = new IncrementalIngestionDatabaseManager({\n        client: this.options.client,\n      });\n      const engine = new IncrementalIngestionEngine({\n        ...providerOptions,\n        ready: this.readySignal,\n        manager,\n        logger,\n        provider,\n        restLength,\n        connection,\n        metrics: this.options.metrics,\n      });\n\n      let frequency = Duration.isDuration(burstInterval)\n        ? burstInterval\n        : Duration.fromObject(burstInterval);\n      if (frequency.as('milliseconds') < 5000) {\n        frequency = Duration.fromObject({ seconds: 5 }); // don't let it be silly low, to not overload the scheduler\n      }\n\n      let length = Duration.isDuration(burstLength)\n        ? burstLength\n        : Duration.fromObject(burstLength);\n      length = length.plus(Duration.fromObject({ minutes: 1 })); // some margin from the actual completion\n\n      await this.options.scheduler.scheduleTask({\n        id: provider.getProviderName(),\n        fn: engine.taskFn.bind(engine),\n        frequency,\n        timeout: length,\n      });\n\n      const topics = engine.supportsEventTopics();\n      if (topics.length > 0) {\n        logger.info(\n          `Provider ${provider.getProviderName()} subscribing to events for topics: ${topics.join(\n            ',',\n          )}`,\n        );\n        await this.options.events.subscribe({\n          topics,\n          id: `catalog-backend-module-incremental-ingestion:${provider.getProviderName()}`,\n          onEvent: evt => engine.onEvent(evt),\n        });\n      }\n    } catch (error) {\n      logger.warn(\n        `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${stringifyError(\n          error,\n        )}`,\n      );\n      throw error;\n    }\n  }\n}\n"],"names":["createDeferred","IncrementalProviderRouter","IncrementalIngestionDatabaseManager","applyDatabaseMigrations","IncrementalIngestionEngine","Duration","stringifyError"],"mappings":";;;;;;;;;;AA+CO,MAAM,gBAAA,CAAiB;AAAA,EAK5B,YACmB,OAAA,EAWjB;AAXiB,IAAA,IAAA,CAAA,OAAA,GAAA,OAAA;AAAA,EAWhB;AAAA,EAhBK,OAAA;AAAA,EACA,0BAAA,GAA6B,CAAA;AAAA,EACpB,cAAcA,oBAAA,EAAe;AAAA,EAgB9C,IAAA,CACE,UACA,OAAA,EACgB;AAChB,IAAA,IAAA,CAAK,0BAAA,IAA8B,CAAA;AACnC,IAAA,OAAO;AAAA,MACL,eAAA,EAAiB,MAAM,QAAA,CAAS,eAAA,EAAgB;AAAA,MAChD,OAAA,EAAS,OAAM,UAAA,KAAc;AAC3B,QAAA,MAAM,IAAA,CAAK,aAAA,CAAc,QAAA,EAAU,OAAA,EAAS,UAAU,CAAA;AACtD,QAAA,IAAA,CAAK,0BAAA,IAA8B,CAAA;AACnC,QAAA,IAAI,IAAA,CAAK,+BAA+B,CAAA,EAAG;AACzC,UAAA,IAAA,CAAK,YAAY,OAAA,EAAQ;AAAA,QAC3B;AAAA,MACF;AAAA,KACF;AAAA,EACF;AAAA,EAEA,WAAA,GAA8B;AAC5B,IAAA,OAAO,IAAIC,gCAAA;AAAA,MACT,IAAIC,uEAAA,CAAoC,EAAE,QAAQ,IAAA,CAAK,OAAA,CAAQ,QAAQ,CAAA;AAAA,MACvE,KAAK,OAAA,CAAQ,MAAA;AAAA,MACb,KAAK,OAAA,CAAQ,WAAA;AAAA,MACb,KAAK,OAAA,CAAQ;AAAA,MACb,YAAA,EAAa;AAAA,EACjB;AAAA,EAEA,MAAc,aAAA,CACZ,QAAA,EACA,eAAA,EACA,UAAA,EACA;AACA,IAAA,MAAM,MAAA,GAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM;AAAA,MACvC,cAAA,EAAgB,SAAS,eAAA;AAAgB,KAC1C,CAAA;AAED,IAAA,IAAI;AACF,MAAA,IAAI,CAAC,KAAK,OAAA,EAAS;AACjB,QAAA,IAAA,CAAK,OAAA,GAAU,OAAA,CAAQ,OAAA,EAAQ,CAAE,KAAK,YAAY;AAChD,UAAA,MAAM,KAAA,GACJ,IAAA,CAAK,OAAA,CAAQ,uBAAA,IAA2BC,kCAAA;AAC1C,UAAA,MAAM,KAAA,CAAM,IAAA,CAAK,OAAA,CAAQ,MAAM,CAAA;AAAA,QACjC,CAAC,CAAA;AAAA,MACH;AAEA,MAAA,MAAM,IAAA,CAAK,OAAA;AAEX,MAAA,MAAM,EAAE,aAAA,EAAe,WAAA,EAAa,UAAA,EAAW,GAAI,eAAA;AAEnD,MAAA,MAAA,CAAO,KAAK,CAAA,UAAA,CAAY,CAAA;AAExB,MAAA,MAAM,OAAA,GAAU,IAAID,uEAAA,CAAoC;AAAA,QACtD,MAAA,EAAQ,KAAK,OAAA,CAAQ;AAAA,OACtB,CAAA;AACD,MAAA,MAAM,MAAA,GAAS,IAAIE,qDAAA,CAA2B;AAAA,QAC5C,GAAG,eAAA;AAAA,QACH,OAAO,IAAA,CAAK,WAAA;AAAA,QACZ,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,UAAA;AAAA,QACA,OAAA,EAAS,KAAK,OAAA,CAAQ;AAAA,OACvB,CAAA;AAED,MAAA,IAAI,SAAA,GAAYC,eAAS,UAAA,CAAW,aAAa,IAC7C,aAAA,GACAA,cAAA,CAAS,WAAW,aAAa,CAAA;AACrC,MAAA,IAAI,SAAA,CAAU,EAAA,CAAG,cAAc,CAAA,GAAI,GAAA,EAAM;AACvC,QAAA,SAAA,GAAYA,cAAA,CAAS,UAAA,CAAW,EAAE,OAAA,EAAS,GAAG,CAAA;AAAA,MAChD;AAEA,MAAA,IAAI,MAAA,GAASA,eAAS,UAAA,CAAW,WAAW,IACxC,WAAA,GACAA,cAAA,CAAS,WAAW,WAAW,CAAA;AACnC,MAAA,MAAA,GAAS,MAAA,CAAO,KAAKA,cAAA,CAAS,UAAA,CAAW,EAAE,OAAA,EAAS,CAAA,EAAG,CAAC,CAAA;AAExD,MAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,SAAA,CAAU,YAAA,CAAa;AAAA,QACxC,EAAA,EAAI,SAAS,eAAA,EAAgB;AAAA,QAC7B,EAAA,EAAI,MAAA,CAAO,MAAA,CAAO,IAAA,CAAK,MAAM,CAAA;AAAA,QAC7B,SAAA;AAAA,QACA,OAAA,EAAS;AAAA,OACV,CAAA;AAED,MAAA,MAAM,MAAA,GAAS,OAAO,mBAAA,EAAoB;AAC1C,MAAA,IAAI,MAAA,CAAO,SAAS,CAAA,EAAG;AACrB,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,SAAA,EAAY,QAAA,CAAS,eAAA,EAAiB,sCAAsC,MAAA,CAAO,IAAA;AAAA,YACjF;AAAA,WACD,CAAA;AAAA,SACH;AACA,QAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,SAAA,CAAU;AAAA,UAClC,MAAA;AAAA,UACA,EAAA,EAAI,CAAA,6CAAA,EAAgD,QAAA,CAAS,eAAA,EAAiB,CAAA,CAAA;AAAA,UAC9E,OAAA,EAAS,CAAA,GAAA,KAAO,MAAA,CAAO,OAAA,CAAQ,GAAG;AAAA,SACnC,CAAA;AAAA,MACH;AAAA,IACF,SAAS,KAAA,EAAO;AACd,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,oDAAA,EAAuD,QAAA,CAAS,eAAA,EAAiB,CAAA,EAAA,EAAKC,qBAAA;AAAA,UACpF;AAAA,SACD,CAAA;AAAA,OACH;AACA,MAAA,MAAM,KAAA;AAAA,IACR;AAAA,EACF;AACF;;;;"}

package/dist/module/catalogModuleIncrementalIngestionEntityProvider.cjs.js

@@ -28,7 +28,9 @@ const catalogModuleIncrementalIngestionEntityProvider = backendPluginApi.createB
         logger: backendPluginApi.coreServices.logger,
         scheduler: backendPluginApi.coreServices.scheduler,
         events: pluginEventsNode.eventsServiceRef,
-        metrics: alpha.metricsServiceRef
+        metrics: alpha.metricsServiceRef,
+        permissions: backendPluginApi.coreServices.permissions,
+        httpAuth: backendPluginApi.coreServices.httpAuth
       },
       async init({
         catalog,
@@ -38,7 +40,9 @@ const catalogModuleIncrementalIngestionEntityProvider = backendPluginApi.createB
         logger,
         scheduler,
         events,
-        metrics
+        metrics,
+        permissions,
+        httpAuth
       }) {
         const client = await database.getClient();
         const providers = new WrapperProviders.WrapperProviders({
@@ -47,7 +51,9 @@ const catalogModuleIncrementalIngestionEntityProvider = backendPluginApi.createB
           client,
           scheduler,
           events,
-          metrics
+          metrics,
+          permissions,
+          httpAuth
         });
         for (const entry of addedProviders) {
           const wrapped = providers.wrap(entry.provider, entry.options);

package/dist/module/catalogModuleIncrementalIngestionEntityProvider.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"catalogModuleIncrementalIngestionEntityProvider.cjs.js","sources":["../../src/module/catalogModuleIncrementalIngestionEntityProvider.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  coreServices,\n  createBackendModule,\n  createExtensionPoint,\n} from '@backstage/backend-plugin-api';\nimport { metricsServiceRef } from '@backstage/backend-plugin-api/alpha';\nimport { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node';\nimport { WrapperProviders } from './WrapperProviders';\nimport { eventsServiceRef } from '@backstage/plugin-events-node';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '../types';\n\n/**\n * @public\n * Interface for {@link incrementalIngestionProvidersExtensionPoint}.\n */\nexport interface IncrementalIngestionProviderExtensionPoint {\n  /** Adds a new incremental entity provider */\n  addProvider<TCursor, TContext>(config: {\n    options: IncrementalEntityProviderOptions;\n    provider: IncrementalEntityProvider<TCursor, TContext>;\n  }): void;\n}\n\n/**\n * @public\n *\n * Extension point for registering incremental ingestion providers.\n * The `catalogModuleIncrementalIngestionEntityProvider` must be installed for these providers to work.\n *\n * @example\n *\n * ```ts\n * backend.add(createBackendModule({\n *   pluginId: 'catalog',\n *   moduleId: 'my-incremental-provider',\n *   register(env) {\n *     env.registerInit({\n *       deps: {\n *         extension: incrementalIngestionProvidersExtensionPoint,\n *       },\n *       async init({ extension }) {\n *         extension.addProvider({\n *           burstInterval: ...,\n *           burstLength: ...,\n *           restLength: ...,\n *         }, {\n *           next(context, cursor) {\n *             ...\n *           },\n *           ...\n *         })\n *       })\n *     })\n *   }\n * }))\n * ```\n */\nexport const incrementalIngestionProvidersExtensionPoint =\n  createExtensionPoint<IncrementalIngestionProviderExtensionPoint>({\n    id: 'catalog.incrementalIngestionProvider.providers',\n  });\n\n/**\n * Registers the incremental entity provider with the catalog processing extension point.\n *\n * @public\n */\nexport const catalogModuleIncrementalIngestionEntityProvider =\n  createBackendModule({\n    pluginId: 'catalog',\n    moduleId: 'incremental-ingestion-entity-provider',\n    register(env) {\n      const addedProviders = new Array<{\n        provider: IncrementalEntityProvider<unknown, unknown>;\n        options: IncrementalEntityProviderOptions;\n      }>();\n\n      env.registerExtensionPoint(incrementalIngestionProvidersExtensionPoint, {\n        addProvider({ options, provider }) {\n          addedProviders.push({ options, provider });\n        },\n      });\n\n      env.registerInit({\n        deps: {\n          catalog: catalogProcessingExtensionPoint,\n          config: coreServices.rootConfig,\n          database: coreServices.database,\n          httpRouter: coreServices.httpRouter,\n          logger: coreServices.logger,\n          scheduler: coreServices.scheduler,\n          events: eventsServiceRef,\n          metrics: metricsServiceRef,\n        },\n        async init({\n          catalog,\n          config,\n          database,\n          httpRouter,\n          logger,\n          scheduler,\n          events,\n          metrics,\n        }) {\n          const client = await database.getClient();\n\n          const providers = new WrapperProviders({\n            config,\n            logger,\n            client,\n            scheduler,\n            events,\n            metrics,\n          });\n\n          for (const entry of addedProviders) {\n            const wrapped = providers.wrap(entry.provider, entry.options);\n            catalog.addEntityProvider(wrapped);\n          }\n\n          httpRouter.use(providers.adminRouter());\n        },\n      });\n    },\n  });\n"],"names":["createExtensionPoint","createBackendModule","catalogProcessingExtensionPoint","coreServices","eventsServiceRef","metricsServiceRef","WrapperProviders"],"mappings":";;;;;;;;AA4EO,MAAM,8CACXA,qCAAA,CAAiE;AAAA,EAC/D,EAAA,EAAI;AACN,CAAC;AAOI,MAAM,kDACXC,oCAAA,CAAoB;AAAA,EAClB,QAAA,EAAU,SAAA;AAAA,EACV,QAAA,EAAU,uCAAA;AAAA,EACV,SAAS,GAAA,EAAK;AACZ,IAAA,MAAM,cAAA,GAAiB,IAAI,KAAA,EAGxB;AAEH,IAAA,GAAA,CAAI,uBAAuB,2CAAA,EAA6C;AAAA,MACtE,WAAA,CAAY,EAAE,OAAA,EAAS,QAAA,EAAS,EAAG;AACjC,QAAA,cAAA,CAAe,IAAA,CAAK,EAAE,OAAA,EAAS,QAAA,EAAU,CAAA;AAAA,MAC3C;AAAA,KACD,CAAA;AAED,IAAA,GAAA,CAAI,YAAA,CAAa;AAAA,MACf,IAAA,EAAM;AAAA,QACJ,OAAA,EAASC,iDAAA;AAAA,QACT,QAAQC,6BAAA,CAAa,UAAA;AAAA,QACrB,UAAUA,6BAAA,CAAa,QAAA;AAAA,QACvB,YAAYA,6BAAA,CAAa,UAAA;AAAA,QACzB,QAAQA,6BAAA,CAAa,MAAA;AAAA,QACrB,WAAWA,6BAAA,CAAa,SAAA;AAAA,QACxB,MAAA,EAAQC,iCAAA;AAAA,QACR,OAAA,EAASC;AAAA,OACX;AAAA,MACA,MAAM,IAAA,CAAK;AAAA,QACT,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,MAAA;AAAA,QACA,SAAA;AAAA,QACA,MAAA;AAAA,QACA;AAAA,OACF,EAAG;AACD,QAAA,MAAM,MAAA,GAAS,MAAM,QAAA,CAAS,SAAA,EAAU;AAExC,QAAA,MAAM,SAAA,GAAY,IAAIC,iCAAA,CAAiB;AAAA,UACrC,MAAA;AAAA,UACA,MAAA;AAAA,UACA,MAAA;AAAA,UACA,SAAA;AAAA,UACA,MAAA;AAAA,UACA;AAAA,SACD,CAAA;AAED,QAAA,KAAA,MAAW,SAAS,cAAA,EAAgB;AAClC,UAAA,MAAM,UAAU,SAAA,CAAU,IAAA,CAAK,KAAA,CAAM,QAAA,EAAU,MAAM,OAAO,CAAA;AAC5D,UAAA,OAAA,CAAQ,kBAAkB,OAAO,CAAA;AAAA,QACnC;AAEA,QAAA,UAAA,CAAW,GAAA,CAAI,SAAA,CAAU,WAAA,EAAa,CAAA;AAAA,MACxC;AAAA,KACD,CAAA;AAAA,EACH;AACF,CAAC;;;;;"}
+{"version":3,"file":"catalogModuleIncrementalIngestionEntityProvider.cjs.js","sources":["../../src/module/catalogModuleIncrementalIngestionEntityProvider.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  coreServices,\n  createBackendModule,\n  createExtensionPoint,\n} from '@backstage/backend-plugin-api';\nimport { metricsServiceRef } from '@backstage/backend-plugin-api/alpha';\nimport { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node';\nimport { WrapperProviders } from './WrapperProviders';\nimport { eventsServiceRef } from '@backstage/plugin-events-node';\nimport {\n  IncrementalEntityProvider,\n  IncrementalEntityProviderOptions,\n} from '../types';\n\n/**\n * @public\n * Interface for {@link incrementalIngestionProvidersExtensionPoint}.\n */\nexport interface IncrementalIngestionProviderExtensionPoint {\n  /** Adds a new incremental entity provider */\n  addProvider<TCursor, TContext>(config: {\n    options: IncrementalEntityProviderOptions;\n    provider: IncrementalEntityProvider<TCursor, TContext>;\n  }): void;\n}\n\n/**\n * @public\n *\n * Extension point for registering incremental ingestion providers.\n * The `catalogModuleIncrementalIngestionEntityProvider` must be installed for these providers to work.\n *\n * @example\n *\n * ```ts\n * backend.add(createBackendModule({\n *   pluginId: 'catalog',\n *   moduleId: 'my-incremental-provider',\n *   register(env) {\n *     env.registerInit({\n *       deps: {\n *         extension: incrementalIngestionProvidersExtensionPoint,\n *       },\n *       async init({ extension }) {\n *         extension.addProvider({\n *           burstInterval: ...,\n *           burstLength: ...,\n *           restLength: ...,\n *         }, {\n *           next(context, cursor) {\n *             ...\n *           },\n *           ...\n *         })\n *       })\n *     })\n *   }\n * }))\n * ```\n */\nexport const incrementalIngestionProvidersExtensionPoint =\n  createExtensionPoint<IncrementalIngestionProviderExtensionPoint>({\n    id: 'catalog.incrementalIngestionProvider.providers',\n  });\n\n/**\n * Registers the incremental entity provider with the catalog processing extension point.\n *\n * @public\n */\nexport const catalogModuleIncrementalIngestionEntityProvider =\n  createBackendModule({\n    pluginId: 'catalog',\n    moduleId: 'incremental-ingestion-entity-provider',\n    register(env) {\n      const addedProviders = new Array<{\n        provider: IncrementalEntityProvider<unknown, unknown>;\n        options: IncrementalEntityProviderOptions;\n      }>();\n\n      env.registerExtensionPoint(incrementalIngestionProvidersExtensionPoint, {\n        addProvider({ options, provider }) {\n          addedProviders.push({ options, provider });\n        },\n      });\n\n      env.registerInit({\n        deps: {\n          catalog: catalogProcessingExtensionPoint,\n          config: coreServices.rootConfig,\n          database: coreServices.database,\n          httpRouter: coreServices.httpRouter,\n          logger: coreServices.logger,\n          scheduler: coreServices.scheduler,\n          events: eventsServiceRef,\n          metrics: metricsServiceRef,\n          permissions: coreServices.permissions,\n          httpAuth: coreServices.httpAuth,\n        },\n        async init({\n          catalog,\n          config,\n          database,\n          httpRouter,\n          logger,\n          scheduler,\n          events,\n          metrics,\n          permissions,\n          httpAuth,\n        }) {\n          const client = await database.getClient();\n\n          const providers = new WrapperProviders({\n            config,\n            logger,\n            client,\n            scheduler,\n            events,\n            metrics,\n            permissions,\n            httpAuth,\n          });\n\n          for (const entry of addedProviders) {\n            const wrapped = providers.wrap(entry.provider, entry.options);\n            catalog.addEntityProvider(wrapped);\n          }\n\n          httpRouter.use(providers.adminRouter());\n        },\n      });\n    },\n  });\n"],"names":["createExtensionPoint","createBackendModule","catalogProcessingExtensionPoint","coreServices","eventsServiceRef","metricsServiceRef","WrapperProviders"],"mappings":";;;;;;;;AA4EO,MAAM,8CACXA,qCAAA,CAAiE;AAAA,EAC/D,EAAA,EAAI;AACN,CAAC;AAOI,MAAM,kDACXC,oCAAA,CAAoB;AAAA,EAClB,QAAA,EAAU,SAAA;AAAA,EACV,QAAA,EAAU,uCAAA;AAAA,EACV,SAAS,GAAA,EAAK;AACZ,IAAA,MAAM,cAAA,GAAiB,IAAI,KAAA,EAGxB;AAEH,IAAA,GAAA,CAAI,uBAAuB,2CAAA,EAA6C;AAAA,MACtE,WAAA,CAAY,EAAE,OAAA,EAAS,QAAA,EAAS,EAAG;AACjC,QAAA,cAAA,CAAe,IAAA,CAAK,EAAE,OAAA,EAAS,QAAA,EAAU,CAAA;AAAA,MAC3C;AAAA,KACD,CAAA;AAED,IAAA,GAAA,CAAI,YAAA,CAAa;AAAA,MACf,IAAA,EAAM;AAAA,QACJ,OAAA,EAASC,iDAAA;AAAA,QACT,QAAQC,6BAAA,CAAa,UAAA;AAAA,QACrB,UAAUA,6BAAA,CAAa,QAAA;AAAA,QACvB,YAAYA,6BAAA,CAAa,UAAA;AAAA,QACzB,QAAQA,6BAAA,CAAa,MAAA;AAAA,QACrB,WAAWA,6BAAA,CAAa,SAAA;AAAA,QACxB,MAAA,EAAQC,iCAAA;AAAA,QACR,OAAA,EAASC,uBAAA;AAAA,QACT,aAAaF,6BAAA,CAAa,WAAA;AAAA,QAC1B,UAAUA,6BAAA,CAAa;AAAA,OACzB;AAAA,MACA,MAAM,IAAA,CAAK;AAAA,QACT,OAAA;AAAA,QACA,MAAA;AAAA,QACA,QAAA;AAAA,QACA,UAAA;AAAA,QACA,MAAA;AAAA,QACA,SAAA;AAAA,QACA,MAAA;AAAA,QACA,OAAA;AAAA,QACA,WAAA;AAAA,QACA;AAAA,OACF,EAAG;AACD,QAAA,MAAM,MAAA,GAAS,MAAM,QAAA,CAAS,SAAA,EAAU;AAExC,QAAA,MAAM,SAAA,GAAY,IAAIG,iCAAA,CAAiB;AAAA,UACrC,MAAA;AAAA,UACA,MAAA;AAAA,UACA,MAAA;AAAA,UACA,SAAA;AAAA,UACA,MAAA;AAAA,UACA,OAAA;AAAA,UACA,WAAA;AAAA,UACA;AAAA,SACD,CAAA;AAED,QAAA,KAAA,MAAW,SAAS,cAAA,EAAgB;AAClC,UAAA,MAAM,UAAU,SAAA,CAAU,IAAA,CAAK,KAAA,CAAM,QAAA,EAAU,MAAM,OAAO,CAAA;AAC5D,UAAA,OAAA,CAAQ,kBAAkB,OAAO,CAAA;AAAA,QACnC;AAEA,QAAA,UAAA,CAAW,GAAA,CAAI,SAAA,CAAU,WAAA,EAAa,CAAA;AAAA,MACxC;AAAA,KACD,CAAA;AAAA,EACH;AACF,CAAC;;;;;"}

package/dist/router/routes.cjs.js

@@ -1,5 +1,8 @@
 'use strict';
 
+var errors = require('@backstage/errors');
+var alpha = require('@backstage/plugin-catalog-common/alpha');
+var pluginPermissionCommon = require('@backstage/plugin-permission-common');
 var express = require('express');
 var Router = require('express-promise-router');
 
@@ -9,15 +12,27 @@ var express__default = /*#__PURE__*/_interopDefaultCompat(express);
 var Router__default = /*#__PURE__*/_interopDefaultCompat(Router);
 
 class IncrementalProviderRouter {
-  manager;
-  logger;
-  constructor(manager, logger) {
+  constructor(manager, logger, permissions, httpAuth) {
+    this.permissions = permissions;
+    this.httpAuth = httpAuth;
     this.manager = manager;
     this.logger = logger;
   }
+  manager;
+  logger;
   createRouter() {
     const router = Router__default.default();
     router.use(express__default.default.json());
+    router.use("/incremental", async (req, _res, next) => {
+      const permission = req.method === "GET" || req.method === "HEAD" ? alpha.catalogIngestionReadPermission : alpha.catalogIngestionManagePermission;
+      const [decision] = await this.permissions.authorize([{ permission }], {
+        credentials: await this.httpAuth.credentials(req)
+      });
+      if (decision.result === pluginPermissionCommon.AuthorizeResult.DENY) {
+        throw new errors.NotAllowedError("Unauthorized");
+      }
+      next();
+    });
     router.get("/incremental/health", async (_, res) => {
       const records = await this.manager.healthcheck();
       const providers = records.map((record) => record.provider_name);

package/dist/router/routes.cjs.js.map

@@ -1 +1 @@
-{"version":3,"file":"routes.cjs.js","sources":["../../src/router/routes.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport express from 'express';\nimport Router from 'express-promise-router';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\nexport class IncrementalProviderRouter {\n  private manager: IncrementalIngestionDatabaseManager;\n  private logger: LoggerService;\n\n  constructor(\n    manager: IncrementalIngestionDatabaseManager,\n    logger: LoggerService,\n  ) {\n    this.manager = manager;\n    this.logger = logger;\n  }\n\n  createRouter(): express.Router {\n    const router = Router();\n    router.use(express.json());\n\n    // Get the overall health of all incremental providers\n    router.get('/incremental/health', async (_, res) => {\n      const records = await this.manager.healthcheck();\n      const providers = records.map(record => record.provider_name);\n      const duplicates = [\n        ...new Set(providers.filter((e, i, a) => a.indexOf(e) !== i)),\n      ];\n\n      if (duplicates.length > 0) {\n        res.json({ healthy: false, duplicateIngestions: duplicates });\n      } else {\n        res.json({ healthy: true });\n      }\n    });\n\n    // Clean up and pause all providers\n    router.post('/incremental/cleanup', async (_, res) => {\n      const result = await this.manager.cleanupProviders();\n      res.json(result);\n    });\n\n    // Get basic status of the provider\n    router.get('/incremental/providers/:provider', async (req, res) => {\n      const { provider } = req.params;\n      const record = await this.manager.getCurrentIngestionRecord(provider);\n      if (record) {\n        res.json({\n          success: true,\n          status: {\n            current_action: record.status,\n            next_action_at: new Date(record.next_action_at),\n          },\n          last_error: record.last_error,\n        });\n      } else {\n        const providers: string[] = await this.manager.listProviders();\n        if (providers.includes(provider)) {\n          res.json({\n            success: true,\n            status: {\n              current_action: 'rest complete, waiting to start',\n            },\n          });\n        } else {\n          this.logger.error(\n            `${provider} - No ingestion record found in the database!`,\n          );\n          res.status(404).json({\n            success: false,\n            status: {},\n            last_error: `Provider '${provider}' not found`,\n          });\n        }\n      }\n    });\n\n    // Trigger the provider's next action\n    router.post(\n      `/incremental/providers/:provider/trigger`,\n      async (req, res) => {\n        const { provider } = req.params;\n        const record = await this.manager.getCurrentIngestionRecord(provider);\n        if (record) {\n          await this.manager.triggerNextProviderAction(provider);\n          res.json({\n            success: true,\n            message: `${provider}: Next action triggered.`,\n          });\n        } else {\n          const providers: string[] = await this.manager.listProviders();\n          if (providers.includes(provider)) {\n            this.logger.debug(`${provider} - Ingestion record found`);\n            res.json({\n              success: true,\n              message: 'Unable to trigger next action (provider is restarting)',\n            });\n          } else {\n            res.status(404).json({\n              success: false,\n              message: `Provider '${provider}' not found`,\n            });\n          }\n        }\n      },\n    );\n\n    // Start a brand-new ingestion cycle for the provider.\n    // (Cancel's the current run if active, or marks it complete if resting)\n    router.post(`/incremental/providers/:provider/start`, async (req, res) => {\n      const { provider } = req.params;\n\n      const record = await this.manager.getCurrentIngestionRecord(provider);\n      if (record) {\n        const ingestionId = record.id;\n        if (record.status === 'resting') {\n          await this.manager.setProviderComplete(ingestionId);\n        } else {\n          await this.manager.setProviderCanceling(ingestionId);\n        }\n        res.json({\n          success: true,\n          message: `${provider}: Next cycle triggered.`,\n        });\n      } else {\n        const providers: string[] = await this.manager.listProviders();\n        if (providers.includes(provider)) {\n          this.logger.debug(`${provider} - Ingestion record found`);\n          res.json({\n            success: true,\n            message: 'Provider is already restarting',\n          });\n        } else {\n          res.status(404).json({\n            success: false,\n            message: `Provider '${provider}' not found`,\n          });\n        }\n      }\n    });\n\n    router.get(`/incremental/providers`, async (_req, res) => {\n      const providers = await this.manager.listProviders();\n\n      res.json({\n        success: true,\n        providers,\n      });\n    });\n\n    // Stop the provider and pause it for 24 hours\n    router.post(`/incremental/providers/:provider/cancel`, async (req, res) => {\n      const { provider } = req.params;\n      const record = await this.manager.getCurrentIngestionRecord(provider);\n      if (record) {\n        const next_action_at = new Date();\n        next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1000);\n        await this.manager.updateByName(provider, {\n          next_action: 'nothing (done)',\n          ingestion_completed_at: new Date(),\n          next_action_at,\n          status: 'resting',\n        });\n        res.json({\n          success: true,\n          message: `${provider}: Current ingestion canceled.`,\n        });\n      } else {\n        const providers: string[] = await this.manager.listProviders();\n        if (providers.includes(provider)) {\n          this.logger.debug(`${provider} - Ingestion record found`);\n          res.json({\n            success: true,\n            message: 'Provider is currently restarting, please wait.',\n          });\n        } else {\n          res.status(404).json({\n            success: false,\n            message: `Provider '${provider}' not found`,\n          });\n        }\n      }\n    });\n\n    // Wipe out all ingestion records for the provider and pause for 24 hours\n    router.delete('/incremental/providers/:provider', async (req, res) => {\n      const { provider } = req.params;\n      const result = await this.manager.purgeAndResetProvider(provider);\n      res.json(result);\n    });\n\n    // Get the ingestion marks for the current cycle\n    router.get(`/incremental/providers/:provider/marks`, async (req, res) => {\n      const { provider } = req.params;\n      const record = await this.manager.getCurrentIngestionRecord(provider);\n      if (record) {\n        const id = record.id;\n        const records = await this.manager.getAllMarks(id);\n        res.json({ success: true, records });\n      } else {\n        const providers: string[] = await this.manager.listProviders();\n        if (providers.includes(provider)) {\n          this.logger.debug(`${provider} - Ingestion record found`);\n          res.json({\n            success: true,\n            message: 'No records yet (provider is restarting)',\n          });\n        } else {\n          this.logger.error(\n            `${provider} - No ingestion record found in the database!`,\n          );\n          res.status(404).json({\n            success: false,\n            status: {},\n            last_error: `Provider '${provider}' not found`,\n          });\n        }\n      }\n    });\n\n    router.delete(\n      `/incremental/providers/:provider/marks`,\n      async (req, res) => {\n        const { provider } = req.params;\n        const deletions = await this.manager.clearFinishedIngestions(provider);\n\n        res.json({\n          success: true,\n          message: `Expired marks for provider '${provider}' removed.`,\n          deletions,\n        });\n      },\n    );\n\n    return router;\n  }\n}\n"],"names":["Router","express"],"mappings":";;;;;;;;;;AAqBO,MAAM,yBAAA,CAA0B;AAAA,EAC7B,OAAA;AAAA,EACA,MAAA;AAAA,EAER,WAAA,CACE,SACA,MAAA,EACA;AACA,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AACf,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AAAA,EAChB;AAAA,EAEA,YAAA,GAA+B;AAC7B,IAAA,MAAM,SAASA,uBAAA,EAAO;AACtB,IAAA,MAAA,CAAO,GAAA,CAAIC,wBAAA,CAAQ,IAAA,EAAM,CAAA;AAGzB,IAAA,MAAA,CAAO,GAAA,CAAI,qBAAA,EAAuB,OAAO,CAAA,EAAG,GAAA,KAAQ;AAClD,MAAA,MAAM,OAAA,GAAU,MAAM,IAAA,CAAK,OAAA,CAAQ,WAAA,EAAY;AAC/C,MAAA,MAAM,SAAA,GAAY,OAAA,CAAQ,GAAA,CAAI,CAAA,MAAA,KAAU,OAAO,aAAa,CAAA;AAC5D,MAAA,MAAM,UAAA,GAAa;AAAA,QACjB,GAAG,IAAI,GAAA,CAAI,SAAA,CAAU,OAAO,CAAC,CAAA,EAAG,CAAA,EAAG,CAAA,KAAM,CAAA,CAAE,OAAA,CAAQ,CAAC,CAAA,KAAM,CAAC,CAAC;AAAA,OAC9D;AAEA,MAAA,IAAI,UAAA,CAAW,SAAS,CAAA,EAAG;AACzB,QAAA,GAAA,CAAI,KAAK,EAAE,OAAA,EAAS,KAAA,EAAO,mBAAA,EAAqB,YAAY,CAAA;AAAA,MAC9D,CAAA,MAAO;AACL,QAAA,GAAA,CAAI,IAAA,CAAK,EAAE,OAAA,EAAS,IAAA,EAAM,CAAA;AAAA,MAC5B;AAAA,IACF,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,IAAA,CAAK,sBAAA,EAAwB,OAAO,CAAA,EAAG,GAAA,KAAQ;AACpD,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,gBAAA,EAAiB;AACnD,MAAA,GAAA,CAAI,KAAK,MAAM,CAAA;AAAA,IACjB,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,GAAA,CAAI,kCAAA,EAAoC,OAAO,GAAA,EAAK,GAAA,KAAQ;AACjE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,GAAA,CAAI,IAAA,CAAK;AAAA,UACP,OAAA,EAAS,IAAA;AAAA,UACT,MAAA,EAAQ;AAAA,YACN,gBAAgB,MAAA,CAAO,MAAA;AAAA,YACvB,cAAA,EAAgB,IAAI,IAAA,CAAK,MAAA,CAAO,cAAc;AAAA,WAChD;AAAA,UACA,YAAY,MAAA,CAAO;AAAA,SACpB,CAAA;AAAA,MACH,CAAA,MAAO;AACL,QAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,QAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,MAAA,EAAQ;AAAA,cACN,cAAA,EAAgB;AAAA;AAClB,WACD,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,YACV,GAAG,QAAQ,CAAA,6CAAA;AAAA,WACb;AACA,UAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,YACnB,OAAA,EAAS,KAAA;AAAA,YACT,QAAQ,EAAC;AAAA,YACT,UAAA,EAAY,aAAa,QAAQ,CAAA,WAAA;AAAA,WAClC,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,CAAA,wCAAA,CAAA;AAAA,MACA,OAAO,KAAK,GAAA,KAAQ;AAClB,QAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,QAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,QAAA,IAAI,MAAA,EAAQ;AACV,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,yBAAA,CAA0B,QAAQ,CAAA;AACrD,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,OAAA,EAAS,GAAG,QAAQ,CAAA,wBAAA;AAAA,WACrB,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,UAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,YAAA,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA,yBAAA,CAA2B,CAAA;AACxD,YAAA,GAAA,CAAI,IAAA,CAAK;AAAA,cACP,OAAA,EAAS,IAAA;AAAA,cACT,OAAA,EAAS;AAAA,aACV,CAAA;AAAA,UACH,CAAA,MAAO;AACL,YAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,cACnB,OAAA,EAAS,KAAA;AAAA,cACT,OAAA,EAAS,aAAa,QAAQ,CAAA,WAAA;AAAA,aAC/B,CAAA;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,KACF;AAIA,IAAA,MAAA,CAAO,IAAA,CAAK,CAAA,sCAAA,CAAA,EAA0C,OAAO,GAAA,EAAK,GAAA,KAAQ;AACxE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AAEzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,MAAM,cAAc,MAAA,CAAO,EAAA;AAC3B,QAAA,IAAI,MAAA,CAAO,WAAW,SAAA,EAAW;AAC/B,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,mBAAA,CAAoB,WAAW,CAAA;AAAA,QACpD,CAAA,MAAO;AACL,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,oBAAA,CAAqB,WAAW,CAAA;AAAA,QACrD;AACA,QAAA,GAAA,CAAI,IAAA,CAAK;AAAA,UACP,OAAA,EAAS,IAAA;AAAA,UACT,OAAA,EAAS,GAAG,QAAQ,CAAA,uBAAA;AAAA,SACrB,CAAA;AAAA,MACH,CAAA,MAAO;AACL,QAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,QAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA,yBAAA,CAA2B,CAAA;AACxD,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,OAAA,EAAS;AAAA,WACV,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,YACnB,OAAA,EAAS,KAAA;AAAA,YACT,OAAA,EAAS,aAAa,QAAQ,CAAA,WAAA;AAAA,WAC/B,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAED,IAAA,MAAA,CAAO,GAAA,CAAI,CAAA,sBAAA,CAAA,EAA0B,OAAO,IAAA,EAAM,GAAA,KAAQ;AACxD,MAAA,MAAM,SAAA,GAAY,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAEnD,MAAA,GAAA,CAAI,IAAA,CAAK;AAAA,QACP,OAAA,EAAS,IAAA;AAAA,QACT;AAAA,OACD,CAAA;AAAA,IACH,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,IAAA,CAAK,CAAA,uCAAA,CAAA,EAA2C,OAAO,GAAA,EAAK,GAAA,KAAQ;AACzE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,MAAM,cAAA,uBAAqB,IAAA,EAAK;AAChC,QAAA,cAAA,CAAe,QAAQ,cAAA,CAAe,OAAA,KAAY,EAAA,GAAK,EAAA,GAAK,KAAK,GAAI,CAAA;AACrE,QAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,YAAA,CAAa,QAAA,EAAU;AAAA,UACxC,WAAA,EAAa,gBAAA;AAAA,UACb,sBAAA,sBAA4B,IAAA,EAAK;AAAA,UACjC,cAAA;AAAA,UACA,MAAA,EAAQ;AAAA,SACT,CAAA;AACD,QAAA,GAAA,CAAI,IAAA,CAAK;AAAA,UACP,OAAA,EAAS,IAAA;AAAA,UACT,OAAA,EAAS,GAAG,QAAQ,CAAA,6BAAA;AAAA,SACrB,CAAA;AAAA,MACH,CAAA,MAAO;AACL,QAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,QAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA,yBAAA,CAA2B,CAAA;AACxD,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,OAAA,EAAS;AAAA,WACV,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,YACnB,OAAA,EAAS,KAAA;AAAA,YACT,OAAA,EAAS,aAAa,QAAQ,CAAA,WAAA;AAAA,WAC/B,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,MAAA,CAAO,kCAAA,EAAoC,OAAO,GAAA,EAAK,GAAA,KAAQ;AACpE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,sBAAsB,QAAQ,CAAA;AAChE,MAAA,GAAA,CAAI,KAAK,MAAM,CAAA;AAAA,IACjB,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,GAAA,CAAI,CAAA,sCAAA,CAAA,EAA0C,OAAO,GAAA,EAAK,GAAA,KAAQ;AACvE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,MAAM,KAAK,MAAA,CAAO,EAAA;AAClB,QAAA,MAAM,OAAA,GAAU,MAAM,IAAA,CAAK,OAAA,CAAQ,YAAY,EAAE,CAAA;AACjD,QAAA,GAAA,CAAI,IAAA,CAAK,EAAE,OAAA,EAAS,IAAA,EAAM,SAAS,CAAA;AAAA,MACrC,CAAA,MAAO;AACL,QAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,QAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA,yBAAA,CAA2B,CAAA;AACxD,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,OAAA,EAAS;AAAA,WACV,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,YACV,GAAG,QAAQ,CAAA,6CAAA;AAAA,WACb;AACA,UAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,YACnB,OAAA,EAAS,KAAA;AAAA,YACT,QAAQ,EAAC;AAAA,YACT,UAAA,EAAY,aAAa,QAAQ,CAAA,WAAA;AAAA,WAClC,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAED,IAAA,MAAA,CAAO,MAAA;AAAA,MACL,CAAA,sCAAA,CAAA;AAAA,MACA,OAAO,KAAK,GAAA,KAAQ;AAClB,QAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,QAAA,MAAM,SAAA,GAAY,MAAM,IAAA,CAAK,OAAA,CAAQ,wBAAwB,QAAQ,CAAA;AAErE,QAAA,GAAA,CAAI,IAAA,CAAK;AAAA,UACP,OAAA,EAAS,IAAA;AAAA,UACT,OAAA,EAAS,+BAA+B,QAAQ,CAAA,UAAA,CAAA;AAAA,UAChD;AAAA,SACD,CAAA;AAAA,MACH;AAAA,KACF;AAEA,IAAA,OAAO,MAAA;AAAA,EACT;AACF;;;;"}
+{"version":3,"file":"routes.cjs.js","sources":["../../src/router/routes.ts"],"sourcesContent":["/*\n * Copyright 2022 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  HttpAuthService,\n  LoggerService,\n  PermissionsService,\n} from '@backstage/backend-plugin-api';\nimport { NotAllowedError } from '@backstage/errors';\nimport {\n  catalogIngestionManagePermission,\n  catalogIngestionReadPermission,\n} from '@backstage/plugin-catalog-common/alpha';\nimport { AuthorizeResult } from '@backstage/plugin-permission-common';\nimport express from 'express';\nimport Router from 'express-promise-router';\nimport { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';\n\nexport class IncrementalProviderRouter {\n  private manager: IncrementalIngestionDatabaseManager;\n  private logger: LoggerService;\n\n  constructor(\n    manager: IncrementalIngestionDatabaseManager,\n    logger: LoggerService,\n    private readonly permissions: PermissionsService,\n    private readonly httpAuth: HttpAuthService,\n  ) {\n    this.manager = manager;\n    this.logger = logger;\n  }\n\n  createRouter(): express.Router {\n    const router = Router();\n    router.use(express.json());\n    router.use('/incremental', async (req, _res, next) => {\n      const permission =\n        req.method === 'GET' || req.method === 'HEAD'\n          ? catalogIngestionReadPermission\n          : catalogIngestionManagePermission;\n      const [decision] = await this.permissions.authorize([{ permission }], {\n        credentials: await this.httpAuth.credentials(req),\n      });\n      if (decision.result === AuthorizeResult.DENY) {\n        throw new NotAllowedError('Unauthorized');\n      }\n      next();\n    });\n\n    // Get the overall health of all incremental providers\n    router.get('/incremental/health', async (_, res) => {\n      const records = await this.manager.healthcheck();\n      const providers = records.map(record => record.provider_name);\n      const duplicates = [\n        ...new Set(providers.filter((e, i, a) => a.indexOf(e) !== i)),\n      ];\n\n      if (duplicates.length > 0) {\n        res.json({ healthy: false, duplicateIngestions: duplicates });\n      } else {\n        res.json({ healthy: true });\n      }\n    });\n\n    // Clean up and pause all providers\n    router.post('/incremental/cleanup', async (_, res) => {\n      const result = await this.manager.cleanupProviders();\n      res.json(result);\n    });\n\n    // Get basic status of the provider\n    router.get('/incremental/providers/:provider', async (req, res) => {\n      const { provider } = req.params;\n      const record = await this.manager.getCurrentIngestionRecord(provider);\n      if (record) {\n        res.json({\n          success: true,\n          status: {\n            current_action: record.status,\n            next_action_at: new Date(record.next_action_at),\n          },\n          last_error: record.last_error,\n        });\n      } else {\n        const providers: string[] = await this.manager.listProviders();\n        if (providers.includes(provider)) {\n          res.json({\n            success: true,\n            status: {\n              current_action: 'rest complete, waiting to start',\n            },\n          });\n        } else {\n          this.logger.error(\n            `${provider} - No ingestion record found in the database!`,\n          );\n          res.status(404).json({\n            success: false,\n            status: {},\n            last_error: `Provider '${provider}' not found`,\n          });\n        }\n      }\n    });\n\n    // Trigger the provider's next action\n    router.post(\n      `/incremental/providers/:provider/trigger`,\n      async (req, res) => {\n        const { provider } = req.params;\n        const record = await this.manager.getCurrentIngestionRecord(provider);\n        if (record) {\n          await this.manager.triggerNextProviderAction(provider);\n          res.json({\n            success: true,\n            message: `${provider}: Next action triggered.`,\n          });\n        } else {\n          const providers: string[] = await this.manager.listProviders();\n          if (providers.includes(provider)) {\n            this.logger.debug(`${provider} - Ingestion record found`);\n            res.json({\n              success: true,\n              message: 'Unable to trigger next action (provider is restarting)',\n            });\n          } else {\n            res.status(404).json({\n              success: false,\n              message: `Provider '${provider}' not found`,\n            });\n          }\n        }\n      },\n    );\n\n    // Start a brand-new ingestion cycle for the provider.\n    // (Cancel's the current run if active, or marks it complete if resting)\n    router.post(`/incremental/providers/:provider/start`, async (req, res) => {\n      const { provider } = req.params;\n\n      const record = await this.manager.getCurrentIngestionRecord(provider);\n      if (record) {\n        const ingestionId = record.id;\n        if (record.status === 'resting') {\n          await this.manager.setProviderComplete(ingestionId);\n        } else {\n          await this.manager.setProviderCanceling(ingestionId);\n        }\n        res.json({\n          success: true,\n          message: `${provider}: Next cycle triggered.`,\n        });\n      } else {\n        const providers: string[] = await this.manager.listProviders();\n        if (providers.includes(provider)) {\n          this.logger.debug(`${provider} - Ingestion record found`);\n          res.json({\n            success: true,\n            message: 'Provider is already restarting',\n          });\n        } else {\n          res.status(404).json({\n            success: false,\n            message: `Provider '${provider}' not found`,\n          });\n        }\n      }\n    });\n\n    router.get(`/incremental/providers`, async (_req, res) => {\n      const providers = await this.manager.listProviders();\n\n      res.json({\n        success: true,\n        providers,\n      });\n    });\n\n    // Stop the provider and pause it for 24 hours\n    router.post(`/incremental/providers/:provider/cancel`, async (req, res) => {\n      const { provider } = req.params;\n      const record = await this.manager.getCurrentIngestionRecord(provider);\n      if (record) {\n        const next_action_at = new Date();\n        next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1000);\n        await this.manager.updateByName(provider, {\n          next_action: 'nothing (done)',\n          ingestion_completed_at: new Date(),\n          next_action_at,\n          status: 'resting',\n        });\n        res.json({\n          success: true,\n          message: `${provider}: Current ingestion canceled.`,\n        });\n      } else {\n        const providers: string[] = await this.manager.listProviders();\n        if (providers.includes(provider)) {\n          this.logger.debug(`${provider} - Ingestion record found`);\n          res.json({\n            success: true,\n            message: 'Provider is currently restarting, please wait.',\n          });\n        } else {\n          res.status(404).json({\n            success: false,\n            message: `Provider '${provider}' not found`,\n          });\n        }\n      }\n    });\n\n    // Wipe out all ingestion records for the provider and pause for 24 hours\n    router.delete('/incremental/providers/:provider', async (req, res) => {\n      const { provider } = req.params;\n      const result = await this.manager.purgeAndResetProvider(provider);\n      res.json(result);\n    });\n\n    // Get the ingestion marks for the current cycle\n    router.get(`/incremental/providers/:provider/marks`, async (req, res) => {\n      const { provider } = req.params;\n      const record = await this.manager.getCurrentIngestionRecord(provider);\n      if (record) {\n        const id = record.id;\n        const records = await this.manager.getAllMarks(id);\n        res.json({ success: true, records });\n      } else {\n        const providers: string[] = await this.manager.listProviders();\n        if (providers.includes(provider)) {\n          this.logger.debug(`${provider} - Ingestion record found`);\n          res.json({\n            success: true,\n            message: 'No records yet (provider is restarting)',\n          });\n        } else {\n          this.logger.error(\n            `${provider} - No ingestion record found in the database!`,\n          );\n          res.status(404).json({\n            success: false,\n            status: {},\n            last_error: `Provider '${provider}' not found`,\n          });\n        }\n      }\n    });\n\n    router.delete(\n      `/incremental/providers/:provider/marks`,\n      async (req, res) => {\n        const { provider } = req.params;\n        const deletions = await this.manager.clearFinishedIngestions(provider);\n\n        res.json({\n          success: true,\n          message: `Expired marks for provider '${provider}' removed.`,\n          deletions,\n        });\n      },\n    );\n\n    return router;\n  }\n}\n"],"names":["Router","express","catalogIngestionReadPermission","catalogIngestionManagePermission","AuthorizeResult","NotAllowedError"],"mappings":";;;;;;;;;;;;;AA+BO,MAAM,yBAAA,CAA0B;AAAA,EAIrC,WAAA,CACE,OAAA,EACA,MAAA,EACiB,WAAA,EACA,QAAA,EACjB;AAFiB,IAAA,IAAA,CAAA,WAAA,GAAA,WAAA;AACA,IAAA,IAAA,CAAA,QAAA,GAAA,QAAA;AAEjB,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AACf,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AAAA,EAChB;AAAA,EAXQ,OAAA;AAAA,EACA,MAAA;AAAA,EAYR,YAAA,GAA+B;AAC7B,IAAA,MAAM,SAASA,uBAAA,EAAO;AACtB,IAAA,MAAA,CAAO,GAAA,CAAIC,wBAAA,CAAQ,IAAA,EAAM,CAAA;AACzB,IAAA,MAAA,CAAO,GAAA,CAAI,cAAA,EAAgB,OAAO,GAAA,EAAK,MAAM,IAAA,KAAS;AACpD,MAAA,MAAM,aACJ,GAAA,CAAI,MAAA,KAAW,SAAS,GAAA,CAAI,MAAA,KAAW,SACnCC,oCAAA,GACAC,sCAAA;AACN,MAAA,MAAM,CAAC,QAAQ,CAAA,GAAI,MAAM,IAAA,CAAK,WAAA,CAAY,SAAA,CAAU,CAAC,EAAE,UAAA,EAAY,CAAA,EAAG;AAAA,QACpE,WAAA,EAAa,MAAM,IAAA,CAAK,QAAA,CAAS,YAAY,GAAG;AAAA,OACjD,CAAA;AACD,MAAA,IAAI,QAAA,CAAS,MAAA,KAAWC,sCAAA,CAAgB,IAAA,EAAM;AAC5C,QAAA,MAAM,IAAIC,uBAAgB,cAAc,CAAA;AAAA,MAC1C;AACA,MAAA,IAAA,EAAK;AAAA,IACP,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,GAAA,CAAI,qBAAA,EAAuB,OAAO,CAAA,EAAG,GAAA,KAAQ;AAClD,MAAA,MAAM,OAAA,GAAU,MAAM,IAAA,CAAK,OAAA,CAAQ,WAAA,EAAY;AAC/C,MAAA,MAAM,SAAA,GAAY,OAAA,CAAQ,GAAA,CAAI,CAAA,MAAA,KAAU,OAAO,aAAa,CAAA;AAC5D,MAAA,MAAM,UAAA,GAAa;AAAA,QACjB,GAAG,IAAI,GAAA,CAAI,SAAA,CAAU,OAAO,CAAC,CAAA,EAAG,CAAA,EAAG,CAAA,KAAM,CAAA,CAAE,OAAA,CAAQ,CAAC,CAAA,KAAM,CAAC,CAAC;AAAA,OAC9D;AAEA,MAAA,IAAI,UAAA,CAAW,SAAS,CAAA,EAAG;AACzB,QAAA,GAAA,CAAI,KAAK,EAAE,OAAA,EAAS,KAAA,EAAO,mBAAA,EAAqB,YAAY,CAAA;AAAA,MAC9D,CAAA,MAAO;AACL,QAAA,GAAA,CAAI,IAAA,CAAK,EAAE,OAAA,EAAS,IAAA,EAAM,CAAA;AAAA,MAC5B;AAAA,IACF,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,IAAA,CAAK,sBAAA,EAAwB,OAAO,CAAA,EAAG,GAAA,KAAQ;AACpD,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,gBAAA,EAAiB;AACnD,MAAA,GAAA,CAAI,KAAK,MAAM,CAAA;AAAA,IACjB,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,GAAA,CAAI,kCAAA,EAAoC,OAAO,GAAA,EAAK,GAAA,KAAQ;AACjE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,GAAA,CAAI,IAAA,CAAK;AAAA,UACP,OAAA,EAAS,IAAA;AAAA,UACT,MAAA,EAAQ;AAAA,YACN,gBAAgB,MAAA,CAAO,MAAA;AAAA,YACvB,cAAA,EAAgB,IAAI,IAAA,CAAK,MAAA,CAAO,cAAc;AAAA,WAChD;AAAA,UACA,YAAY,MAAA,CAAO;AAAA,SACpB,CAAA;AAAA,MACH,CAAA,MAAO;AACL,QAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,QAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,MAAA,EAAQ;AAAA,cACN,cAAA,EAAgB;AAAA;AAClB,WACD,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,YACV,GAAG,QAAQ,CAAA,6CAAA;AAAA,WACb;AACA,UAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,YACnB,OAAA,EAAS,KAAA;AAAA,YACT,QAAQ,EAAC;AAAA,YACT,UAAA,EAAY,aAAa,QAAQ,CAAA,WAAA;AAAA,WAClC,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,CAAA,wCAAA,CAAA;AAAA,MACA,OAAO,KAAK,GAAA,KAAQ;AAClB,QAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,QAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,QAAA,IAAI,MAAA,EAAQ;AACV,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,yBAAA,CAA0B,QAAQ,CAAA;AACrD,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,OAAA,EAAS,GAAG,QAAQ,CAAA,wBAAA;AAAA,WACrB,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,UAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,YAAA,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA,yBAAA,CAA2B,CAAA;AACxD,YAAA,GAAA,CAAI,IAAA,CAAK;AAAA,cACP,OAAA,EAAS,IAAA;AAAA,cACT,OAAA,EAAS;AAAA,aACV,CAAA;AAAA,UACH,CAAA,MAAO;AACL,YAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,cACnB,OAAA,EAAS,KAAA;AAAA,cACT,OAAA,EAAS,aAAa,QAAQ,CAAA,WAAA;AAAA,aAC/B,CAAA;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,KACF;AAIA,IAAA,MAAA,CAAO,IAAA,CAAK,CAAA,sCAAA,CAAA,EAA0C,OAAO,GAAA,EAAK,GAAA,KAAQ;AACxE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AAEzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,MAAM,cAAc,MAAA,CAAO,EAAA;AAC3B,QAAA,IAAI,MAAA,CAAO,WAAW,SAAA,EAAW;AAC/B,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,mBAAA,CAAoB,WAAW,CAAA;AAAA,QACpD,CAAA,MAAO;AACL,UAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,oBAAA,CAAqB,WAAW,CAAA;AAAA,QACrD;AACA,QAAA,GAAA,CAAI,IAAA,CAAK;AAAA,UACP,OAAA,EAAS,IAAA;AAAA,UACT,OAAA,EAAS,GAAG,QAAQ,CAAA,uBAAA;AAAA,SACrB,CAAA;AAAA,MACH,CAAA,MAAO;AACL,QAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,QAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA,yBAAA,CAA2B,CAAA;AACxD,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,OAAA,EAAS;AAAA,WACV,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,YACnB,OAAA,EAAS,KAAA;AAAA,YACT,OAAA,EAAS,aAAa,QAAQ,CAAA,WAAA;AAAA,WAC/B,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAED,IAAA,MAAA,CAAO,GAAA,CAAI,CAAA,sBAAA,CAAA,EAA0B,OAAO,IAAA,EAAM,GAAA,KAAQ;AACxD,MAAA,MAAM,SAAA,GAAY,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAEnD,MAAA,GAAA,CAAI,IAAA,CAAK;AAAA,QACP,OAAA,EAAS,IAAA;AAAA,QACT;AAAA,OACD,CAAA;AAAA,IACH,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,IAAA,CAAK,CAAA,uCAAA,CAAA,EAA2C,OAAO,GAAA,EAAK,GAAA,KAAQ;AACzE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,MAAM,cAAA,uBAAqB,IAAA,EAAK;AAChC,QAAA,cAAA,CAAe,QAAQ,cAAA,CAAe,OAAA,KAAY,EAAA,GAAK,EAAA,GAAK,KAAK,GAAI,CAAA;AACrE,QAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,YAAA,CAAa,QAAA,EAAU;AAAA,UACxC,WAAA,EAAa,gBAAA;AAAA,UACb,sBAAA,sBAA4B,IAAA,EAAK;AAAA,UACjC,cAAA;AAAA,UACA,MAAA,EAAQ;AAAA,SACT,CAAA;AACD,QAAA,GAAA,CAAI,IAAA,CAAK;AAAA,UACP,OAAA,EAAS,IAAA;AAAA,UACT,OAAA,EAAS,GAAG,QAAQ,CAAA,6BAAA;AAAA,SACrB,CAAA;AAAA,MACH,CAAA,MAAO;AACL,QAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,QAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA,yBAAA,CAA2B,CAAA;AACxD,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,OAAA,EAAS;AAAA,WACV,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,YACnB,OAAA,EAAS,KAAA;AAAA,YACT,OAAA,EAAS,aAAa,QAAQ,CAAA,WAAA;AAAA,WAC/B,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,MAAA,CAAO,kCAAA,EAAoC,OAAO,GAAA,EAAK,GAAA,KAAQ;AACpE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,sBAAsB,QAAQ,CAAA;AAChE,MAAA,GAAA,CAAI,KAAK,MAAM,CAAA;AAAA,IACjB,CAAC,CAAA;AAGD,IAAA,MAAA,CAAO,GAAA,CAAI,CAAA,sCAAA,CAAA,EAA0C,OAAO,GAAA,EAAK,GAAA,KAAQ;AACvE,MAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,CAAQ,0BAA0B,QAAQ,CAAA;AACpE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,MAAM,KAAK,MAAA,CAAO,EAAA;AAClB,QAAA,MAAM,OAAA,GAAU,MAAM,IAAA,CAAK,OAAA,CAAQ,YAAY,EAAE,CAAA;AACjD,QAAA,GAAA,CAAI,IAAA,CAAK,EAAE,OAAA,EAAS,IAAA,EAAM,SAAS,CAAA;AAAA,MACrC,CAAA,MAAO;AACL,QAAA,MAAM,SAAA,GAAsB,MAAM,IAAA,CAAK,OAAA,CAAQ,aAAA,EAAc;AAC7D,QAAA,IAAI,SAAA,CAAU,QAAA,CAAS,QAAQ,CAAA,EAAG;AAChC,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,QAAQ,CAAA,yBAAA,CAA2B,CAAA;AACxD,UAAA,GAAA,CAAI,IAAA,CAAK;AAAA,YACP,OAAA,EAAS,IAAA;AAAA,YACT,OAAA,EAAS;AAAA,WACV,CAAA;AAAA,QACH,CAAA,MAAO;AACL,UAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,YACV,GAAG,QAAQ,CAAA,6CAAA;AAAA,WACb;AACA,UAAA,GAAA,CAAI,MAAA,CAAO,GAAG,CAAA,CAAE,IAAA,CAAK;AAAA,YACnB,OAAA,EAAS,KAAA;AAAA,YACT,QAAQ,EAAC;AAAA,YACT,UAAA,EAAY,aAAa,QAAQ,CAAA,WAAA;AAAA,WAClC,CAAA;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC,CAAA;AAED,IAAA,MAAA,CAAO,MAAA;AAAA,MACL,CAAA,sCAAA,CAAA;AAAA,MACA,OAAO,KAAK,GAAA,KAAQ;AAClB,QAAA,MAAM,EAAE,QAAA,EAAS,GAAI,GAAA,CAAI,MAAA;AACzB,QAAA,MAAM,SAAA,GAAY,MAAM,IAAA,CAAK,OAAA,CAAQ,wBAAwB,QAAQ,CAAA;AAErE,QAAA,GAAA,CAAI,IAAA,CAAK;AAAA,UACP,OAAA,EAAS,IAAA;AAAA,UACT,OAAA,EAAS,+BAA+B,QAAQ,CAAA,UAAA,CAAA;AAAA,UAChD;AAAA,SACD,CAAA;AAAA,MACH;AAAA,KACF;AAEA,IAAA,OAAO,MAAA;AAAA,EACT;AACF;;;;"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend-module-incremental-ingestion",
-  "version": "0.7.15",
+  "version": "0.8.0-next.1",
   "description": "An entity provider for streaming large asset sources into the catalog",
   "backstage": {
     "role": "backend-plugin-module",
@@ -55,23 +55,27 @@
     "test": "backstage-cli package test"
   },
   "dependencies": {
-    "@backstage/backend-defaults": "^0.17.7",
-    "@backstage/backend-plugin-api": "^1.10.0",
-    "@backstage/catalog-model": "^1.10.0",
-    "@backstage/config": "^1.3.8",
-    "@backstage/errors": "^1.3.1",
-    "@backstage/plugin-catalog-backend": "^3.9.0",
-    "@backstage/plugin-catalog-node": "^2.2.4",
-    "@backstage/plugin-events-node": "^0.4.25",
-    "@backstage/types": "^1.2.2",
+    "@backstage/backend-defaults": "0.17.9-next.1",
+    "@backstage/backend-plugin-api": "1.10.1-next.1",
+    "@backstage/catalog-model": "1.10.0",
+    "@backstage/config": "1.3.9-next.0",
+    "@backstage/errors": "1.3.1",
+    "@backstage/plugin-catalog-backend": "4.0.0-next.1",
+    "@backstage/plugin-catalog-common": "1.2.0-next.0",
+    "@backstage/plugin-catalog-node": "2.2.5-next.1",
+    "@backstage/plugin-events-node": "0.4.26-next.0",
+    "@backstage/plugin-permission-common": "0.9.11-next.0",
+    "@backstage/types": "1.2.2",
     "express": "^4.22.0",
     "express-promise-router": "^4.1.0",
     "knex": "^3.0.0",
     "luxon": "^3.0.0"
   },
   "devDependencies": {
-    "@backstage/backend-test-utils": "^1.11.6",
-    "@backstage/cli": "^0.36.5",
-    "@types/express": "^4.17.6"
+    "@backstage/backend-test-utils": "1.11.7-next.1",
+    "@backstage/cli": "0.36.6-next.1",
+    "@types/express": "^4.17.6",
+    "@types/supertest": "^2.0.8",
+    "supertest": "^7.0.0"
   }
 }