@backstage/plugin-catalog-backend-module-incremental-ingestion 0.5.2 → 0.5.3-next.1

package/CHANGELOG.md

@@ -1,22 +1,36 @@
 # @backstage/plugin-catalog-backend-module-incremental-ingestion
 
-## 0.5.2
+## 0.5.3-next.1
 
 ### Patch Changes
 
 - Updated dependencies
-  - @backstage/backend-common@0.24.1
-  - @backstage/plugin-catalog-backend@1.25.2
-  - @backstage/plugin-events-node@0.3.10
-  - @backstage/backend-plugin-api@0.8.1
-  - @backstage/plugin-catalog-node@1.12.6
+  - @backstage/backend-common@0.25.0-next.1
+  - @backstage/plugin-catalog-backend@1.25.3-next.1
+  - @backstage/backend-plugin-api@0.9.0-next.1
+  - @backstage/catalog-model@1.6.0
+  - @backstage/config@1.2.0
+  - @backstage/errors@1.2.4
+  - @backstage/plugin-catalog-node@1.12.7-next.1
+  - @backstage/plugin-events-node@0.4.0-next.1
+  - @backstage/plugin-permission-common@0.8.1
 
-## 0.5.1
+## 0.5.3-next.0
 
 ### Patch Changes
 
+- d425fc4: Modules, plugins, and services are now `BackendFeature`, not a function that returns a feature.
+- 4b28e39: Updated the README to include documentation for the new backend support
 - Updated dependencies
-  - @backstage/plugin-catalog-backend@1.25.1
+  - @backstage/backend-plugin-api@0.9.0-next.0
+  - @backstage/backend-common@0.25.0-next.0
+  - @backstage/plugin-catalog-backend@1.25.3-next.0
+  - @backstage/plugin-events-node@0.4.0-next.0
+  - @backstage/plugin-catalog-node@1.12.7-next.0
+  - @backstage/catalog-model@1.6.0
+  - @backstage/config@1.2.0
+  - @backstage/errors@1.2.4
+  - @backstage/plugin-permission-common@0.8.1
 
 ## 0.5.0
 

package/README.md

@@ -42,57 +42,25 @@ The Incremental Entity Provider backend is designed for data sources that provid
 ## Installation
 
 1. Install `@backstage/plugin-catalog-backend-module-incremental-ingestion` with `yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-incremental-ingestion` from the Backstage root directory.
-2. In your catalog.ts, import `IncrementalCatalogBuilder` from `@backstage/plugin-catalog-backend-module-incremental-ingestion` and instantiate it with `await IncrementalCatalogBuilder.create(env, builder)`. You have to pass `builder` into `IncrementalCatalogBuilder.create` function because `IncrementalCatalogBuilder` will convert an `IncrementalEntityProvider` into an `EntityProvider` and call `builder.addEntityProvider`.
 
-```ts
-const builder = CatalogBuilder.create(env);
-// incremental builder receives builder because it'll register
-// incremental entity providers with the builder
-const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder);
-```
+2. Add the following code to the `packages/backend/src/index.ts` file:
 
-3. After building the regular `CatalogBuilder`, build the incremental builder:
+```diff
++ import { catalogModuleCustomIncrementalIngestionProvider } from './extensions/catalogCustomIncrementalIngestion';
 
-```ts
-// Must be run first to ensure CatalogBuilder database migrations run before Incremental Entity Provider database migrations
-const { processingEngine, router } = await builder.build();
 
-// Returns an optional - but highly recommended - set of administrative routes
-const { incrementalAdminRouter } = await incrementBuilder.build();
-```
+const backend = createBackend();
 
-The final result should look something like this,
++ backend.add(
++   import(
++     '@backstage/plugin-catalog-backend-module-incremental-ingestion/alpha'
++   ),
++ );
 
-```ts
-import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
-import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
-import { IncrementalCatalogBuilder } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
-import { Router } from 'express';
-import { Duration } from 'luxon';
-import { PluginEnvironment } from '../types';
-
-export default async function createPlugin(
-  env: PluginEnvironment,
-): Promise<Router> {
-  const builder = CatalogBuilder.create(env);
-  // incremental builder receives builder because it'll register
-  // incremental entity providers with the builder
-  const incrementalBuilder = await IncrementalCatalogBuilder.create(
-    env,
-    builder,
-  );
-
-  builder.addProcessor(new ScaffolderEntitiesProcessor());
-
-  const { processingEngine, router } = await builder.build();
-  const { incrementalAdminRouter } = await incrementalBuilder.build();
-
-  router.use(incrementalAdminRouter);
-
-  await processingEngine.start();
-
-  return router;
-}
+// We have created this in section **Adding an Incremental Entity Provider to the catalog**
++ backend.add(catalogModuleCustomIncrementalIngestionProvider);
+
+backend.start();
 ```
 
 ## Administrative Routes
@@ -116,7 +84,7 @@ In all cases, `:provider` is the name of the incremental entity provider.
 
 ## Writing an Incremental Entity Provider
 
-To create an Incremental Entity Provider, you need to know how to retrieve a single page of the data that you wish to ingest into the Backstage catalog. If the API has pagination and you know how to make a paginated request to that API, you'll be able to implement an Incremental Entity Provider for this API. For more information about compatibility, check out the <a href="#requirements">requirements</a> section of this page.
+To create an Incremental Entity Provider, you need to know how to retrieve a single page of the data that you wish to ingest into the Backstage catalog. If the API has pagination and you know how to make a paginated request to that API, you'll be able to implement an Incremental Entity Provider for this API. For more information about compatibility, check out the [requirements](#requirements) section of this page.
 
 Here is the type definition for an Incremental Entity Provider.
 
@@ -309,45 +277,73 @@ Now that you have your new Incremental Entity Provider, we can connect it to the
 
 ## Adding an Incremental Entity Provider to the catalog
 
-We'll assume you followed the <a href="#installation">Installation</a> instructions. After you create your `incrementalBuilder`, you can instantiate your Entity Provider and pass it to the `addIncrementalEntityProvider` method.
+We'll assume you followed the [Installation](#installation) instructions. Now create a module inside `packages/backend/src/extensions/catalogCustomIncrementalIngestion.ts`.
 
 ```ts
-const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder);
-
-// Assuming the token for the API comes from config
-const token = config.getString('myApiClient.token');
-
-const myEntityProvider = new MyIncrementalEntityProvider(token);
-
-incrementalBuilder.addIncrementalEntityProvider(myEntityProvider, {
-  // How long should it attempt to read pages from the API in a
-  // single burst? Keep this short. The Incremental Entity Provider
-  // will attempt to read as many pages as it can in this time
-  burstLength: { seconds: 3 },
-
-  // How long should it wait between bursts?
-  burstInterval: { seconds: 3 },
-
-  // How long should it rest before re-ingesting again?
-  restLength: { day: 1 },
-
-  // Optional back-off configuration - how long should it wait to retry
-  // in the event of an error?
-  backoff: [{ seconds: 5 }, { seconds: 30 }, { minutes: 10 }, { hours: 3 }],
-
-  // Optional. Use this to prevent removal of entities above a given
-  // percentage. This can be helpful if a data source is flaky and
-  // sometimes returns a successful status, but fewer than expected
-  // assets to add or maintain in the catalog.
-  rejectRemovalsAbovePercentage: 5,
-
-  // Optional. Similar to rejectRemovalsAbovePercentage, except it
-  // applies to complete, 100% failure of a data source. If true,
-  // a data source that returns a successful status but does not
-  // provide any assets to turn into entities will have its empty
-  // data set rejected.
-  rejectEmptySourceCollections: true,
-});
+import {
+  coreServices,
+  createBackendModule,
+} from '@backstage/backend-plugin-api';
+import { incrementalIngestionProvidersExtensionPoint } from '@backstage/plugin-catalog-backend-module-incremental-ingestion/alpha';
+
+export const catalogModuleCustomIncrementalIngestionProvider =
+  createBackendModule({
+    pluginId: 'catalog',
+    moduleId: 'custom-incremental-ingestion-provider',
+    register(env) {
+      env.registerInit({
+        deps: {
+          incrementalBuilder: incrementalIngestionProvidersExtensionPoint,
+          config: coreServices.rootConfig,
+        },
+        async init({ incrementalBuilder, config }) {
+          // Assuming the token for the API comes from config
+          const token = config.getString('myApiClient.token');
+          const myEntityProvider = new MyIncrementalEntityProvider(token);
+
+          const options = {
+            // How long should it attempt to read pages from the API in a
+            // single burst? Keep this short. The Incremental Entity Provider
+            // will attempt to read as many pages as it can in this time
+            burstLength: { seconds: 3 },
+
+            // How long should it wait between bursts?
+            burstInterval: { seconds: 3 },
+
+            // How long should it rest before re-ingesting again?
+            restLength: { day: 1 },
+
+            // Optional back-off configuration - how long should it wait to retry
+            // in the event of an error?
+            backoff: [
+              { seconds: 5 },
+              { seconds: 30 },
+              { minutes: 10 },
+              { hours: 3 },
+            ],
+
+            // Optional. Use this to prevent removal of entities above a given
+            // percentage. This can be helpful if a data source is flaky and
+            // sometimes returns a successful status, but fewer than expected
+            // assets to add or maintain in the catalog.
+            rejectRemovalsAbovePercentage: 5,
+
+            // Optional. Similar to rejectRemovalsAbovePercentage, except it
+            // applies to complete, 100% failure of a data source. If true,
+            // a data source that returns a successful status but does not
+            // provide any assets to turn into entities will have its empty
+            // data set rejected.
+            rejectEmptySourceCollections: true,
+          };
+
+          incrementalBuilder.addProvider({
+            provider: myEntityProvider,
+            options,
+          });
+        },
+      });
+    },
+  });
 ```
 
 That's it!!!

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend-module-incremental-ingestion__alpha",
-  "version": "0.5.2",
+  "version": "0.5.3-next.1",
   "main": "../dist/alpha.cjs.js",
   "types": "../dist/alpha.d.ts"
 }

package/dist/alpha.d.ts

@@ -52,6 +52,6 @@ declare const incrementalIngestionProvidersExtensionPoint: _backstage_backend_pl
  *
  * @alpha
  */
-declare const catalogModuleIncrementalIngestionEntityProvider: _backstage_backend_plugin_api.BackendFeatureCompat;
+declare const catalogModuleIncrementalIngestionEntityProvider: _backstage_backend_plugin_api.BackendFeature;
 
 export { type IncrementalIngestionProviderExtensionPoint, catalogModuleIncrementalIngestionEntityProvider as default, incrementalIngestionProvidersExtensionPoint };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend-module-incremental-ingestion",
-  "version": "0.5.2",
+  "version": "0.5.3-next.1",
   "description": "An entity provider for streaming large asset sources into the catalog",
   "backstage": {
     "role": "backend-plugin-module",
@@ -50,14 +50,14 @@
     "test": "backstage-cli package test"
   },
   "dependencies": {
-    "@backstage/backend-common": "^0.24.1",
-    "@backstage/backend-plugin-api": "^0.8.1",
+    "@backstage/backend-common": "^0.25.0-next.1",
+    "@backstage/backend-plugin-api": "^0.9.0-next.1",
     "@backstage/catalog-model": "^1.6.0",
     "@backstage/config": "^1.2.0",
     "@backstage/errors": "^1.2.4",
-    "@backstage/plugin-catalog-backend": "^1.25.2",
-    "@backstage/plugin-catalog-node": "^1.12.6",
-    "@backstage/plugin-events-node": "^0.3.10",
+    "@backstage/plugin-catalog-backend": "^1.25.3-next.1",
+    "@backstage/plugin-catalog-node": "^1.12.7-next.1",
+    "@backstage/plugin-events-node": "^0.4.0-next.1",
     "@backstage/plugin-permission-common": "^0.8.1",
     "@types/express": "^4.17.6",
     "@types/luxon": "^3.0.0",
@@ -68,8 +68,8 @@
     "uuid": "^9.0.0"
   },
   "devDependencies": {
-    "@backstage/backend-defaults": "^0.4.4",
-    "@backstage/backend-test-utils": "^0.5.1",
-    "@backstage/cli": "^0.27.0"
+    "@backstage/backend-defaults": "^0.5.0-next.1",
+    "@backstage/backend-test-utils": "^0.6.0-next.1",
+    "@backstage/cli": "^0.27.1-next.1"
   }
 }