@backstage/plugin-catalog-backend-module-unprocessed 0.0.0-nightly-20230531023430

package/CHANGELOG.md

@@ -0,0 +1,27 @@
+# @backstage/plugin-catalog-backend-module-unprocessed
+
+## 0.0.0-nightly-20230531023430
+
+### Minor Changes
+
+- d44fcd9829c2: Added a new plugin to expose entities which are unprocessed or have errors processing
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/backend-plugin-api@0.0.0-nightly-20230531023430
+  - @backstage/catalog-model@0.0.0-nightly-20230531023430
+  - @backstage/plugin-auth-node@0.0.0-nightly-20230531023430
+
+## 0.1.0-next.0
+
+### Minor Changes
+
+- d44fcd9829c2: Added a new plugin to expose entities which are unprocessed or have errors processing
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/backend-plugin-api@0.5.3-next.1
+  - @backstage/catalog-model@1.4.0-next.0
+  - @backstage/plugin-auth-node@0.2.15-next.1

package/README.md

@@ -0,0 +1,38 @@
+# @backstage/plugin-catalog-backend-module-unprocessed-node
+
+This catalog-backend module adds support for viewing unprocessed entities. An unprocessed entity is one that doesn't show up in the catalog.
+
+A distinction is made between `failed` and `pending` entities.
+
+A `failed` entity has validation error that breaks processing.
+
+A `pending` entity has not been processed yet.
+
+## Installation
+
+### backend
+
+In `packages/backend/src/plugins/catalog.ts` import the module and initialize it after invoking `CatalogBuilder.build()`:
+
+```ts
+import { UnprocessedEntitesModule } from '@backstage/plugin-catalog-backend-module-unprocessed';
+
+//...
+
+const unprocessed = new UnprocessedEntitesModule(
+  await env.database.getClient(),
+  router,
+  env.logger,
+);
+unprocessed.registerRoutes();
+```
+
+### backend-next
+
+In `packages/backend-next/src/index.ts` add the module:
+
+```ts
+backend.add(catalogModuleUnprocessedEntities());
+```
+
+_This plugin was created through the Backstage CLI_

package/dist/index.cjs.js

@@ -0,0 +1,122 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+var backendPluginApi = require('@backstage/backend-plugin-api');
+var Router = require('express-promise-router');
+var pluginAuthNode = require('@backstage/plugin-auth-node');
+
+function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
+
+var Router__default = /*#__PURE__*/_interopDefaultLegacy(Router);
+
+class UnprocessedEntitesModule {
+  constructor(database, router) {
+    this.database = database;
+    this.router = router;
+    this.moduleRouter = Router__default["default"]();
+    this.router.use(this.moduleRouter);
+  }
+  async unprocessed(request) {
+    if (request.reason === "pending") {
+      return {
+        type: "pending",
+        entities: await this.pending(request.owner)
+      };
+    }
+    return {
+      type: "failed",
+      entities: await this.failed(request.owner)
+    };
+  }
+  hydrateRefreshState(r) {
+    return {
+      ...r,
+      unprocessed_entity: JSON.parse(r.unprocessed_entity),
+      ...r.processed_entity && {
+        processed_entity: JSON.parse(r.processed_entity)
+      },
+      ...r.errors && { errors: JSON.parse(r.errors) },
+      ...r.cache && { cache: JSON.parse(r.cache) }
+    };
+  }
+  async pending(owner) {
+    const res = (await this.database("refresh_state.*").from("refresh_state").leftJoin(
+      "final_entities",
+      "final_entities.entity_id",
+      "refresh_state.entity_id"
+    ).whereNull("final_entities.entity_id")).map(this.hydrateRefreshState);
+    if (owner) {
+      return res.filter((r) => {
+        var _a;
+        return ((_a = r.unprocessed_entity.spec) == null ? void 0 : _a.owner) === owner;
+      });
+    }
+    return res;
+  }
+  async failed(owner) {
+    const res = (await this.database("refresh_state.*").from("refresh_state").rightJoin(
+      "final_entities",
+      "final_entities.entity_id",
+      "refresh_state.entity_id"
+    ).whereNull("final_entities.final_entity")).map(this.hydrateRefreshState);
+    if (owner) {
+      return res.filter((r) => {
+        var _a;
+        return ((_a = r.unprocessed_entity.spec) == null ? void 0 : _a.owner) === owner;
+      });
+    }
+    return res;
+  }
+  registerRoutes() {
+    this.moduleRouter.get("/entities/unprocessed/failed", async (req, res) => {
+      return res.json(
+        await this.unprocessed({
+          reason: "failed",
+          owner: req.query.owner,
+          authorizationToken: pluginAuthNode.getBearerTokenFromAuthorizationHeader(
+            req.header("authorization")
+          )
+        })
+      );
+    }).get("/entities/unprocessed/pending", async (req, res) => {
+      return res.json(
+        await this.unprocessed({
+          reason: "pending",
+          owner: req.query.owner,
+          authorizationToken: pluginAuthNode.getBearerTokenFromAuthorizationHeader(
+            req.header("authorization")
+          )
+        })
+      );
+    });
+  }
+}
+
+const catalogModuleUnprocessedEntities = backendPluginApi.createBackendModule({
+  pluginId: "catalog",
+  moduleId: "catalogModuleUnprocessedEntities",
+  register(env) {
+    env.registerInit({
+      deps: {
+        database: backendPluginApi.coreServices.database,
+        router: backendPluginApi.coreServices.httpRouter,
+        logger: backendPluginApi.coreServices.logger
+      },
+      async init({ database, router, logger }) {
+        const module = new UnprocessedEntitesModule(
+          await database.getClient(),
+          router
+        );
+        module.registerRoutes();
+        logger.info(
+          "registered additional routes for catalogModuleUnprocessedEntities"
+        );
+      }
+    });
+  }
+});
+
+exports.UnprocessedEntitesModule = UnprocessedEntitesModule;
+exports.catalogModuleUnprocessedEntities = catalogModuleUnprocessedEntities;
+//# sourceMappingURL=index.cjs.js.map

package/dist/index.cjs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.cjs.js","sources":["../src/UnprocessedEntitiesModule.ts","../src/module.ts"],"sourcesContent":["/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  HydratedRefreshState,\n  RefreshState,\n  UnprocessedEntitiesRequest,\n  UnprocessedEntitiesResponse,\n} from './types';\nimport { Knex } from 'knex';\nimport { HttpRouterService } from '@backstage/backend-plugin-api';\nimport Router from 'express-promise-router';\nimport { getBearerTokenFromAuthorizationHeader } from '@backstage/plugin-auth-node';\n\n/**\n * Module providing Unprocessed Entities API endpoints\n *\n * @public\n */\nexport class UnprocessedEntitesModule {\n  private readonly moduleRouter;\n\n  constructor(\n    private readonly database: Knex,\n    private readonly router: HttpRouterService,\n  ) {\n    this.moduleRouter = Router();\n    this.router.use(this.moduleRouter);\n  }\n\n  private async unprocessed(\n    request: UnprocessedEntitiesRequest,\n  ): Promise<UnprocessedEntitiesResponse> {\n    if (request.reason === 'pending') {\n      return {\n        type: 'pending',\n        entities: await this.pending(request.owner),\n      };\n    }\n    return {\n      type: 'failed',\n      entities: await this.failed(request.owner),\n    };\n  }\n\n  private hydrateRefreshState(r: RefreshState): HydratedRefreshState {\n    return {\n      ...r,\n      unprocessed_entity: JSON.parse(r.unprocessed_entity),\n      ...(r.processed_entity && {\n        processed_entity: JSON.parse(r.processed_entity),\n      }),\n      ...(r.errors && { errors: JSON.parse(r.errors) }),\n      ...(r.cache && { cache: JSON.parse(r.cache) }),\n    };\n  }\n\n  private async pending(owner?: string): Promise<HydratedRefreshState[]> {\n    const res = (\n      await this.database('refresh_state.*')\n        .from('refresh_state')\n        .leftJoin(\n          'final_entities',\n          'final_entities.entity_id',\n          'refresh_state.entity_id',\n        )\n        .whereNull('final_entities.entity_id')\n    ).map(this.hydrateRefreshState);\n    if (owner) {\n      return res.filter(r => r.unprocessed_entity.spec?.owner === owner);\n    }\n\n    return res;\n  }\n\n  private async failed(owner?: string): Promise<HydratedRefreshState[]> {\n    const res = (\n      await this.database('refresh_state.*')\n        .from('refresh_state')\n        .rightJoin(\n          'final_entities',\n          'final_entities.entity_id',\n          'refresh_state.entity_id',\n        )\n        .whereNull('final_entities.final_entity')\n    ).map(this.hydrateRefreshState);\n    if (owner) {\n      return res.filter(r => r.unprocessed_entity.spec?.owner === owner);\n    }\n\n    return res;\n  }\n\n  registerRoutes() {\n    this.moduleRouter\n      .get('/entities/unprocessed/failed', async (req, res) => {\n        return res.json(\n          await this.unprocessed({\n            reason: 'failed',\n            owner: req.query.owner as string,\n            authorizationToken: getBearerTokenFromAuthorizationHeader(\n              req.header('authorization'),\n            ),\n          }),\n        );\n      })\n      .get('/entities/unprocessed/pending', async (req, res) => {\n        return res.json(\n          await this.unprocessed({\n            reason: 'pending',\n            owner: req.query.owner as string,\n            authorizationToken: getBearerTokenFromAuthorizationHeader(\n              req.header('authorization'),\n            ),\n          }),\n        );\n      });\n  }\n}\n","/*\n * Copyright 2023 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  coreServices,\n  createBackendModule,\n} from '@backstage/backend-plugin-api';\nimport { UnprocessedEntitesModule } from './UnprocessedEntitiesModule';\n\n/**\n * Catalog Module for Unprocessed Entities\n *\n * @public\n */\nexport const catalogModuleUnprocessedEntities = createBackendModule({\n  pluginId: 'catalog',\n  moduleId: 'catalogModuleUnprocessedEntities',\n  register(env) {\n    env.registerInit({\n      deps: {\n        database: coreServices.database,\n        router: coreServices.httpRouter,\n        logger: coreServices.logger,\n      },\n      async init({ database, router, logger }) {\n        const module = new UnprocessedEntitesModule(\n          await database.getClient(),\n          router,\n        );\n\n        module.registerRoutes();\n        logger.info(\n          'registered additional routes for catalogModuleUnprocessedEntities',\n        );\n      },\n    });\n  },\n});\n"],"names":["Router","getBearerTokenFromAuthorizationHeader","createBackendModule","coreServices"],"mappings":";;;;;;;;;;;;AAgCO,MAAM,wBAAyB,CAAA;AAAA,EAGpC,WAAA,CACmB,UACA,MACjB,EAAA;AAFiB,IAAA,IAAA,CAAA,QAAA,GAAA,QAAA,CAAA;AACA,IAAA,IAAA,CAAA,MAAA,GAAA,MAAA,CAAA;AAEjB,IAAA,IAAA,CAAK,eAAeA,0BAAO,EAAA,CAAA;AAC3B,IAAK,IAAA,CAAA,MAAA,CAAO,GAAI,CAAA,IAAA,CAAK,YAAY,CAAA,CAAA;AAAA,GACnC;AAAA,EAEA,MAAc,YACZ,OACsC,EAAA;AACtC,IAAI,IAAA,OAAA,CAAQ,WAAW,SAAW,EAAA;AAChC,MAAO,OAAA;AAAA,QACL,IAAM,EAAA,SAAA;AAAA,QACN,QAAU,EAAA,MAAM,IAAK,CAAA,OAAA,CAAQ,QAAQ,KAAK,CAAA;AAAA,OAC5C,CAAA;AAAA,KACF;AACA,IAAO,OAAA;AAAA,MACL,IAAM,EAAA,QAAA;AAAA,MACN,QAAU,EAAA,MAAM,IAAK,CAAA,MAAA,CAAO,QAAQ,KAAK,CAAA;AAAA,KAC3C,CAAA;AAAA,GACF;AAAA,EAEQ,oBAAoB,CAAuC,EAAA;AACjE,IAAO,OAAA;AAAA,MACL,GAAG,CAAA;AAAA,MACH,kBAAoB,EAAA,IAAA,CAAK,KAAM,CAAA,CAAA,CAAE,kBAAkB,CAAA;AAAA,MACnD,GAAI,EAAE,gBAAoB,IAAA;AAAA,QACxB,gBAAkB,EAAA,IAAA,CAAK,KAAM,CAAA,CAAA,CAAE,gBAAgB,CAAA;AAAA,OACjD;AAAA,MACA,GAAI,EAAE,MAAU,IAAA,EAAE,QAAQ,IAAK,CAAA,KAAA,CAAM,CAAE,CAAA,MAAM,CAAE,EAAA;AAAA,MAC/C,GAAI,EAAE,KAAS,IAAA,EAAE,OAAO,IAAK,CAAA,KAAA,CAAM,CAAE,CAAA,KAAK,CAAE,EAAA;AAAA,KAC9C,CAAA;AAAA,GACF;AAAA,EAEA,MAAc,QAAQ,KAAiD,EAAA;AACrE,IAAM,MAAA,GAAA,GAAA,CACJ,MAAM,IAAK,CAAA,QAAA,CAAS,iBAAiB,CAClC,CAAA,IAAA,CAAK,eAAe,CACpB,CAAA,QAAA;AAAA,MACC,gBAAA;AAAA,MACA,0BAAA;AAAA,MACA,yBAAA;AAAA,MAED,SAAU,CAAA,0BAA0B,CACvC,EAAA,GAAA,CAAI,KAAK,mBAAmB,CAAA,CAAA;AAC9B,IAAA,IAAI,KAAO,EAAA;AACT,MAAO,OAAA,GAAA,CAAI,OAAO,CAAE,CAAA,KAAA;AAlF1B,QAAA,IAAA,EAAA,CAAA;AAkF6B,QAAE,OAAA,CAAA,CAAA,EAAA,GAAA,CAAA,CAAA,kBAAA,CAAmB,IAArB,KAAA,IAAA,GAAA,KAAA,CAAA,GAAA,EAAA,CAA2B,KAAU,MAAA,KAAA,CAAA;AAAA,OAAK,CAAA,CAAA;AAAA,KACnE;AAEA,IAAO,OAAA,GAAA,CAAA;AAAA,GACT;AAAA,EAEA,MAAc,OAAO,KAAiD,EAAA;AACpE,IAAM,MAAA,GAAA,GAAA,CACJ,MAAM,IAAK,CAAA,QAAA,CAAS,iBAAiB,CAClC,CAAA,IAAA,CAAK,eAAe,CACpB,CAAA,SAAA;AAAA,MACC,gBAAA;AAAA,MACA,0BAAA;AAAA,MACA,yBAAA;AAAA,MAED,SAAU,CAAA,6BAA6B,CAC1C,EAAA,GAAA,CAAI,KAAK,mBAAmB,CAAA,CAAA;AAC9B,IAAA,IAAI,KAAO,EAAA;AACT,MAAO,OAAA,GAAA,CAAI,OAAO,CAAE,CAAA,KAAA;AApG1B,QAAA,IAAA,EAAA,CAAA;AAoG6B,QAAE,OAAA,CAAA,CAAA,EAAA,GAAA,CAAA,CAAA,kBAAA,CAAmB,IAArB,KAAA,IAAA,GAAA,KAAA,CAAA,GAAA,EAAA,CAA2B,KAAU,MAAA,KAAA,CAAA;AAAA,OAAK,CAAA,CAAA;AAAA,KACnE;AAEA,IAAO,OAAA,GAAA,CAAA;AAAA,GACT;AAAA,EAEA,cAAiB,GAAA;AACf,IAAA,IAAA,CAAK,YACF,CAAA,GAAA,CAAI,8BAAgC,EAAA,OAAO,KAAK,GAAQ,KAAA;AACvD,MAAA,OAAO,GAAI,CAAA,IAAA;AAAA,QACT,MAAM,KAAK,WAAY,CAAA;AAAA,UACrB,MAAQ,EAAA,QAAA;AAAA,UACR,KAAA,EAAO,IAAI,KAAM,CAAA,KAAA;AAAA,UACjB,kBAAoB,EAAAC,oDAAA;AAAA,YAClB,GAAA,CAAI,OAAO,eAAe,CAAA;AAAA,WAC5B;AAAA,SACD,CAAA;AAAA,OACH,CAAA;AAAA,KACD,CACA,CAAA,GAAA,CAAI,+BAAiC,EAAA,OAAO,KAAK,GAAQ,KAAA;AACxD,MAAA,OAAO,GAAI,CAAA,IAAA;AAAA,QACT,MAAM,KAAK,WAAY,CAAA;AAAA,UACrB,MAAQ,EAAA,SAAA;AAAA,UACR,KAAA,EAAO,IAAI,KAAM,CAAA,KAAA;AAAA,UACjB,kBAAoB,EAAAA,oDAAA;AAAA,YAClB,GAAA,CAAI,OAAO,eAAe,CAAA;AAAA,WAC5B;AAAA,SACD,CAAA;AAAA,OACH,CAAA;AAAA,KACD,CAAA,CAAA;AAAA,GACL;AACF;;ACxGO,MAAM,mCAAmCC,oCAAoB,CAAA;AAAA,EAClE,QAAU,EAAA,SAAA;AAAA,EACV,QAAU,EAAA,kCAAA;AAAA,EACV,SAAS,GAAK,EAAA;AACZ,IAAA,GAAA,CAAI,YAAa,CAAA;AAAA,MACf,IAAM,EAAA;AAAA,QACJ,UAAUC,6BAAa,CAAA,QAAA;AAAA,QACvB,QAAQA,6BAAa,CAAA,UAAA;AAAA,QACrB,QAAQA,6BAAa,CAAA,MAAA;AAAA,OACvB;AAAA,MACA,MAAM,IAAK,CAAA,EAAE,QAAU,EAAA,MAAA,EAAQ,QAAU,EAAA;AACvC,QAAA,MAAM,SAAS,IAAI,wBAAA;AAAA,UACjB,MAAM,SAAS,SAAU,EAAA;AAAA,UACzB,MAAA;AAAA,SACF,CAAA;AAEA,QAAA,MAAA,CAAO,cAAe,EAAA,CAAA;AACtB,QAAO,MAAA,CAAA,IAAA;AAAA,UACL,mEAAA;AAAA,SACF,CAAA;AAAA,OACF;AAAA,KACD,CAAA,CAAA;AAAA,GACH;AACF,CAAC;;;;;"}

package/dist/index.d.ts

@@ -0,0 +1,29 @@
+import * as _backstage_backend_plugin_api from '@backstage/backend-plugin-api';
+import { HttpRouterService } from '@backstage/backend-plugin-api';
+import { Knex } from 'knex';
+
+/**
+ * Catalog Module for Unprocessed Entities
+ *
+ * @public
+ */
+declare const catalogModuleUnprocessedEntities: () => _backstage_backend_plugin_api.BackendFeature;
+
+/**
+ * Module providing Unprocessed Entities API endpoints
+ *
+ * @public
+ */
+declare class UnprocessedEntitesModule {
+    private readonly database;
+    private readonly router;
+    private readonly moduleRouter;
+    constructor(database: Knex, router: HttpRouterService);
+    private unprocessed;
+    private hydrateRefreshState;
+    private pending;
+    private failed;
+    registerRoutes(): void;
+}
+
+export { UnprocessedEntitesModule, catalogModuleUnprocessedEntities };

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "@backstage/plugin-catalog-backend-module-unprocessed",
+  "description": "Backstage Catalog module to view unprocessed entities",
+  "version": "0.0.0-nightly-20230531023430",
+  "main": "dist/index.cjs.js",
+  "types": "dist/index.d.ts",
+  "license": "Apache-2.0",
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "backend-plugin-module"
+  },
+  "scripts": {
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "clean": "backstage-cli package clean",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack"
+  },
+  "devDependencies": {
+    "@backstage/cli": "^0.0.0-nightly-20230531023430"
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "@backstage/backend-plugin-api": "^0.0.0-nightly-20230531023430",
+    "@backstage/catalog-model": "^0.0.0-nightly-20230531023430",
+    "@backstage/plugin-auth-node": "^0.0.0-nightly-20230531023430",
+    "express-promise-router": "^4.1.1",
+    "knex": "^2.4.2"
+  }
+}