@backstage/plugin-catalog-backend 1.6.0-next.0 → 1.6.0-next.1

package/CHANGELOG.md

@@ -1,5 +1,53 @@
 # @backstage/plugin-catalog-backend
 
+## 1.6.0-next.1
+
+### Minor Changes
+
+- c395abb5b2: The catalog no longer stops after the first processor `validateEntityKind`
+  method returns `true` when validating entity kind shapes. Instead, it continues
+  through all registered processors that have this method, and requires that _at
+  least one_ of them returned true.
+
+  The old behavior of stopping early made it harder to extend existing core kinds
+  with additional fields, since the `BuiltinKindsEntityProcessor` is always
+  present at the top of the processing chain and ensures that your additional
+  validation code would never be run.
+
+  This is technically a breaking change, although it should not affect anybody
+  under normal circumstances, except if you had problematic validation code that
+  you were unaware that it was not being run. That code may now start to exhibit
+  those problems.
+
+  If you need to disable this new behavior, `CatalogBuilder` as used in your
+  `packages/backend/src/plugins/catalog.ts` file now has a
+  `useLegacySingleProcessorValidation()` method to go back to the old behavior.
+
+  ```diff
+   const builder = await CatalogBuilder.create(env);
+  +builder.useLegacySingleProcessorValidation();
+  ```
+
+### Patch Changes
+
+- 2a8e3cc0b5: Optimize `Stitcher` process to be more memory efficient
+- 5b3e2afa45: Fixed deprecated use of `substr` into `substring`.
+- Updated dependencies
+  - @backstage/backend-common@0.17.0-next.1
+  - @backstage/types@1.0.2-next.1
+  - @backstage/backend-plugin-api@0.1.5-next.1
+  - @backstage/plugin-catalog-node@1.2.2-next.1
+  - @backstage/plugin-permission-node@0.7.2-next.1
+  - @backstage/config@1.0.5-next.1
+  - @backstage/integration@1.4.1-next.1
+  - @backstage/catalog-client@1.2.0-next.1
+  - @backstage/catalog-model@1.1.4-next.1
+  - @backstage/errors@1.1.4-next.1
+  - @backstage/plugin-catalog-common@1.0.9-next.1
+  - @backstage/plugin-permission-common@0.7.2-next.1
+  - @backstage/plugin-scaffolder-common@1.2.3-next.1
+  - @backstage/plugin-search-common@1.1.2-next.1
+
 ## 1.6.0-next.0
 
 ### Minor Changes

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend",
-  "version": "1.6.0-next.0",
+  "version": "1.6.0-next.1",
   "main": "../dist/index.cjs.js",
   "types": "../dist/index.alpha.d.ts"
 }

package/dist/index.alpha.d.ts

@@ -177,6 +177,7 @@ export declare class CatalogBuilder {
     private locationAnalyzer;
     private readonly permissionRules;
     private allowedLocationType;
+    private legacySingleProcessorValidation;
     /**
      * Creates a catalog builder.
      */
@@ -306,13 +307,18 @@ export declare class CatalogBuilder {
      * @param permissionRules - Additional permission rules
      * @alpha
      */
-    addPermissionRules(...permissionRules: Array<CatalogPermissionRule | Array<CatalogPermissionRule>>): void;
+    addPermissionRules(...permissionRules: Array<CatalogPermissionRule | Array<CatalogPermissionRule>>): this;
     /**
      * Sets up the allowed location types from being registered via the location service.
      *
      * @param allowedLocationTypes - the allowed location types
      */
     setAllowedLocationTypes(allowedLocationTypes: string[]): CatalogBuilder;
+    /**
+     * Enables the legacy behaviour of canceling validation early whenever only a
+     * single processor declares an entity kind to be valid.
+     */
+    useLegacySingleProcessorValidation(): this;
     /**
      * Wires up and returns all of the component parts of the catalog
      */

package/dist/index.beta.d.ts

@@ -177,6 +177,7 @@ export declare class CatalogBuilder {
     private locationAnalyzer;
     private readonly permissionRules;
     private allowedLocationType;
+    private legacySingleProcessorValidation;
     /**
      * Creates a catalog builder.
      */
@@ -305,6 +306,11 @@ export declare class CatalogBuilder {
      * @param allowedLocationTypes - the allowed location types
      */
     setAllowedLocationTypes(allowedLocationTypes: string[]): CatalogBuilder;
+    /**
+     * Enables the legacy behaviour of canceling validation early whenever only a
+     * single processor declares an entity kind to be valid.
+     */
+    useLegacySingleProcessorValidation(): this;
     /**
      * Wires up and returns all of the component parts of the catalog
      */

package/dist/index.cjs.js

@@ -601,7 +601,7 @@ class PlaceholderProcessor {
       } else if (keys.length !== 1) {
         return [data, false];
       }
-      const resolverKey = keys[0].substr(1);
+      const resolverKey = keys[0].substring(1);
       const resolverValue = data[keys[0]];
       const resolver = this.options.resolvers[resolverKey];
       if (!resolver) {
@@ -2930,13 +2930,16 @@ class DefaultCatalogProcessingOrchestrator {
         e
       );
     }
-    let foundKind = false;
+    let valid = false;
     for (const processor of this.options.processors) {
       if (processor.validateEntityKind) {
         try {
-          foundKind = await processor.validateEntityKind(entity);
-          if (foundKind) {
-            break;
+          const thisValid = await processor.validateEntityKind(entity);
+          if (thisValid) {
+            valid = true;
+            if (this.options.legacySingleProcessorValidation) {
+              break;
+            }
           }
         } catch (e) {
           throw new errors.InputError(
@@ -2946,7 +2949,7 @@ class DefaultCatalogProcessingOrchestrator {
         }
       }
     }
-    if (!foundKind) {
+    if (!valid) {
       throw new errors.InputError(
         `No processor recognized the entity ${context.entityRef} as valid, possibly caused by a foreign kind or apiVersion`
       );
@@ -3159,22 +3162,24 @@ class Stitcher {
       hash: "",
       stitch_ticket: ticket
     }).onConflict("entity_id").merge(["stitch_ticket"]);
-    const result = await this.database.with("incoming_references", function incomingReferences(builder) {
-      return builder.from("refresh_state_references").where({ target_entity_ref: entityRef }).count({ count: "*" });
-    }).select({
-      entityId: "refresh_state.entity_id",
-      processedEntity: "refresh_state.processed_entity",
-      errors: "refresh_state.errors",
-      incomingReferenceCount: "incoming_references.count",
-      previousHash: "final_entities.hash",
-      relationType: "relations.type",
-      relationTarget: "relations.target_entity_ref"
-    }).from("refresh_state").where({ "refresh_state.entity_ref": entityRef }).crossJoin(this.database.raw("incoming_references")).leftOuterJoin("final_entities", {
-      "final_entities.entity_id": "refresh_state.entity_id"
-    }).leftOuterJoin("relations", {
-      "relations.source_entity_ref": "refresh_state.entity_ref"
-    }).orderBy("relationType", "asc").orderBy("relationTarget", "asc");
-    if (!result.length) {
+    const [processedResult, relationsResult] = await Promise.all([
+      this.database.with("incoming_references", function incomingReferences(builder) {
+        return builder.from("refresh_state_references").where({ target_entity_ref: entityRef }).count({ count: "*" });
+      }).select({
+        entityId: "refresh_state.entity_id",
+        processedEntity: "refresh_state.processed_entity",
+        errors: "refresh_state.errors",
+        incomingReferenceCount: "incoming_references.count",
+        previousHash: "final_entities.hash"
+      }).from("refresh_state").where({ "refresh_state.entity_ref": entityRef }).crossJoin(this.database.raw("incoming_references")).leftOuterJoin("final_entities", {
+        "final_entities.entity_id": "refresh_state.entity_id"
+      }),
+      this.database.distinct({
+        relationType: "type",
+        relationTarget: "target_entity_ref"
+      }).from("relations").where({ source_entity_ref: entityRef }).orderBy("relationType", "asc").orderBy("relationTarget", "asc")
+    ]);
+    if (!processedResult.length) {
       this.logger.error(
         `Unable to stitch ${entityRef}, item does not exist in refresh state table`
       );
@@ -3186,7 +3191,7 @@ class Stitcher {
       errors,
       incomingReferenceCount,
       previousHash
-    } = result[0];
+    } = processedResult[0];
     if (!processedEntity) {
       this.logger.debug(
         `Unable to stitch ${entityRef}, the entity has not yet been processed`
@@ -3214,11 +3219,7 @@ class Stitcher {
         }));
       }
     }
-    const uniqueRelationRows = lodash.uniqBy(
-      result,
-      (r) => `${r.relationType}:${r.relationTarget}`
-    );
-    entity.relations = uniqueRelationRows.filter((row) => row.relationType).map((row) => ({
+    entity.relations = relationsResult.filter((row) => row.relationType).map((row) => ({
       type: row.relationType,
       targetRef: row.relationTarget
     }));
@@ -3330,8 +3331,8 @@ function parseEntityFilterString(filterString) {
   const filtersByKey = {};
   for (const statement of statements) {
     const equalsIndex = statement.indexOf("=");
-    const key = equalsIndex === -1 ? statement : statement.substr(0, equalsIndex).trim();
-    const value = equalsIndex === -1 ? void 0 : statement.substr(equalsIndex + 1).trim();
+    const key = equalsIndex === -1 ? statement : statement.substring(0, equalsIndex).trim();
+    const value = equalsIndex === -1 ? void 0 : statement.substring(equalsIndex + 1).trim();
     if (!key) {
       throw new errors.InputError(
         `Invalid filter, '${statement}' is not a valid statement (expected a string on the form a=b or a= or a)`
@@ -4057,6 +4058,7 @@ class CatalogBuilder {
       maxSeconds: 150
     });
     this.locationAnalyzer = void 0;
+    this.legacySingleProcessorValidation = false;
     this.env = env;
     this.entityPolicies = [];
     this.entityPoliciesReplace = false;
@@ -4138,11 +4140,16 @@ class CatalogBuilder {
   }
   addPermissionRules(...permissionRules) {
     this.permissionRules.push(...permissionRules.flat());
+    return this;
   }
   setAllowedLocationTypes(allowedLocationTypes) {
     this.allowedLocationType = allowedLocationTypes;
     return this;
   }
+  useLegacySingleProcessorValidation() {
+    this.legacySingleProcessorValidation = true;
+    return this;
+  }
   async build() {
     var _a, _b;
     const { config, database, logger, permissions } = this.env;
@@ -4167,7 +4174,8 @@ class CatalogBuilder {
       rulesEnforcer,
       logger,
       parser,
-      policy
+      policy,
+      legacySingleProcessorValidation: this.legacySingleProcessorValidation
     });
     const stitcher = new Stitcher(dbClient, logger);
     const unauthorizedEntitiesCatalog = new DefaultEntitiesCatalog(