@backstage/plugin-catalog-backend 1.6.0-next.0 → 1.6.0-next.2

package/CHANGELOG.md

@@ -1,5 +1,81 @@
 # @backstage/plugin-catalog-backend
 
+## 1.6.0-next.2
+
+### Minor Changes
+
+- 3072ebfdd7: The search table also holds the original entity value now and the facets endpoint fetches the filtered entity data from the search table.
+
+### Patch Changes
+
+- c507aee8a2: Ensured typescript type checks in migration files.
+- 884d749b14: Refactored to use `coreServices` from `@backstage/backend-plugin-api`.
+- eacc8e2b55: Make it possible for entity providers to supply only entity refs, instead of full entities, in `delta` mutation deletions.
+- 20a5161f04: Adds MySQL support for the catalog-backend
+- Updated dependencies
+  - @backstage/plugin-catalog-node@1.3.0-next.2
+  - @backstage/backend-common@0.17.0-next.2
+  - @backstage/backend-plugin-api@0.2.0-next.2
+  - @backstage/plugin-search-common@1.2.0-next.2
+  - @backstage/plugin-permission-node@0.7.2-next.2
+  - @backstage/catalog-client@1.2.0-next.1
+  - @backstage/catalog-model@1.1.4-next.1
+  - @backstage/config@1.0.5-next.1
+  - @backstage/errors@1.1.4-next.1
+  - @backstage/integration@1.4.1-next.1
+  - @backstage/types@1.0.2-next.1
+  - @backstage/plugin-catalog-common@1.0.9-next.2
+  - @backstage/plugin-permission-common@0.7.2-next.1
+  - @backstage/plugin-scaffolder-common@1.2.3-next.1
+
+## 1.6.0-next.1
+
+### Minor Changes
+
+- c395abb5b2: The catalog no longer stops after the first processor `validateEntityKind`
+  method returns `true` when validating entity kind shapes. Instead, it continues
+  through all registered processors that have this method, and requires that _at
+  least one_ of them returned true.
+
+  The old behavior of stopping early made it harder to extend existing core kinds
+  with additional fields, since the `BuiltinKindsEntityProcessor` is always
+  present at the top of the processing chain and ensures that your additional
+  validation code would never be run.
+
+  This is technically a breaking change, although it should not affect anybody
+  under normal circumstances, except if you had problematic validation code that
+  you were unaware that it was not being run. That code may now start to exhibit
+  those problems.
+
+  If you need to disable this new behavior, `CatalogBuilder` as used in your
+  `packages/backend/src/plugins/catalog.ts` file now has a
+  `useLegacySingleProcessorValidation()` method to go back to the old behavior.
+
+  ```diff
+   const builder = await CatalogBuilder.create(env);
+  +builder.useLegacySingleProcessorValidation();
+  ```
+
+### Patch Changes
+
+- 2a8e3cc0b5: Optimize `Stitcher` process to be more memory efficient
+- 5b3e2afa45: Fixed deprecated use of `substr` into `substring`.
+- Updated dependencies
+  - @backstage/backend-common@0.17.0-next.1
+  - @backstage/types@1.0.2-next.1
+  - @backstage/backend-plugin-api@0.1.5-next.1
+  - @backstage/plugin-catalog-node@1.2.2-next.1
+  - @backstage/plugin-permission-node@0.7.2-next.1
+  - @backstage/config@1.0.5-next.1
+  - @backstage/integration@1.4.1-next.1
+  - @backstage/catalog-client@1.2.0-next.1
+  - @backstage/catalog-model@1.1.4-next.1
+  - @backstage/errors@1.1.4-next.1
+  - @backstage/plugin-catalog-common@1.0.9-next.1
+  - @backstage/plugin-permission-common@0.7.2-next.1
+  - @backstage/plugin-scaffolder-common@1.2.3-next.1
+  - @backstage/plugin-search-common@1.1.2-next.1
+
 ## 1.6.0-next.0
 
 ### Minor Changes

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend",
-  "version": "1.6.0-next.0",
+  "version": "1.6.0-next.2",
   "main": "../dist/index.cjs.js",
   "types": "../dist/index.alpha.d.ts"
 }

package/dist/index.alpha.d.ts

@@ -177,6 +177,7 @@ export declare class CatalogBuilder {
     private locationAnalyzer;
     private readonly permissionRules;
     private allowedLocationType;
+    private legacySingleProcessorValidation;
     /**
      * Creates a catalog builder.
      */
@@ -306,13 +307,18 @@ export declare class CatalogBuilder {
      * @param permissionRules - Additional permission rules
      * @alpha
      */
-    addPermissionRules(...permissionRules: Array<CatalogPermissionRule | Array<CatalogPermissionRule>>): void;
+    addPermissionRules(...permissionRules: Array<CatalogPermissionRule | Array<CatalogPermissionRule>>): this;
     /**
      * Sets up the allowed location types from being registered via the location service.
      *
      * @param allowedLocationTypes - the allowed location types
      */
     setAllowedLocationTypes(allowedLocationTypes: string[]): CatalogBuilder;
+    /**
+     * Enables the legacy behaviour of canceling validation early whenever only a
+     * single processor declares an entity kind to be valid.
+     */
+    useLegacySingleProcessorValidation(): this;
     /**
      * Wires up and returns all of the component parts of the catalog
      */

package/dist/index.beta.d.ts

@@ -177,6 +177,7 @@ export declare class CatalogBuilder {
     private locationAnalyzer;
     private readonly permissionRules;
     private allowedLocationType;
+    private legacySingleProcessorValidation;
     /**
      * Creates a catalog builder.
      */
@@ -305,6 +306,11 @@ export declare class CatalogBuilder {
      * @param allowedLocationTypes - the allowed location types
      */
     setAllowedLocationTypes(allowedLocationTypes: string[]): CatalogBuilder;
+    /**
+     * Enables the legacy behaviour of canceling validation early whenever only a
+     * single processor declares an entity kind to be valid.
+     */
+    useLegacySingleProcessorValidation(): this;
     /**
      * Wires up and returns all of the component parts of the catalog
      */

package/dist/index.cjs.js

@@ -601,7 +601,7 @@ class PlaceholderProcessor {
       } else if (keys.length !== 1) {
         return [data, false];
       }
-      const resolverKey = keys[0].substr(1);
+      const resolverKey = keys[0].substring(1);
       const resolverValue = data[keys[0]];
       const resolver = this.options.resolvers[resolverKey];
       if (!resolver) {
@@ -1517,6 +1517,7 @@ class DefaultProcessingDatabase {
       refreshKeys,
       locationKey
     } = options;
+    const configClient = tx.client.config.client;
     const refreshResult = await tx("refresh_state").update({
       processed_entity: JSON.stringify(processedEntity),
       result_hash: resultHash,
@@ -1539,7 +1540,7 @@ class DefaultProcessingDatabase {
       sourceEntityRef
     });
     let previousRelationRows;
-    if (tx.client.config.client.includes("sqlite3") || tx.client.config.client.includes("mysql")) {
+    if (configClient.includes("sqlite3") || configClient.includes("mysql")) {
       previousRelationRows = await tx("relations").select("*").where({ originating_entity_id: id });
       await tx("relations").where({ originating_entity_id: id }).delete();
     } else {
@@ -1868,16 +1869,20 @@ class DefaultProcessingDatabase {
         next_update_at: tx.fn.now(),
         last_discovery_at: tx.fn.now()
       });
-      if (!tx.client.config.client.includes("sqlite3")) {
+      if (tx.client.config.client.includes("pg")) {
         query = query.onConflict("entity_ref").ignore();
       }
       const result = await query;
       return result.rowCount === 1 || result.length === 1;
     } catch (error) {
-      if (errors.isError(error) && error.message.includes("UNIQUE constraint failed")) {
+      if (!backendCommon.isDatabaseConflictError(error)) {
+        throw error;
+      } else {
+        this.options.logger.debug(
+          `Unable to insert a new refresh state row, ${error}`
+        );
         return false;
       }
-      throw error;
     }
   }
   async checkLocationKeyConflict(tx, entityRef, locationKey) {
@@ -1906,7 +1911,7 @@ class DefaultProcessingDatabase {
           deferred: e,
           hash: generateStableHash$1(e.entity)
         })),
-        toRemove: options.removed.map((e) => catalogModel.stringifyEntityRef(e.entity))
+        toRemove: options.removed.map((e) => e.entityRef)
       };
     }
     const oldRefs = await tx(
@@ -2425,8 +2430,8 @@ function stringifyPagination(input) {
   const base64 = Buffer.from(json, "utf8").toString("base64");
   return base64;
 }
-function addCondition(queryBuilder, db, filter, negate = false) {
-  const matchQuery = db("search").select("entity_id").where({ key: filter.key.toLowerCase() }).andWhere(function keyFilter() {
+function addCondition(queryBuilder, db, filter, negate = false, entityIdField = "entity_id") {
+  const matchQuery = db("search").select(entityIdField).where({ key: filter.key.toLowerCase() }).andWhere(function keyFilter() {
     if (filter.values) {
       if (filter.values.length === 1) {
         this.where({ value: filter.values[0].toLowerCase() });
@@ -2439,7 +2444,7 @@ function addCondition(queryBuilder, db, filter, negate = false) {
       }
     }
   });
-  queryBuilder.andWhere("entity_id", negate ? "not in" : "in", matchQuery);
+  queryBuilder.andWhere(entityIdField, negate ? "not in" : "in", matchQuery);
 }
 function isEntitiesSearchFilter(filter) {
   return filter.hasOwnProperty("key");
@@ -2450,24 +2455,28 @@ function isOrEntityFilter(filter) {
 function isNegationEntityFilter(filter) {
   return filter.hasOwnProperty("not");
 }
-function parseFilter(filter, query, db, negate = false) {
+function parseFilter(filter, query, db, negate = false, entityIdField = "entity_id") {
   if (isEntitiesSearchFilter(filter)) {
     return query.andWhere(function filterFunction() {
-      addCondition(this, db, filter, negate);
+      addCondition(this, db, filter, negate, entityIdField);
     });
   }
   if (isNegationEntityFilter(filter)) {
-    return parseFilter(filter.not, query, db, !negate);
+    return parseFilter(filter.not, query, db, !negate, entityIdField);
   }
   return query[negate ? "andWhereNot" : "andWhere"](function filterFunction() {
     var _a, _b;
     if (isOrEntityFilter(filter)) {
       for (const subFilter of (_a = filter.anyOf) != null ? _a : []) {
-        this.orWhere((subQuery) => parseFilter(subFilter, subQuery, db));
+        this.orWhere(
+          (subQuery) => parseFilter(subFilter, subQuery, db, false, entityIdField)
+        );
       }
     } else {
       for (const subFilter of (_b = filter.allOf) != null ? _b : []) {
-        this.andWhere((subQuery) => parseFilter(subFilter, subQuery, db));
+        this.andWhere(
+          (subQuery) => parseFilter(subFilter, subQuery, db, false, entityIdField)
+        );
       }
     }
   });
@@ -2551,14 +2560,36 @@ class DefaultEntitiesCatalog {
     return { items };
   }
   async removeEntityByUid(uid) {
-    await this.database("refresh_state").update({
-      result_hash: "child-was-deleted",
-      next_update_at: this.database.fn.now()
-    }).whereIn("entity_ref", function parents(builder) {
-      return builder.from("refresh_state").innerJoin("refresh_state_references", {
-        "refresh_state_references.target_entity_ref": "refresh_state.entity_ref"
-      }).where("refresh_state.entity_id", "=", uid).select("refresh_state_references.source_entity_ref");
-    });
+    const dbConfig = this.database.client.config;
+    if (dbConfig.client.includes("mysql")) {
+      const results = await this.database("refresh_state").select("entity_id").whereIn("entity_ref", function parents(builder) {
+        return builder.from("refresh_state").innerJoin(
+          "refresh_state_references",
+          {
+            "refresh_state_references.target_entity_ref": "refresh_state.entity_ref"
+          }
+        ).where("refresh_state.entity_id", "=", uid).select("refresh_state_references.source_entity_ref");
+      });
+      await this.database("refresh_state").update({
+        result_hash: "child-was-deleted",
+        next_update_at: this.database.fn.now()
+      }).whereIn(
+        "entity_id",
+        results.map((key) => key.entity_id)
+      );
+    } else {
+      await this.database("refresh_state").update({
+        result_hash: "child-was-deleted",
+        next_update_at: this.database.fn.now()
+      }).whereIn("entity_ref", function parents(builder) {
+        return builder.from("refresh_state").innerJoin(
+          "refresh_state_references",
+          {
+            "refresh_state_references.target_entity_ref": "refresh_state.entity_ref"
+          }
+        ).where("refresh_state.entity_id", "=", uid).select("refresh_state_references.source_entity_ref");
+      });
+    }
     const relationPeers = await this.database.from("relations").innerJoin("refresh_state", {
       "refresh_state.entity_ref": "relations.target_entity_ref"
     }).where("relations.originating_entity_id", "=", uid).andWhere("refresh_state.entity_id", "!=", uid).select({ ref: "relations.target_entity_ref" }).union(
@@ -2614,32 +2645,17 @@ class DefaultEntitiesCatalog {
     };
   }
   async facets(request) {
-    const { entities } = await this.entities({
-      filter: request.filter,
-      authorizationToken: request.authorizationToken
-    });
     const facets = {};
+    const db = this.database;
     for (const facet of request.facets) {
-      const values = entities.map((entity) => {
-        var _a, _b;
-        if (facet.startsWith("metadata.annotations.")) {
-          return (_a = entity.metadata.annotations) == null ? void 0 : _a[facet.substring("metadata.annotations.".length)];
-        } else if (facet.startsWith("metadata.labels.")) {
-          return (_b = entity.metadata.labels) == null ? void 0 : _b[facet.substring("metadata.labels.".length)];
-        }
-        return lodash__default["default"].get(entity, facet);
-      }).flatMap((field) => {
-        if (typeof field === "string") {
-          return [field];
-        } else if (Array.isArray(field)) {
-          return field.filter((i) => typeof i === "string");
-        }
-        return [];
-      }).sort();
-      const counts = lodash__default["default"].countBy(values, lodash__default["default"].identity);
-      facets[facet] = Object.entries(counts).map(([value, count]) => ({
-        value,
-        count
+      const dbQuery = db("search").join("final_entities", "search.entity_id", "final_entities.entity_id").where("search.key", facet.toLocaleLowerCase("en-US")).count("search.entity_id as count").select({ value: "search.original_value" }).groupBy("search.original_value");
+      if (request == null ? void 0 : request.filter) {
+        parseFilter(request.filter, dbQuery, db, false, "search.entity_id");
+      }
+      const result = await dbQuery;
+      facets[facet] = result.map((data) => ({
+        value: String(data.value),
+        count: Number(data.count)
       }));
     }
     return { facets };
@@ -2930,13 +2946,16 @@ class DefaultCatalogProcessingOrchestrator {
         e
       );
     }
-    let foundKind = false;
+    let valid = false;
     for (const processor of this.options.processors) {
       if (processor.validateEntityKind) {
         try {
-          foundKind = await processor.validateEntityKind(entity);
-          if (foundKind) {
-            break;
+          const thisValid = await processor.validateEntityKind(entity);
+          if (thisValid) {
+            valid = true;
+            if (this.options.legacySingleProcessorValidation) {
+              break;
+            }
           }
         } catch (e) {
           throw new errors.InputError(
@@ -2946,7 +2965,7 @@ class DefaultCatalogProcessingOrchestrator {
         }
       }
     }
-    if (!foundKind) {
+    if (!valid) {
       throw new errors.InputError(
         `No processor recognized the entity ${context.entityRef} as valid, possibly caused by a foreign kind or apiVersion`
       );
@@ -3079,15 +3098,30 @@ function mapToRows(input, entityId) {
   for (const { key: rawKey, value: rawValue } of input) {
     const key = rawKey.toLocaleLowerCase("en-US");
     if (rawValue === void 0 || rawValue === null) {
-      result.push({ entity_id: entityId, key, value: null });
+      result.push({
+        entity_id: entityId,
+        key,
+        original_value: null,
+        value: null
+      });
     } else {
       const value = String(rawValue).toLocaleLowerCase("en-US");
       if (key.length <= MAX_KEY_LENGTH) {
-        result.push({
-          entity_id: entityId,
-          key,
-          value: value.length <= MAX_VALUE_LENGTH ? value : null
-        });
+        if (value.length <= MAX_VALUE_LENGTH) {
+          result.push({
+            entity_id: entityId,
+            key,
+            original_value: String(rawValue),
+            value
+          });
+        } else {
+          result.push({
+            entity_id: entityId,
+            key,
+            original_value: null,
+            value: null
+          });
+        }
       }
     }
   }
@@ -3159,22 +3193,24 @@ class Stitcher {
       hash: "",
       stitch_ticket: ticket
     }).onConflict("entity_id").merge(["stitch_ticket"]);
-    const result = await this.database.with("incoming_references", function incomingReferences(builder) {
-      return builder.from("refresh_state_references").where({ target_entity_ref: entityRef }).count({ count: "*" });
-    }).select({
-      entityId: "refresh_state.entity_id",
-      processedEntity: "refresh_state.processed_entity",
-      errors: "refresh_state.errors",
-      incomingReferenceCount: "incoming_references.count",
-      previousHash: "final_entities.hash",
-      relationType: "relations.type",
-      relationTarget: "relations.target_entity_ref"
-    }).from("refresh_state").where({ "refresh_state.entity_ref": entityRef }).crossJoin(this.database.raw("incoming_references")).leftOuterJoin("final_entities", {
-      "final_entities.entity_id": "refresh_state.entity_id"
-    }).leftOuterJoin("relations", {
-      "relations.source_entity_ref": "refresh_state.entity_ref"
-    }).orderBy("relationType", "asc").orderBy("relationTarget", "asc");
-    if (!result.length) {
+    const [processedResult, relationsResult] = await Promise.all([
+      this.database.with("incoming_references", function incomingReferences(builder) {
+        return builder.from("refresh_state_references").where({ target_entity_ref: entityRef }).count({ count: "*" });
+      }).select({
+        entityId: "refresh_state.entity_id",
+        processedEntity: "refresh_state.processed_entity",
+        errors: "refresh_state.errors",
+        incomingReferenceCount: "incoming_references.count",
+        previousHash: "final_entities.hash"
+      }).from("refresh_state").where({ "refresh_state.entity_ref": entityRef }).crossJoin(this.database.raw("incoming_references")).leftOuterJoin("final_entities", {
+        "final_entities.entity_id": "refresh_state.entity_id"
+      }),
+      this.database.distinct({
+        relationType: "type",
+        relationTarget: "target_entity_ref"
+      }).from("relations").where({ source_entity_ref: entityRef }).orderBy("relationType", "asc").orderBy("relationTarget", "asc")
+    ]);
+    if (!processedResult.length) {
       this.logger.error(
         `Unable to stitch ${entityRef}, item does not exist in refresh state table`
       );
@@ -3186,7 +3222,7 @@ class Stitcher {
       errors,
       incomingReferenceCount,
       previousHash
-    } = result[0];
+    } = processedResult[0];
     if (!processedEntity) {
       this.logger.debug(
         `Unable to stitch ${entityRef}, the entity has not yet been processed`
@@ -3214,11 +3250,7 @@ class Stitcher {
         }));
       }
     }
-    const uniqueRelationRows = lodash.uniqBy(
-      result,
-      (r) => `${r.relationType}:${r.relationTarget}`
-    );
-    entity.relations = uniqueRelationRows.filter((row) => row.relationType).map((row) => ({
+    entity.relations = relationsResult.filter((row) => row.relationType).map((row) => ({
       type: row.relationType,
       targetRef: row.relationTarget
     }));
@@ -3330,8 +3362,8 @@ function parseEntityFilterString(filterString) {
   const filtersByKey = {};
   for (const statement of statements) {
     const equalsIndex = statement.indexOf("=");
-    const key = equalsIndex === -1 ? statement : statement.substr(0, equalsIndex).trim();
-    const value = equalsIndex === -1 ? void 0 : statement.substr(equalsIndex + 1).trim();
+    const key = equalsIndex === -1 ? statement : statement.substring(0, equalsIndex).trim();
+    const value = equalsIndex === -1 ? void 0 : statement.substring(equalsIndex + 1).trim();
     if (!key) {
       throw new errors.InputError(
         `Invalid filter, '${statement}' is not a valid statement (expected a string on the form a=b or a= or a)`
@@ -3807,13 +3839,20 @@ class Connection {
       });
     } else if (mutation.type === "delta") {
       this.check(mutation.added.map((e) => e.entity));
-      this.check(mutation.removed.map((e) => e.entity));
+      this.check(
+        mutation.removed.map((e) => "entity" in e ? e.entity : void 0).filter((e) => Boolean(e))
+      );
       await db.transaction(async (tx) => {
         await db.replaceUnprocessedEntities(tx, {
           sourceKey: this.config.id,
           type: "delta",
           added: mutation.added,
-          removed: mutation.removed
+          removed: mutation.removed.map(
+            (r) => "entityRef" in r ? r : {
+              entityRef: catalogModel.stringifyEntityRef(r.entity),
+              locationKey: r.locationKey
+            }
+          )
         });
       });
     }
@@ -4057,6 +4096,7 @@ class CatalogBuilder {
       maxSeconds: 150
     });
     this.locationAnalyzer = void 0;
+    this.legacySingleProcessorValidation = false;
     this.env = env;
     this.entityPolicies = [];
     this.entityPoliciesReplace = false;
@@ -4138,11 +4178,16 @@ class CatalogBuilder {
   }
   addPermissionRules(...permissionRules) {
     this.permissionRules.push(...permissionRules.flat());
+    return this;
   }
   setAllowedLocationTypes(allowedLocationTypes) {
     this.allowedLocationType = allowedLocationTypes;
     return this;
   }
+  useLegacySingleProcessorValidation() {
+    this.legacySingleProcessorValidation = true;
+    return this;
+  }
   async build() {
     var _a, _b;
     const { config, database, logger, permissions } = this.env;
@@ -4167,7 +4212,8 @@ class CatalogBuilder {
       rulesEnforcer,
       logger,
       parser,
-      policy
+      policy,
+      legacySingleProcessorValidation: this.legacySingleProcessorValidation
     });
     const stitcher = new Stitcher(dbClient, logger);
     const unauthorizedEntitiesCatalog = new DefaultEntitiesCatalog(
@@ -4436,13 +4482,13 @@ const catalogPlugin = backendPluginApi.createBackendPlugin({
     );
     env.registerInit({
       deps: {
-        logger: backendPluginApi.loggerServiceRef,
-        config: backendPluginApi.configServiceRef,
-        reader: backendPluginApi.urlReaderServiceRef,
-        permissions: backendPluginApi.permissionsServiceRef,
-        database: backendPluginApi.databaseServiceRef,
-        httpRouter: backendPluginApi.httpRouterServiceRef,
-        lifecycle: backendPluginApi.lifecycleServiceRef
+        logger: backendPluginApi.coreServices.logger,
+        config: backendPluginApi.coreServices.config,
+        reader: backendPluginApi.coreServices.urlReader,
+        permissions: backendPluginApi.coreServices.permissions,
+        database: backendPluginApi.coreServices.database,
+        httpRouter: backendPluginApi.coreServices.httpRouter,
+        lifecycle: backendPluginApi.coreServices.lifecycle
       },
       async init({
         logger,