@backstage/plugin-catalog-backend 1.6.0-next.1 → 1.6.0-next.2

package/CHANGELOG.md

@@ -1,5 +1,33 @@
 # @backstage/plugin-catalog-backend
 
+## 1.6.0-next.2
+
+### Minor Changes
+
+- 3072ebfdd7: The search table also holds the original entity value now and the facets endpoint fetches the filtered entity data from the search table.
+
+### Patch Changes
+
+- c507aee8a2: Ensured typescript type checks in migration files.
+- 884d749b14: Refactored to use `coreServices` from `@backstage/backend-plugin-api`.
+- eacc8e2b55: Make it possible for entity providers to supply only entity refs, instead of full entities, in `delta` mutation deletions.
+- 20a5161f04: Adds MySQL support for the catalog-backend
+- Updated dependencies
+  - @backstage/plugin-catalog-node@1.3.0-next.2
+  - @backstage/backend-common@0.17.0-next.2
+  - @backstage/backend-plugin-api@0.2.0-next.2
+  - @backstage/plugin-search-common@1.2.0-next.2
+  - @backstage/plugin-permission-node@0.7.2-next.2
+  - @backstage/catalog-client@1.2.0-next.1
+  - @backstage/catalog-model@1.1.4-next.1
+  - @backstage/config@1.0.5-next.1
+  - @backstage/errors@1.1.4-next.1
+  - @backstage/integration@1.4.1-next.1
+  - @backstage/types@1.0.2-next.1
+  - @backstage/plugin-catalog-common@1.0.9-next.2
+  - @backstage/plugin-permission-common@0.7.2-next.1
+  - @backstage/plugin-scaffolder-common@1.2.3-next.1
+
 ## 1.6.0-next.1
 
 ### Minor Changes

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-catalog-backend",
-  "version": "1.6.0-next.1",
+  "version": "1.6.0-next.2",
   "main": "../dist/index.cjs.js",
   "types": "../dist/index.alpha.d.ts"
 }

package/dist/index.cjs.js

@@ -1517,6 +1517,7 @@ class DefaultProcessingDatabase {
       refreshKeys,
       locationKey
     } = options;
+    const configClient = tx.client.config.client;
     const refreshResult = await tx("refresh_state").update({
       processed_entity: JSON.stringify(processedEntity),
       result_hash: resultHash,
@@ -1539,7 +1540,7 @@ class DefaultProcessingDatabase {
       sourceEntityRef
     });
     let previousRelationRows;
-    if (tx.client.config.client.includes("sqlite3") || tx.client.config.client.includes("mysql")) {
+    if (configClient.includes("sqlite3") || configClient.includes("mysql")) {
       previousRelationRows = await tx("relations").select("*").where({ originating_entity_id: id });
       await tx("relations").where({ originating_entity_id: id }).delete();
     } else {
@@ -1868,16 +1869,20 @@ class DefaultProcessingDatabase {
         next_update_at: tx.fn.now(),
         last_discovery_at: tx.fn.now()
       });
-      if (!tx.client.config.client.includes("sqlite3")) {
+      if (tx.client.config.client.includes("pg")) {
         query = query.onConflict("entity_ref").ignore();
       }
       const result = await query;
       return result.rowCount === 1 || result.length === 1;
     } catch (error) {
-      if (errors.isError(error) && error.message.includes("UNIQUE constraint failed")) {
+      if (!backendCommon.isDatabaseConflictError(error)) {
+        throw error;
+      } else {
+        this.options.logger.debug(
+          `Unable to insert a new refresh state row, ${error}`
+        );
         return false;
       }
-      throw error;
     }
   }
   async checkLocationKeyConflict(tx, entityRef, locationKey) {
@@ -1906,7 +1911,7 @@ class DefaultProcessingDatabase {
           deferred: e,
           hash: generateStableHash$1(e.entity)
         })),
-        toRemove: options.removed.map((e) => catalogModel.stringifyEntityRef(e.entity))
+        toRemove: options.removed.map((e) => e.entityRef)
       };
     }
     const oldRefs = await tx(
@@ -2425,8 +2430,8 @@ function stringifyPagination(input) {
   const base64 = Buffer.from(json, "utf8").toString("base64");
   return base64;
 }
-function addCondition(queryBuilder, db, filter, negate = false) {
-  const matchQuery = db("search").select("entity_id").where({ key: filter.key.toLowerCase() }).andWhere(function keyFilter() {
+function addCondition(queryBuilder, db, filter, negate = false, entityIdField = "entity_id") {
+  const matchQuery = db("search").select(entityIdField).where({ key: filter.key.toLowerCase() }).andWhere(function keyFilter() {
     if (filter.values) {
       if (filter.values.length === 1) {
         this.where({ value: filter.values[0].toLowerCase() });
@@ -2439,7 +2444,7 @@ function addCondition(queryBuilder, db, filter, negate = false) {
       }
     }
   });
-  queryBuilder.andWhere("entity_id", negate ? "not in" : "in", matchQuery);
+  queryBuilder.andWhere(entityIdField, negate ? "not in" : "in", matchQuery);
 }
 function isEntitiesSearchFilter(filter) {
   return filter.hasOwnProperty("key");
@@ -2450,24 +2455,28 @@ function isOrEntityFilter(filter) {
 function isNegationEntityFilter(filter) {
   return filter.hasOwnProperty("not");
 }
-function parseFilter(filter, query, db, negate = false) {
+function parseFilter(filter, query, db, negate = false, entityIdField = "entity_id") {
   if (isEntitiesSearchFilter(filter)) {
     return query.andWhere(function filterFunction() {
-      addCondition(this, db, filter, negate);
+      addCondition(this, db, filter, negate, entityIdField);
     });
   }
   if (isNegationEntityFilter(filter)) {
-    return parseFilter(filter.not, query, db, !negate);
+    return parseFilter(filter.not, query, db, !negate, entityIdField);
   }
   return query[negate ? "andWhereNot" : "andWhere"](function filterFunction() {
     var _a, _b;
     if (isOrEntityFilter(filter)) {
       for (const subFilter of (_a = filter.anyOf) != null ? _a : []) {
-        this.orWhere((subQuery) => parseFilter(subFilter, subQuery, db));
+        this.orWhere(
+          (subQuery) => parseFilter(subFilter, subQuery, db, false, entityIdField)
+        );
       }
     } else {
       for (const subFilter of (_b = filter.allOf) != null ? _b : []) {
-        this.andWhere((subQuery) => parseFilter(subFilter, subQuery, db));
+        this.andWhere(
+          (subQuery) => parseFilter(subFilter, subQuery, db, false, entityIdField)
+        );
       }
     }
   });
@@ -2551,14 +2560,36 @@ class DefaultEntitiesCatalog {
     return { items };
   }
   async removeEntityByUid(uid) {
-    await this.database("refresh_state").update({
-      result_hash: "child-was-deleted",
-      next_update_at: this.database.fn.now()
-    }).whereIn("entity_ref", function parents(builder) {
-      return builder.from("refresh_state").innerJoin("refresh_state_references", {
-        "refresh_state_references.target_entity_ref": "refresh_state.entity_ref"
-      }).where("refresh_state.entity_id", "=", uid).select("refresh_state_references.source_entity_ref");
-    });
+    const dbConfig = this.database.client.config;
+    if (dbConfig.client.includes("mysql")) {
+      const results = await this.database("refresh_state").select("entity_id").whereIn("entity_ref", function parents(builder) {
+        return builder.from("refresh_state").innerJoin(
+          "refresh_state_references",
+          {
+            "refresh_state_references.target_entity_ref": "refresh_state.entity_ref"
+          }
+        ).where("refresh_state.entity_id", "=", uid).select("refresh_state_references.source_entity_ref");
+      });
+      await this.database("refresh_state").update({
+        result_hash: "child-was-deleted",
+        next_update_at: this.database.fn.now()
+      }).whereIn(
+        "entity_id",
+        results.map((key) => key.entity_id)
+      );
+    } else {
+      await this.database("refresh_state").update({
+        result_hash: "child-was-deleted",
+        next_update_at: this.database.fn.now()
+      }).whereIn("entity_ref", function parents(builder) {
+        return builder.from("refresh_state").innerJoin(
+          "refresh_state_references",
+          {
+            "refresh_state_references.target_entity_ref": "refresh_state.entity_ref"
+          }
+        ).where("refresh_state.entity_id", "=", uid).select("refresh_state_references.source_entity_ref");
+      });
+    }
     const relationPeers = await this.database.from("relations").innerJoin("refresh_state", {
       "refresh_state.entity_ref": "relations.target_entity_ref"
     }).where("relations.originating_entity_id", "=", uid).andWhere("refresh_state.entity_id", "!=", uid).select({ ref: "relations.target_entity_ref" }).union(
@@ -2614,32 +2645,17 @@ class DefaultEntitiesCatalog {
     };
   }
   async facets(request) {
-    const { entities } = await this.entities({
-      filter: request.filter,
-      authorizationToken: request.authorizationToken
-    });
     const facets = {};
+    const db = this.database;
     for (const facet of request.facets) {
-      const values = entities.map((entity) => {
-        var _a, _b;
-        if (facet.startsWith("metadata.annotations.")) {
-          return (_a = entity.metadata.annotations) == null ? void 0 : _a[facet.substring("metadata.annotations.".length)];
-        } else if (facet.startsWith("metadata.labels.")) {
-          return (_b = entity.metadata.labels) == null ? void 0 : _b[facet.substring("metadata.labels.".length)];
-        }
-        return lodash__default["default"].get(entity, facet);
-      }).flatMap((field) => {
-        if (typeof field === "string") {
-          return [field];
-        } else if (Array.isArray(field)) {
-          return field.filter((i) => typeof i === "string");
-        }
-        return [];
-      }).sort();
-      const counts = lodash__default["default"].countBy(values, lodash__default["default"].identity);
-      facets[facet] = Object.entries(counts).map(([value, count]) => ({
-        value,
-        count
+      const dbQuery = db("search").join("final_entities", "search.entity_id", "final_entities.entity_id").where("search.key", facet.toLocaleLowerCase("en-US")).count("search.entity_id as count").select({ value: "search.original_value" }).groupBy("search.original_value");
+      if (request == null ? void 0 : request.filter) {
+        parseFilter(request.filter, dbQuery, db, false, "search.entity_id");
+      }
+      const result = await dbQuery;
+      facets[facet] = result.map((data) => ({
+        value: String(data.value),
+        count: Number(data.count)
       }));
     }
     return { facets };
@@ -3082,15 +3098,30 @@ function mapToRows(input, entityId) {
   for (const { key: rawKey, value: rawValue } of input) {
     const key = rawKey.toLocaleLowerCase("en-US");
     if (rawValue === void 0 || rawValue === null) {
-      result.push({ entity_id: entityId, key, value: null });
+      result.push({
+        entity_id: entityId,
+        key,
+        original_value: null,
+        value: null
+      });
     } else {
       const value = String(rawValue).toLocaleLowerCase("en-US");
       if (key.length <= MAX_KEY_LENGTH) {
-        result.push({
-          entity_id: entityId,
-          key,
-          value: value.length <= MAX_VALUE_LENGTH ? value : null
-        });
+        if (value.length <= MAX_VALUE_LENGTH) {
+          result.push({
+            entity_id: entityId,
+            key,
+            original_value: String(rawValue),
+            value
+          });
+        } else {
+          result.push({
+            entity_id: entityId,
+            key,
+            original_value: null,
+            value: null
+          });
+        }
       }
     }
   }
@@ -3808,13 +3839,20 @@ class Connection {
       });
     } else if (mutation.type === "delta") {
       this.check(mutation.added.map((e) => e.entity));
-      this.check(mutation.removed.map((e) => e.entity));
+      this.check(
+        mutation.removed.map((e) => "entity" in e ? e.entity : void 0).filter((e) => Boolean(e))
+      );
       await db.transaction(async (tx) => {
         await db.replaceUnprocessedEntities(tx, {
           sourceKey: this.config.id,
           type: "delta",
           added: mutation.added,
-          removed: mutation.removed
+          removed: mutation.removed.map(
+            (r) => "entityRef" in r ? r : {
+              entityRef: catalogModel.stringifyEntityRef(r.entity),
+              locationKey: r.locationKey
+            }
+          )
         });
       });
     }
@@ -4444,13 +4482,13 @@ const catalogPlugin = backendPluginApi.createBackendPlugin({
     );
     env.registerInit({
       deps: {
-        logger: backendPluginApi.loggerServiceRef,
-        config: backendPluginApi.configServiceRef,
-        reader: backendPluginApi.urlReaderServiceRef,
-        permissions: backendPluginApi.permissionsServiceRef,
-        database: backendPluginApi.databaseServiceRef,
-        httpRouter: backendPluginApi.httpRouterServiceRef,
-        lifecycle: backendPluginApi.lifecycleServiceRef
+        logger: backendPluginApi.coreServices.logger,
+        config: backendPluginApi.coreServices.config,
+        reader: backendPluginApi.coreServices.urlReader,
+        permissions: backendPluginApi.coreServices.permissions,
+        database: backendPluginApi.coreServices.database,
+        httpRouter: backendPluginApi.coreServices.httpRouter,
+        lifecycle: backendPluginApi.coreServices.lifecycle
       },
       async init({
         logger,